Badges
+
+
+
+
+
+
+
+
+
+
+
+
+
+-
+
- Versions + ${config.versions.active + .map( + (version) => ` +
- + ${version.slug} + + `, + ) + .join("\n")} +
-
+
- Downloads + ${Object.entries(config.versions.current.downloads) + .map( + ([name, url]) => ` +
- + ${downloadsNameDisplay[name]} + + `, + ) + .join("\n")} +
+
+ Read the Docs
+ v: ${config.versions.current.slug}
+
+
+ Badges — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Documentation-Documentation — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Developer Usage — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.freapp module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.cdoTimeAverager module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.frenctoolsTimeAverager module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.frepytoolsTimeAverager module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.generate_time_averages module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.tests.test_generate_time_averages module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.timeAverager module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.mask_atmos_plevel module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.regrid_xy package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.regrid_xy.regrid_xy module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.catalog.frecatalog module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.catalog package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.catalog.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.catalog.tests.test_fre_catalog module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.check.frecheck module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.check.frecheckexample module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.check package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.cmor_mixer module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.frecmor module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.tests.test_cmor_run_subtool module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.fre module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.lazy_group module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.list.frelist module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.list.frelistexample module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.list package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.createCheckout module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.createCompile module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.createDocker module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.createMakefile module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.fremake module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.runFremake module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.tests.test_create_makefile module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.checkoutScript module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.configure_script_xml module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.configure_script_yaml module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.frepp module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.install module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.run module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.status module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.tests.test_configure_script_yaml module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.tests.test_rose_quoting module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.validate module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.wrapper module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.run.frerun module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.run.frerunexample module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.run package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.test.fretest module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.test.fretestexample module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.test package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_app_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_catalog_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_check_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_cmor_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_list_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_make_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_pp_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_run_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_test_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_yamltools_cli module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.combine_yamls module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.freyamltools module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.freyamltoolsexample module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.tests package — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.tests.test_combine_yamls module — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Index — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Welcome to Fre-Cli’s documentation! — Fre-Cli 1.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ tags in the XML. There is one special
+ # script that contains its own key-value pair: PREANALYSIS. Everything
+ # else gets processed into a list of strings under the REFINEDIAG_SCRIPT
+ # rose-suite setting. Also, if a script referenced can also be found in
+ # Canopy's centralized refineDiag repository, located in the
+ # $CYLC_WORKFLOW_DIR/etc/refineDiag, then THOSE references will be used.
+ # Otherwise, the reference will be whatever path the XML finds.
+ ##########################################################################
+ #preanalysis_path_xml = None #notyetimplemented
+ #preanalysis_path_cylc = "'{}/{}'".format(CYLC_REFINED_DIR, #notyetimplemented
+ # PREANALYSIS_SCRIPT)
+
+ # get the refinediag scripts
+ refine_diag_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} " \
+ "--evaluate postProcess/refineDiag/@script"
+ refine_diag_process = subprocess.run(refine_diag_cmd,
+ shell=True,
+ check=True,
+ capture_output=True,
+ universal_newlines=True)
+ refine_diag_scripts = refine_diag_process.stdout.strip('\n')
+
+ # If one of the refinediag scripts contains "vitals", assume it
+ # won't generate output, so relabel it as a preAnalysis script.
+ # There is only one preAnalysis slot currently, so if there are multiple
+ # refineDiag scripts that match "vitals" throw away the rest.
+ list_refinediags = []
+ str_preanalysis = None
+ for x in refine_diag_scripts.split():
+ if "vitals" in x:
+ if str_preanalysis is None:
+ str_preanalysis = x
+ else:
+ list_refinediags.append(x)
+
+ if list_refinediags:
+ # turn refinediag off by default in favor of the built-in plevel masking
+ rose_suite.set(keys=['template variables', 'DO_REFINEDIAG'], value='False')
+ if dual:
+ rose_suite.set(keys=['template variables', '#HISTORY_DIR_REFINED'],
+ value=f"'{historydir_refined}_canopy'")
+ else:
+ rose_suite.set(keys=['template variables', '#HISTORY_DIR_REFINED'],
+ value=f"'{historydir_refined}'")
+
+ rose_suite.set(keys=['template variables', '#REFINEDIAG_SCRIPTS'],
+ value=f"'{' '.join(list_refinediags)}'")
+
+ logging.info( "refineDiag scripts: %s", ' '.join(list_refinediags) )
+ logging.info( "NOTE: Now turned off by default; please re-enable in config file if needed" )
+ else:
+ rose_suite.set(keys=['template variables', 'DO_REFINEDIAG'], value='False')
+ logging.info("No refineDiag scripts written. ")
+
+ if str_preanalysis is not None:
+ rose_suite.set(keys=['template variables', 'DO_PREANALYSIS'], value='True')
+ rose_suite.set(keys=['template variables', 'PREANALYSIS_SCRIPT'],
+ value=f"'{str_preanalysis}'" )
+ logging.info( "Preanalysis script: %s", str_preanalysis )
+ else:
+ rose_suite.set(keys=['template variables', 'DO_PREANALYSIS'], value='False')
+ logging.info( "No preAnalysis scripts written." )
+
+ # Grab all of the necessary PP component items/elements from the XML
+ comps = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'postProcess/component/@type').split()
+ rose_suite.set(keys=['template variables', 'PP_COMPONENTS'],
+ value=f"'{' '.join(sorted(comps))}'" )
+
+ segment_time = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'runtime/production/segment/@simTime')
+ segment_units = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'runtime/production/segment/@units')
+
+ if segment_units == 'years':
+ segment = f'P{segment_time}Y'
+ elif segment_units == 'months':
+ segment = f'P{segment_time}M'
+ else:
+ logging.error("Unknown segment units: %s", segment_units)
+ raise ValueError
+
+ # P12M is identical to P1Y but the latter looks nicer
+ if segment == 'P12M':
+ segment = 'P1Y'
+ rose_suite.set(keys=['template variables', 'HISTORY_SEGMENT'], value=f"'{segment}'" )
+
+ # Get the namelist current_date as the likely PP_START (unless "start" is used in the PP tags)
+ # frelist --namelist may be better, but sometimes may not work
+ current_date_str = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'input/namelist')
+ match = re.search(r'current_date\s*=\s*(\d+),(\d+),(\d+)', current_date_str)
+ if match:
+ try:
+ current_date = metomi.isodatetime.data.TimePoint(
+ year=match.group(1), month_of_year=match.group(2), day_of_month=match.group(3) )
+ except:
+ logging.warning("Could not parse date from namelist current_date")
+ current_date = None
+ else:
+ current_date = None
+ logging.warning("Could not find current_date in namelists")
+ logging.info("current_date (from namelists): %s", current_date)
+
+ # Take a good guess for the PP_START and PP_STOP
+ # PP_START could be the coupler_nml/current_date
+ # PP_STOP could be the PP_START plus the simulation length
+ if sim_units == "years":
+ oneless = int(sim_time) - 1
+ duration = f"P{oneless}Y"
+ elif sim_units == "months":
+ duration = f"P{sim_time}M"
+ else:
+ raise ValueError(f"Was hoping sim_units would be years or months; got {sim_units}")
+ dur = metomi.isodatetime.parsers.DurationParser().parse(duration)
+ pp_stop = current_date + dur
+ rose_suite.set(keys=['template variables', 'PP_START'], value=f'"{current_date}"')
+ rose_suite.set(keys=['template variables', 'PP_STOP'], value=f'"{pp_stop}"')
+
+ # Loop over all of the PP components, fetching the sources, xyInterp,
+ # and sourceGrid.
+ chunks = set()
+ comp_count = 0
+ for comp in comps:
+ comp_count += 1
+ pp_comp_xpath_header = f'postProcess/component[@type="{comp}"]'
+ logging.info( "Component loop: %s out of %s", comp_count, len(comps))
+
+ # get the comp attributes
+ comp_source = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/@source' )
+ xy_interp = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/@xyInterp' )
+ source_grid = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/@sourceGrid' )
+ interp_method = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/@interpMethod')
+
+ # split some of the stuffs
+ if xy_interp != "":
+ interp_split = xy_interp.split(',')
+ output_grid_lon = interp_split[1]
+ output_grid_lat = interp_split[0]
+ if source_grid != "":
+ sourcegrid_split = source_grid.split('-')
+ input_grid = sourcegrid_split[1]
+ input_realm = sourcegrid_split[0]
+
+ # determine the interp method
+ if xy_interp:
+ if interp_method == "":
+ if input_grid == "cubedsphere":
+ interp_method = 'conserve_order2'
+ elif input_grid == 'tripolar':
+ interp_method = 'conserve_order1'
+ else:
+ raise Exception(f"Expected cubedsphere or tripolar, not {source_grid}")
+
+ # determine the grid label
+ if xy_interp:
+ grid = f"regrid-xy/{output_grid_lon}_{output_grid_lat}.{interp_method}"
+ grid_tail = f"{output_grid_lon}_{output_grid_lat}.{interp_method}"
+ else:
+ grid = "native"
+
+ sources = set()
+ if comp_source.endswith('_refined'):
+ logging.info(
+ "NOTE: Skipping history file %s, refineDiag is turned off by default.", comp_source)
+ else:
+ sources.add(comp_source)
+ timeseries_count = 0
+
+ # Get the number of TS nodes
+ results = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/timeSeries/@freq' ).split()
+ timeseries_count = len(results)
+
+ # Loop over the TS nodes and write out the frequency, chunklength, and
+ # grid to the remap-pp-components Rose app configuration
+ for i in range(1, timeseries_count + 1):
+ #label = "{}.{}".format(comp, str(i)) #notyetimplemented
+
+ source = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/timeSeries[{i}]/@source' )
+ if source.endswith('_refined'):
+ logging.info(
+ "NOTE: Skipping history file %s, refineDiag is turned off by default.", source)
+ else:
+ sources.add(source)
+
+ #freq = freq_from_legacy(frelist_xpath(args,
+ # '{}/timeSeries[{}]/@freq' \
+ # .format(pp_comp_xpath_header, i)))
+ chunk = chunk_from_legacy(
+ frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir,
+ #do_refinediag, pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ f'{pp_comp_xpath_header}/timeSeries[{i}]/@chunkLength'
+ ) )
+ chunks.add(chunk)
+ #rose_remap.set(keys=[label, 'freq'], value=freq)
+ #rose_remap.set(keys=[label, 'chunk'], value=chunk)
+
+ rose_remap.set(keys=[comp, 'sources'], value=' '.join(sources))
+ rose_remap.set(keys=[comp, 'grid'], value=grid)
+
+ if grid == "native":
+ pass
+ else:
+ # Write out values to the 'regrid-xy' Rose app
+ rose_regrid_xy.set(keys=[comp, 'sources'], value=' '.join(sources))
+ rose_regrid_xy.set(keys=[comp, 'inputGrid'], value=input_grid)
+ rose_regrid_xy.set(keys=[comp, 'inputRealm'], value=input_realm)
+ rose_regrid_xy.set(keys=[comp, 'interpMethod'], value=interp_method)
+ rose_regrid_xy.set(keys=[comp, 'outputGridType'], value=grid_tail)
+ rose_regrid_xy.set(keys=[comp, 'outputGridLon'], value=output_grid_lon)
+ rose_regrid_xy.set(keys=[comp, 'outputGridLat'], value=output_grid_lat)
+
+ # Process all of the found PP chunks into the rose-suite configuration
+ if verbose:
+ print("")
+ logging.info("Setting PP chunks...")
+
+ sorted_chunks = list(chunks)
+ sorted_chunks.sort(key=duration_to_seconds, reverse=False)
+
+ if len(chunks) == 0:
+ raise ValueError('no chunks found! exit.')
+
+ logging.info(" Chunks found: %s", ', '.join(sorted_chunks))
+ if len(chunks) == 1:
+ rose_suite.set(['template variables', 'PP_CHUNK_A'],
+ f"'{sorted_chunks[0]}'")
+ else:
+ rose_suite.set(['template variables', 'PP_CHUNK_A'],
+ f"'{sorted_chunks[0]}'")
+ rose_suite.set(['template variables', 'PP_CHUNK_B'],
+ f"'{sorted_chunks[1]}'")
+ logging.info(" Chunks used: %s", ', '.join(sorted_chunks[0:2]) )
+
+ # Write out the final configurations.
+ if verbose:
+ print("")
+ logging.info("Writing output files...")
+
+ dumper = metomi.rose.config.ConfigDumper()
+ rose_outfiles=[ ( rose_suite, 'rose-suite.conf' ),
+ ( rose_remap, 'app/remap-pp-components/rose-app.conf' ),
+ ( rose_regrid_xy, 'app/regrid-xy/rose-app.conf' ) ]
+ for outfile in rose_outfiles:
+ logging.info(" %s", outfile[1])
+ dumper(outfile[0], outfile[1])
+
+ #outfile = "app/remap-pp-components/rose-app.conf"
+ #logging.info(" %s", outfile)
+ #dumper(rose_remap, outfile)
+ #
+ #outfile = "app/regrid-xy/rose-app.conf"
+ #logging.info(" %s", outfile)
+ #dumper(rose_regrid_xy, outfile)
+
+def _convert(xml, platform, target, experiment, do_analysis=False, historydir=None,
+ refinedir=None, ppdir=None, do_refinediag=False, pp_start=None,
+ pp_stop=None, validate=False, verbose=False, quiet=False, dual=False):
+ """
+ Converts a Bronx XML to a Canopy rose-suite.conf
+ """
+
+ # Logging settings. The default is throwing only warning messages
+ if verbose:
+ logging.basicConfig(level=logging.INFO, format=LOGGING_FORMAT)
+ elif quiet:
+ logging.basicConfig(level=logging.ERROR, format=LOGGING_FORMAT)
+ else:
+ logging.basicConfig(level=logging.WARNING, format=LOGGING_FORMAT)
+
+ # Set the name of the directory
+ name = f"{experiment}__{platform}__{target}"
+
+ # Create the directory if it doesn't exist
+ cylc_dir = os.path.expanduser("~/cylc-src")
+ new_dir = os.path.join(cylc_dir, name)
+ os.makedirs(new_dir, exist_ok=True)
+
+ # Change the current working directory
+ xml = os.path.abspath(xml)
+ os.chdir(new_dir)
+
+ ##########################################################################
+ # NOTE: Ideally we would check that "frelist" is in the PATH here,
+ # but in practice the frelist will just fail later if it's not loaded.
+ # If that error scenario is too confusing, let's add a check back here.
+ cylc_loaded = False
+ if validate:
+ if CYLC_PATH in os.getenv('PATH'):
+ cylc_loaded = True
+ else:
+ raise EnvironmentError("Cannot run the validator tool because " \
+ "the Cylc module isn't loaded. Please " \
+ "run 'module load cylc/test' and try again.")
+
+ # Alert the user if only 1 or zero PP years are given as an option, and
+ # notify them that a default of '0000' for those years will be set in the
+ # rose-suite configuration
+ if any( [ pp_stop is None, pp_start is None ] ):
+ if pp_start is None:
+ logging.warning("PP start year was not specified.")
+ if pp_stop is None:
+ logging.warning("PP stop year was not specified.")
+ logging.warning("After the converter has run, please edit the " \
+ "default '0000' values within your rose-suite.conf.")
+
+ # These series of conditionals takes into account input from the user
+ # (for the PP_START and PP_STOP year) that is not 4 digits or other
+ # nonsensical years. The rose-suite config requires 4 digits for years
+ # and if the year is under '1000' (but > 0), then leading zeros must be used.
+ if all( [ pp_start is not None, pp_stop is not None ] ):
+ def format_req_pp_year(pp_year):
+ if len(pp_year) < 4 and int(pp_year) > 0:
+ pp_year = '0' * (4 - len(pp_year)) + pp_year
+ return pp_year
+ pp_start = format_req_pp_year(pp_start)
+ pp_stop = format_req_pp_year(pp_stop)
+
+ if int(pp_start) >= int(pp_stop):
+ logging.warning("Your PP_START date is equal to or later than " \
+ "your PP_STOP date. Please revise these values in " \
+ "your configuration after the converter has run.")
+ if any( [ len(pp_start) > 4, len(pp_stop) > 4,
+ int(pp_start) <= 0, int(pp_stop) <= 0 ] ):
+ logging.warning("At least one of your PP_start or PP_stop years " \
+ "does not make sense. Please revise this value in " \
+ "your configuration after the converter has run.")
+
+ main( xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir,
+ do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual )
+
+ if verbose:
+ print("")
+ logging.info("XML conversion complete!")
+
+ # Run the Cylc validator tool on the current directory if conditions are met.
+ # Note: the user must be running the converter in the parent Cylc Workflow
+ # Directory if the validator is run.
+ if cylc_loaded:
+ if verbose:
+ print("")
+ logging.info("Running the Cylc validator tool...")
+ try:
+ subprocess.run("cylc validate .", shell=True, check=True)
+ except subprocess.CalledProcessError:
+ logging.error("Errant values in rose-suite.conf or other Cylc errors. " \
+ "Please check your configuration and run the validator " \
+ "again separately.")
+ finally:
+ logging.info("Validation step complete!")
+
+##############################################
+
+@click.command()
+def convert():
+ '''
+ Wrapper for convert call - allows users to call without command-line args
+ '''
+ _convert()
+
+if __name__ == '__main__':
+ convert()
diff --git a/lib/fre/pp/configure_script_yaml.py b/lib/fre/pp/configure_script_yaml.py
new file mode 100644
index 00000000..b782e3de
--- /dev/null
+++ b/lib/fre/pp/configure_script_yaml.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+"""
+ Script creates rose-apps and rose-suite
+ files for the workflow from the pp yaml.
+"""
+
+## TO-DO:
+# - condition where there are multiple pp yamls
+# - validation here or in combined-yamls tools
+
+import os
+import json
+import shutil
+import click
+
+from jsonschema import validate
+import yaml
+import metomi.rose.config
+
+# Relative import
+#f = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+#sys.path.append(f)
+import fre.yamltools.combine_yamls as cy
+
+####################
+def yaml_load(yamlfile):
+ """
+ Load the given yaml and validate.
+ """
+ # Load the main yaml and validate
+ with open(yamlfile,'r') as f:
+ y=yaml.safe_load(f)
+
+ return y
+
+######VALIDATE#####
+def validate_yaml(yamlfile):
+ """
+ Using the schema.json file, the yaml format is validated.
+ """
+ # Load the yaml
+ yml = yaml_load(yamlfile)
+
+ package_dir = os.path.dirname(os.path.abspath(__file__))
+ schema_path = os.path.join(package_dir, 'schema.json')
+ # Load the json schema: .load() (vs .loads()) reads and parses the json in one
+ with open(schema_path,'r') as s:
+ schema = json.load(s)
+
+ # Validate yaml
+ # If the yaml is not valid, the schema validation will raise errors and exit
+ if validate(instance=yml,schema=schema) is None:
+ print("COMBINED YAML VALID \n")
+
+####################
+def rose_init(experiment,platform,target):
+ """
+ Initialize the rose suite and app configurations.
+ """
+ # initialize rose suite config
+ rose_suite = metomi.rose.config.ConfigNode()
+ # disagreeable; these should be optional
+ rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False')
+ rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False')
+ rose_suite.set(keys=['template variables', 'PP_DEFAULT_XYINTERP'], value='0,0')
+
+ # set some rose suite vars
+ rose_suite.set(keys=['template variables', 'EXPERIMENT'], value=f'"{experiment}"')
+ rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{platform}"')
+ rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{target}"')
+
+ # initialize rose regrid config
+ rose_regrid = metomi.rose.config.ConfigNode()
+ rose_regrid.set(keys=['command', 'default'], value='regrid-xy')
+
+ # initialize rose remap config
+ rose_remap = metomi.rose.config.ConfigNode()
+ rose_remap.set(keys=['command', 'default'], value='remap-pp-components')
+
+ return(rose_suite,rose_regrid,rose_remap)
+
+####################
+def quote_rose_values(value):
+ """
+ rose-suite.conf template variables must be quoted unless they are
+ boolean, in which case do not quote them.
+ """
+ if isinstance(value, bool):
+ return f"{value}"
+ else:
+ return "'" + value + "'"
+
+####################
+def set_rose_suite(yamlfile,rose_suite):
+ """
+ Set items in the rose suite configuration.
+ """
+ pp=yamlfile.get("postprocess")
+ dirs=yamlfile.get("directories")
+
+ # set rose-suite items
+ if pp is not None:
+ for i in pp.values():
+ if not isinstance(i,list):
+ for key,value in i.items():
+ # rose-suite.conf is somewhat finicky with quoting
+ # cylc validate will reveal any complaints
+ rose_suite.set( keys = ['template variables', key.upper()],
+ value = quote_rose_values(value) )
+ if dirs is not None:
+ for key,value in dirs.items():
+ rose_suite.set(keys=['template variables', key.upper()], value=quote_rose_values(value))
+
+####################
+def set_rose_apps(yamlfile,rose_regrid,rose_remap):
+ """
+ Set items in the regrid and remap rose app configurations.
+ """
+ components = yamlfile.get("postprocess").get("components")
+ for i in components:
+ comp = i.get('type')
+ sources = i.get('sources')
+ interp_method = i.get('interpMethod')
+
+ # set remap items
+ rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}')
+ # if xyInterp doesnt exist, grid is native
+ if i.get("xyInterp") is None:
+ rose_remap.set(keys=[f'{comp}', 'grid'], value='native')
+
+ # if xyInterp exists, component can be regridded
+ else:
+ interp_split = i.get('xyInterp').split(',')
+ rose_remap.set(keys=[f'{comp}', 'grid'],
+ value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}')
+
+ # set regrid items
+ if i.get("xyInterp") is not None:
+ rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}')
+ rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}')
+ rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}')
+ rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}')
+ interp_split = i.get('xyInterp').split(',')
+ rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}')
+ rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}')
+ rose_regrid.set(keys=[f'{comp}', 'outputGridType'],
+ value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}')
+
+####################
+def yamlInfo(yamlfile,experiment,platform,target):
+ """
+ Using a valid pp.yaml, the rose-app and rose-suite
+ configuration files are created in the cylc-src
+ directory. The pp.yaml is also copied to the
+ cylc-src directory.
+ """
+ e = experiment
+ p = platform
+ t = target
+ yml = yamlfile
+
+ # Initialize the rose configurations
+ rose_suite,rose_regrid,rose_remap = rose_init(e,p,t)
+
+ # Combine model, experiment, and analysis yamls
+ comb = cy.init_pp_yaml(yml,e,p,t)
+ full_combined = cy.get_combined_ppyaml(comb)
+
+ # Validate yaml
+ validate_yaml(full_combined)
+
+ # Load the combined yaml
+ comb_pp_yaml = yaml_load(full_combined)
+
+ ## PARSE COMBINED YAML TO CREATE CONFIGS
+ # Set rose-suite items
+ set_rose_suite(comb_pp_yaml,rose_suite)
+
+ # Set regrid and remap rose app items
+ set_rose_apps(comb_pp_yaml,rose_regrid,rose_remap)
+
+ # write output files
+ print("Writing output files...")
+ cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}")
+ outfile = os.path.join(cylc_dir, f"{e}.yaml")
+ shutil.copyfile(full_combined, outfile)
+ print(" " + outfile)
+
+ dumper = metomi.rose.config.ConfigDumper()
+ outfile = os.path.join(cylc_dir, "rose-suite.conf")
+ dumper(rose_suite, outfile)
+ print(" " + outfile)
+
+ outfile = os.path.join(cylc_dir, "app", "regrid-xy", "rose-app.conf")
+ dumper(rose_regrid, outfile)
+ print(" " + outfile)
+
+ outfile = os.path.join(cylc_dir, "app", "remap-pp-components", "rose-app.conf")
+ dumper(rose_remap, outfile)
+ print(" " + outfile)
+
+@click.command()
+def _yamlInfo(yamlfile,experiment,platform,target):
+ '''
+ Wrapper script for calling yamlInfo - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return yamlInfo(yamlfile,experiment,platform,target)
+
+# Use parseyaml function to parse created edits.yaml
+if __name__ == '__main__':
+ yamlInfo()
diff --git a/lib/fre/pp/edits-template.yaml b/lib/fre/pp/edits-template.yaml
new file mode 100644
index 00000000..9bedfe71
--- /dev/null
+++ b/lib/fre/pp/edits-template.yaml
@@ -0,0 +1,271 @@
+## User defined edits
+
+# Paths
+configuration paths:
+ # Path to where rose-suite experiment configuration should be in pp repo
+ rose-suite: "opt/"
+ # Path to remap-pp-components rose-app.conf in pp repo
+ rose-remap: "app/remap-pp-components/rose-app.conf"
+ # Path to regrid rose-app.conf in pp repo
+ rose-regrid: "app/regrid-xy/rose-app.conf"
+
+# Define widely used variables
+# Length of analysis of experiment, string in double quotes
+ana_amip_len: &ANA_AMIP_LEN ""
+# String in double quotes
+pp_amip_chunk: &PP_AMIP_CHUNK96 ""
+# Year to start analysis on experiment, string in double quotes
+ana_amip_start: &ANA_AMIP_START ""
+# Year to stop analysis on experiment, string in double quotes
+ana_amip_stop: &ANA_AMIP_STOP ""
+# Default xy Interp, string in double quotes
+defaultyInterp: &defaultxyInterp ""
+
+# Information for rose-suite experiment configuration
+rose-suite:
+ settings:
+ # ISO8601 duration of the length of the history segment
+ history_segment:
+ # ISO8601 date to begin postprocessing
+ pp_start:
+ # ISO8601 date to stop postprocessing
+ pp_stop:
+ # Length of experiment
+ len:
+ # Specific site used for workflow; must be 'generic', 'gfdl-ws' or 'ppan'
+ site:
+ # Location for shared analysis scripts
+ fre_analysis_home:
+ # ISO8601 duration of the desired post-processed output
+ pp_chunk_a:
+ # ISO8601 duration of a second desired postprocessed output
+ pp_chunk_b:
+ # Components to be post-processed
+ pp_components:
+ # Default regridded resolution
+ pp_default_xyinterp: *defaultxyInterp
+ # Filepath to an FMS gridSpec netCDF file or a tarfile
+ pp_grid_spec:
+
+ switches:
+ # Switch to remove intermediate data files when they are no longer needed
+ clean_work:
+ # Switch to run MDTF on generated pp output
+ do_mdtf:
+ # Switch to turn on/off statics processing
+ do_statics:
+ # Switch to turn on/off time-average file generation
+ do_timeavgs:
+ # Switch to run refine-diag script(s) on history file to generate additional diagnostics
+ do_refinediag:
+ # Switch to mask atmos pressure-level output above/below surface pressure/atmos top
+ do_atmos_plevel_masking:
+ # Switch to run a pre-analysis script on history files
+ do_preanalysis:
+ do_analysis:
+ do_analysis_only:
+
+ directories:
+ # Directory to the primary location for history files
+ history_dir:
+ # Directory to write pp products to
+ pp_dir:
+ # Directory to use for history file cache
+ ptmp_dir:
+ # Filepath to refineDiag scripts
+ refinediag_scripts:
+ # Filepath to the user script
+ preanalysis_script:
+ # Directory to the secondary location for history files
+ history_dir_refined:
+ # Directory to write analysis output
+ analysis:
+
+# Information needed to populate the rose-apps for regrid-xy and remap-pp-components
+# These rose apps will be retired eventually
+components:
+ # Name of component
+ - type:
+ # Analysis start
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ # Default regridded grid
+ xyInterp: *defaultxyInterp
+ # Should be 'conserve_order1', 'conserve_order2' or 'bilinear'
+ interpMethod:
+ timeSeries:
+ - freq:
+ source:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ source:
+ chunkLength: *PP_AMIP_CHUNK96
+ variables:
+ - freq:
+ source:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ calcInterval: *PP_AMIP_CHUNK96
+ - type:
+ zInterp:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ source:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ calcInterval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ xyInterp:
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ source:
+ chunkLength: *PP_AMIP_CHUNK96
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - type:
+ start: *ANA_AMIP_START
+ xyInterp: *defaultxyInterp
+ sources:
+ sourceGrid:
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ source:
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+ timeAverage:
+ - source:
+ interval: *PP_AMIP_CHUNK96
+ - source:
+ interval: *ANA_AMIP_LEN
+ - type:
+ start: *ANA_AMIP_START
+ sources:
+ sourceGrid:
+ xyInterp: *defaultxyInterp
+ interpMethod:
+ cmip:
+ timeSeries:
+ - freq:
+ chunkLength: *PP_AMIP_CHUNK96
+
+# TMP DIRECTORY CREATION
+# This edit is not needed when working on pp/an. If not working on pp/an, a tmpDir must be defined.
+tmpdir:
+ tmpdirpath:
+
+# Optional edit: Add --symlink-dirs option to install cylc-run directory to location with more space available; default installation location for cylc-run directory is $HOME;
+# Utilize this edit if you do not have enough space in home directory
+install-exp-script:
+ - path:
+ - install-option:
+ install:
diff --git a/lib/fre/pp/frepp.py b/lib/fre/pp/frepp.py
new file mode 100644
index 00000000..b3456e31
--- /dev/null
+++ b/lib/fre/pp/frepp.py
@@ -0,0 +1,213 @@
+''' fre pp '''
+
+import click
+from .checkoutScript import checkoutTemplate
+from .configure_script_yaml import yamlInfo
+from .configure_script_xml import convert
+from .validate import validate_subtool
+from .install import install_subtool
+from .run import pp_run_subtool
+from .status import status_subtool
+from .wrapper import runFre2pp
+
+@click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210)))
+def pp_cli():
+ ''' entry point to fre pp click commands '''
+
+
+# fre pp status
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.pass_context
+def status(context, experiment, platform, target):
+ # pylint: disable=unused-argument
+ """ - Report status of PP configuration"""
+ context.forward(status_subtool)
+
+# fre pp run
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.pass_context
+def run(context, experiment, platform, target):
+ # pylint: disable=unused-argument
+ """ - Run PP configuration"""
+ context.forward(pp_run_subtool)
+
+# fre pp validate
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.pass_context
+def validate(context, experiment, platform, target):
+ # pylint: disable=unused-argument
+ """ - Validate PP configuration"""
+ context.forward(validate_subtool)
+
+# fre pp install
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.pass_context
+def install(context, experiment, platform, target):
+ # pylint: disable=unused-argument
+ """ - Install PP configuration"""
+ context.forward(install_subtool)
+
+@pp_cli.command()
+@click.option("-y", "--yamlfile", type=str,
+ help="YAML file to be used for parsing",
+ required=True)
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.pass_context
+def configure_yaml(context,yamlfile,experiment,platform,target):
+ # pylint: disable=unused-argument
+ """ - Execute fre pp configure """
+ context.forward(yamlInfo)
+
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.option("-b", "--branch",
+ show_default=True,
+ default="main", type=str,
+ help="Name of fre2/workflows/postproc branch to clone; " \
+ "defaults to 'main'. Not intended for production use, " \
+ "but needed for branch testing." )
+@click.pass_context
+def checkout(context, experiment, platform, target, branch='main'):
+ # pylint: disable=unused-argument
+ """ - Execute fre pp checkout """
+ context.forward(checkoutTemplate)
+
+@pp_cli.command()
+@click.option('-x', '--xml',
+ required=True,
+ help="Required. The Bronx XML")
+@click.option('-p', '--platform',
+ required=True,
+ help="Required. The Bronx XML Platform")
+@click.option('-t', '--target',
+ required=True,
+ help="Required. The Bronx XML Target")
+@click.option('-e', '--experiment',
+ required=True,
+ help="Required. The Bronx XML Experiment")
+@click.option('--do_analysis',
+ is_flag=True,
+ default=False,
+ help="Optional. Runs the analysis scripts.")
+@click.option('--historydir',
+ help="Optional. History directory to reference. " \
+ "If not specified, the XML's default will be used.")
+@click.option('--refinedir',
+ help="Optional. History refineDiag directory to reference. " \
+ "If not specified, the XML's default will be used.")
+@click.option('--ppdir',
+ help="Optional. Postprocessing directory to reference. " \
+ "If not specified, the XML's default will be used.")
+@click.option('--do_refinediag',
+ is_flag=True,
+ default=False,
+ help="Optional. Process refineDiag scripts")
+@click.option('--pp_start',
+ help="Optional. Starting year of postprocessing. " \
+ "If not specified, a default value of '0000' " \
+ "will be set and must be changed in rose-suite.conf")
+@click.option('--pp_stop',
+ help="Optional. Ending year of postprocessing. " \
+ "If not specified, a default value of '0000' " \
+ "will be set and must be changed in rose-suite.conf")
+@click.option('--validate',
+ is_flag=True,
+ help="Optional. Run the Cylc validator " \
+ "immediately after conversion")
+@click.option('-v', '--verbose',
+ is_flag=True,
+ help="Optional. Display detailed output")
+@click.option('-q', '--quiet',
+ is_flag=True,
+ help="Optional. Display only serious messages and/or errors")
+@click.option('--dual',
+ is_flag=True,
+ help="Optional. Append '_canopy' to pp, analysis, and refinediag dirs")
+@click.pass_context
+def configure_xml(context, xml, platform, target, experiment, do_analysis, historydir, refinedir,
+ ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual):
+ # pylint: disable=unused-argument
+ """ - Converts a Bronx XML to a Canopy rose-suite.conf """
+ context.forward(convert)
+
+#fre pp wrapper
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+ help="Experiment name",
+ required=True)
+@click.option("-p", "--platform", type=str,
+ help="Platform name",
+ required=True)
+@click.option("-t", "--target", type=str,
+ help="Target name",
+ required=True)
+@click.option("-c", "--config-file", type=str,
+ help="Path to a configuration file in either XML or YAML",
+ required=True)
+@click.option("-b", "--branch",
+ show_default=True,
+ default="main", type=str,
+ help="Name of fre2/workflows/postproc branch to clone; " \
+ "defaults to 'main'. Not intended for production use, " \
+ "but needed for branch testing." )
+@click.pass_context
+def wrapper(context, experiment, platform, target, config_file, branch='main'):
+ # pylint: disable=unused-argument
+ """ - Execute fre pp steps in order """
+ context.forward(runFre2pp)
+
+if __name__ == "__main__":
+ ''' entry point for click to fre pp commands '''
+ pp_cli()
diff --git a/lib/fre/pp/install.py b/lib/fre/pp/install.py
new file mode 100644
index 00000000..9ffc00ee
--- /dev/null
+++ b/lib/fre/pp/install.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+''' fre pp install '''
+
+import subprocess
+import click
+
+def _install_subtool(experiment, platform, target):
+ """
+ Install the Cylc workflow definition located in
+ ~/cylc-src/____
+ to
+ ~/cylc-run/____
+ """
+
+ name = experiment + '__' + platform + '__' + target
+ cmd = f"cylc install --no-run-name {name}"
+ subprocess.run(cmd, shell=True, check=True)
+
+@click.command()
+def install_subtool(experiment, platform, target):
+ ''' entry point to install for click '''
+ return _install_subtool(experiment, platform, target)
diff --git a/lib/fre/pp/run.py b/lib/fre/pp/run.py
new file mode 100644
index 00000000..57f2c427
--- /dev/null
+++ b/lib/fre/pp/run.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+''' fre pp run '''
+
+import subprocess
+import click
+
+def _pp_run_subtool(experiment, platform, target):
+ """
+ Start or restart the Cylc workflow identified by:
+ ____
+ """
+
+ name = experiment + '__' + platform + '__' + target
+ cmd = f"cylc play {name}"
+ subprocess.run(cmd, shell=True, check=True)
+
+@click.command()
+def pp_run_subtool(experiment, platform, target):
+ ''' entry point to run for click '''
+ return _pp_run_subtool(experiment, platform, target)
diff --git a/lib/fre/pp/schema.json b/lib/fre/pp/schema.json
new file mode 100644
index 00000000..dfb9cff5
--- /dev/null
+++ b/lib/fre/pp/schema.json
@@ -0,0 +1,75 @@
+{
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "Schema for PP Yaml",
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "platform": {"type": "string"},
+ "target": {"type": "string"},
+ "directories": {
+ "description": "FRE shared directories",
+ "type": "object",
+ "items":{"$ref": "#/$defs/dirs" }
+ },
+ "postprocess": {
+ "description": "FRE post-processing information",
+ "type": "object",
+ "items":{"$ref": "#/$defs/pp" }
+ }
+ },
+ "$defs": {
+ "dirs": {
+ "history_dir": {"type":"string"},
+ "pp_dir": {"type":"string"},
+ "ptmp_dir": {"type":"string"},
+ "refinediag_scripts":{"type":["string","null"]},
+ "preanalysis_script":{"type":["string","null"]},
+ "history_refined":{"type":["string","null"]},
+ "analysis_dir":{"type":["string","null"]},
+ "pp_grid_spec": {"type":"string"},
+ "fre_analysis_home": {"type":["string","null"]}
+ },
+ "pp": {
+ "type": "object",
+ "properties": {
+ "settings": {
+ "type:": "object",
+ "properties": {
+ "history_segment": {"type":"string"},
+ "site": {"type":"string"},
+ "pp_chunk_a": {"type":"string"},
+ "pp_chunk_b": {"type":"string"},
+ "pp_start": {"type":"string"},
+ "pp_stop": {"type":"string"},
+ "pp_components": {"type":"string"}
+ }
+ },
+ "switches": {
+ "type": "object",
+ "properties": {
+ "clean_work": {"type":"boolean"},
+ "do_mdtf": {"type":"boolean"},
+ "do_statics": {"type":"boolean"},
+ "do_timeavgs": {"type":"boolean"},
+ "do_refinediag": {"type":"boolean"},
+ "do_atmos_plevel_masking": {"type":"boolean"},
+ "do_preanalysis": {"type":"boolean"},
+ "do_analysis": {"type":"boolean"},
+ "do_analysis_only": {"type":"boolean"}
+ }
+ },
+ "components": {
+ "type": "array",
+ "properties": {
+ "type": {"type":"string"},
+ "sources": {"type":"string"},
+ "sourceGrid": {"type":"string"},
+ "xyInterp": {"type":"string"},
+ "interpMethod": {"type":"string"},
+ "inputRealm": {"type":"string"}
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/fre/pp/status.py b/lib/fre/pp/status.py
new file mode 100644
index 00000000..6e2c07e1
--- /dev/null
+++ b/lib/fre/pp/status.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+''' fre pp status '''
+
+import subprocess
+import click
+
+def _status_subtool(experiment, platform, target):
+ """
+ Report workflow state for the Cylc workflow
+ ____
+ """
+
+ name = experiment + '__' + platform + '__' + target
+ cmd = f"cylc workflow-state {name}"
+ subprocess.run(cmd, shell=True, check=True, timeout=30)
+
+
+@click.command()
+def status_subtool(experiment, platform, target):
+ ''' entry point to status for click '''
+ return _status_subtool(experiment, platform, target)
diff --git a/lib/fre/pp/tests/AM5_example/am5.yaml b/lib/fre/pp/tests/AM5_example/am5.yaml
new file mode 100644
index 00000000..4f3e5a4d
--- /dev/null
+++ b/lib/fre/pp/tests/AM5_example/am5.yaml
@@ -0,0 +1,115 @@
+# reusable variables
+fre_properties:
+ - &AM5_VERSION "am5f7b12r1"
+ - &FRE_STEM !join [am5/, *AM5_VERSION]
+
+ # amip
+ - &EXP_AMIP_START "19790101T0000Z"
+ - &EXP_AMIP_END "20200101T0000Z"
+ - &ANA_AMIP_START "19800101T0000Z"
+ - &ANA_AMIP_END "20200101T0000Z"
+
+ - &PP_AMIP_CHUNK96 "P1Y"
+ - &PP_AMIP_CHUNK384 "P1Y"
+ - &PP_XYINTERP96 "180,288"
+ - &PP_XYINTERP384 "720,1152"
+
+ # climo
+ - &EXP_CLIMO_START96 "0001"
+ - &EXP_CLIMO_END96 "0011"
+ - &ANA_CLIMO_START96 "0002"
+ - &ANA_CLIMO_END96 "0011"
+
+ - &EXP_CLIMO_START384 "0001"
+ - &EXP_CLIMO_END384 "0006"
+ - &ANA_CLIMO_START384 "0002"
+ - &ANA_CLIMO_END384 "0006"
+
+ # coupled
+ - &PP_CPLD_CHUNK_A "P5Y"
+ - &PP_CPLD_CHUNK_B "P20Y"
+
+ # grids
+ - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+
+ # compile information
+ - &release "f1a1r1"
+ - &INTEL "intel-classic"
+ - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+ - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+
+shared:
+ # compile information
+ compile:
+ compileYaml: &compile_yaml "compile.yaml"
+ platformYaml: "yaml_include/platforms.yaml"
+
+ # directories shared across tools
+ directories: &shared_directories
+ history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+ pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+ analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+ ptmp_dir: "/xtmp/$USER/ptmp"
+ fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
+
+ # shared pp settings
+ postprocess:
+ settings: &shared_settings
+ history_segment: "P1Y"
+ site: "ppan"
+ switches: &shared_switches
+ do_statics: True
+ do_timeavgs: True
+ clean_work: True
+ do_refinediag: False
+ do_atmos_plevel_masking: True
+ do_preanalysis: False
+ do_analysis: True
+
+experiments:
+ - name: "c96L65_am5f7b12r1_amip"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
+ compile: *compile_yaml
+ - name: "c96L65_am5f7b12r1_pdclim1850F"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ compile: *compile_yaml
+ - name: "c96L65_am5f7b12r1_pdclim2010F"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ compile: *compile_yaml
+ - name: "c96L65_am5f7b12r1_pdclim2010AERF"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ compile: *compile_yaml
+ - name: "c384L65_am5f7b12r1_amip"
+ pp:
+ - "yaml_include/pp.c384_amip.yaml"
+ compile: *compile_yaml
+ - name: "c384L65_am5f7b12r1_pdclim2010F"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ compile: *compile_yaml
+ - name: "c384L65_am5f7b12r1_pdclim1850F"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ compile: *compile_yaml
+ - name: "c384L65_am5f7b12r1_pdclim2010AERF"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ compile: *compile_yaml
+ - name: "c384L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg"
+ pp:
+ - "yaml_include/pp.c384_amip.yaml"
+ - "yaml_include/pp.om4.yaml"
+ compile: *compile_yaml
+ - name: "c96L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
+ - "yaml_include/pp.om4.yaml"
+ compile: *compile_yaml
+ - name: "c96L65_am5f7b12r1_amip_cosp"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
+ compile: *compile_yaml
diff --git a/lib/fre/pp/tests/AM5_example/compile.yaml b/lib/fre/pp/tests/AM5_example/compile.yaml
new file mode 100644
index 00000000..5f9a361b
--- /dev/null
+++ b/lib/fre/pp/tests/AM5_example/compile.yaml
@@ -0,0 +1,67 @@
+compile:
+ experiment: "am5"
+ container_addlibs:
+ baremetal_linkerflags:
+ src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF"
+ branch: "2022.01"
+ cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF"
+ otherFlags: *FMSincludes
+ - component: "am5_phys"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git"
+ branch: "2022.01"
+ otherFlags: *FMSincludes
+ - component: "GFDL_atmos_cubed_sphere"
+ requires: ["FMS", "am5_phys"]
+ repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML"
+ branch: "2022.01"
+ paths: ["GFDL_atmos_cubed_sphere/driver/GFDL",
+ "GFDL_atmos_cubed_sphere/model",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90",
+ "GFDL_atmos_cubed_sphere/tools",
+ "GFDL_atmos_cubed_sphere/GFDL_tools"]
+ otherFlags: *FMSincludes
+ - component: "atmos_drivers"
+ requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"]
+ repo: "https://github.com/NOAA-GFDL/atmos_drivers.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE"
+ branch: "2022.01"
+ paths: ["atmos_drivers/coupled"]
+ otherFlags: *FMSincludes
+ - component: "ice_sis"
+ requires: ["FMS", "ice_param", "mom6"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git"
+ branch: "2021.02"
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
+ - component: "ice_param"
+ repo: "https://github.com/NOAA-GFDL/ice_param.git"
+ cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF"
+ branch: "2021.02"
+ requires: ["FMS", "mom6"]
+ otherFlags: !join [*FMSincludes," ", *momIncludes]
+ - component: "land_lad2"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git"
+ branch: "2022.01"
+ branch: "land_lad2_2021.02"
+ doF90Cpp: True
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "mom6"
+ requires: ["FMS"]
+ paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"]
+ branch: ["2021.02","dev/gfdl/2018.04.06"]
+ repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"]
+ makeOverrides: 'OPENMP=""'
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
+ - component: "FMScoupler"
+ paths: ["FMScoupler/full", "FMScoupler/shared"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ branch: "2022.01"
+ requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"]
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
diff --git a/lib/fre/pp/tests/AM5_example/yaml_include/platforms.yaml b/lib/fre/pp/tests/AM5_example/yaml_include/platforms.yaml
new file mode 100644
index 00000000..7e1b9f49
--- /dev/null
+++ b/lib/fre/pp/tests/AM5_example/yaml_include/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/lib/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml
new file mode 100644
index 00000000..117c66c6
--- /dev/null
+++ b/lib/fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml
@@ -0,0 +1,88 @@
+# local reusable variable overrides
+fre_properties:
+ - &custom_interp "180,360"
+
+# directory overrides
+#c96_amip_directories:
+directories:
+ <<: *shared_directories
+ ptmp_dir: "/ptmp/$USER"
+ pp_grid_spec: *GRID_SPEC96
+
+#c96_amip_postprocess:
+postprocess:
+ # pp setting overrides
+ settings:
+ <<: *shared_settings
+ pp_start: *ANA_AMIP_START
+ pp_stop: *ANA_AMIP_END
+ pp_chunk_a: *PP_AMIP_CHUNK96
+ pp_components: "atmos atmos_scalar"
+ switches:
+ <<: *shared_switches
+ do_statics: False
+
+ # main pp instructions
+ components:
+ - type: "atmos_cmip"
+ sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *custom_interp
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos"
+ sources: "atmos_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_level_cmip"
+ sources: "atmos_level_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_level"
+ sources: "atmos_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_month_aer"
+ sources: "atmos_month_aer"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
+ - type: "atmos_diurnal"
+ sources: "atmos_diurnal"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_scalar"
+ sources: "atmos_scalar"
+ - type: "aerosol_cmip"
+ xyInterp: *PP_XYINTERP96
+ sources: "aerosol_month_cmip"
+ sourceGrid: "cubedsphere"
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
+ - type: "land"
+ sources: "land_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'land'
+ - type: "land_cmip"
+ sources: "land_month_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'land'
+ - type: "tracer_level"
+ sources: "atmos_tracer"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
diff --git a/lib/fre/pp/tests/__init__.py b/lib/fre/pp/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/pp/tests/test_configure_script_yaml.py b/lib/fre/pp/tests/test_configure_script_yaml.py
new file mode 100644
index 00000000..eaf1fc2e
--- /dev/null
+++ b/lib/fre/pp/tests/test_configure_script_yaml.py
@@ -0,0 +1,46 @@
+"""
+Test configure_script_yaml
+"""
+import os
+from pathlib import Path
+from fre.pp import configure_script_yaml as csy
+
+# Set what would be click options
+EXPERIMENT = "c96L65_am5f7b12r1_amip"
+PLATFORM = "gfdl.ncrc5-intel22-classic"
+TARGET = "prod-openmp"
+
+# Set example yaml paths, input directory
+test_dir = Path("fre/pp/tests")
+test_yaml = Path("AM5_example/am5.yaml")
+
+def test_combinedyaml_exists():
+ """
+ Make sure combined yaml file exists
+ """
+ assert Path(f"{test_dir}/{test_yaml}").exists()
+
+def test_configure_script():
+ """
+ Tests success of confgure yaml script
+ Creates rose-suite, regrid rose-app, remap rose-app
+ TO-DO: will break this up for better tests
+ """
+ # Set home for ~/cylc-src location in script
+ os.environ["HOME"]=str(Path(f"{test_dir}/configure_yaml_out"))
+
+ # Set output directory
+ out_dir = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}")
+ Path(out_dir).mkdir(parents=True,exist_ok=True)
+
+ # Define combined yaml
+ model_yaml = str(Path(f"{test_dir}/{test_yaml}"))
+
+ # Invoke configure_yaml_script.py
+ csy.yamlInfo(model_yaml,EXPERIMENT,PLATFORM,TARGET)
+
+ # Check for configuration creation and final combined yaml
+ assert all([Path(f"{out_dir}/{EXPERIMENT}.yaml").exists(),
+ Path(f"{out_dir}/rose-suite.conf").exists(),
+ Path(f"{out_dir}/app/regrid-xy/rose-app.conf").exists(),
+ Path(f"{out_dir}/app/remap-pp-components/rose-app.conf").exists()])
diff --git a/lib/fre/pp/tests/test_rose_quoting.py b/lib/fre/pp/tests/test_rose_quoting.py
new file mode 100644
index 00000000..bd5353b1
--- /dev/null
+++ b/lib/fre/pp/tests/test_rose_quoting.py
@@ -0,0 +1,7 @@
+from fre.pp.configure_script_yaml import quote_rose_values
+
+def test_boolean():
+ assert quote_rose_values(True) == 'True'
+
+def test_string():
+ assert quote_rose_values('foo') == "'foo'"
diff --git a/lib/fre/pp/validate.py b/lib/fre/pp/validate.py
new file mode 100644
index 00000000..9c07340f
--- /dev/null
+++ b/lib/fre/pp/validate.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+''' fre pp validate '''
+
+import os
+import subprocess
+import click
+
+def _validate_subtool(experiment, platform, target):
+ """
+ Validate the Cylc workflow definition located in
+ ~/cylc-src/____
+ """
+
+ directory = os.path.expanduser('~/cylc-src/' + experiment + '__' + platform + '__' + target)
+
+ # Change the current working directory
+ os.chdir(directory)
+
+ # Run the Rose validation macros
+ cmd = "rose macro --validate"
+ subprocess.run(cmd, shell=True, check=True)
+
+ # Validate the Cylc configuration
+ cmd = "cylc validate ."
+ subprocess.run(cmd, shell=True, check=True)
+
+@click.command()
+def validate_subtool(experiment, platform, target):
+ ''' entry point to validate for click '''
+ return _validate_subtool(experiment, platform, target)
diff --git a/lib/fre/pp/wrapper.py b/lib/fre/pp/wrapper.py
new file mode 100644
index 00000000..0f0aad39
--- /dev/null
+++ b/lib/fre/pp/wrapper.py
@@ -0,0 +1,85 @@
+"""
+frepp.py, a replacement for the frepp bash script located at:
+https://gitlab.gfdl.noaa.gov/fre2/system-settings/-/blob/main/bin/frepp
+Author: Carolyn.Whitlock
+"""
+
+#todo:
+# add relative path import to rest of pp tools
+# add command-line args using same format as fre.py
+# include arg for pp start / stop
+# test yaml path
+# error handling
+
+import os
+import time
+import click
+
+# Import from the local packages
+from .checkoutScript import _checkoutTemplate
+from .configure_script_xml import _convert
+from .configure_script_yaml import _yamlInfo
+from .validate import _validate_subtool
+from .install import _install_subtool
+from .run import _pp_run_subtool
+from .status import _status_subtool
+
+@click.command()
+def runFre2pp(experiment, platform, target, config_file, branch):
+ '''
+ Wrapper script for calling a FRE2 pp experiment with the canopy-style
+ infrastructure and fre-cli
+ time=0000
+ '''
+
+ config_file = os.path.abspath(config_file)
+
+ #env_setup
+ #todo: check for experiment existing, call frepp_stop to clean experiment,
+ try:
+ print("calling _checkoutTemplate")
+ _checkoutTemplate(experiment, platform, target, branch)
+ except Exception as err:
+ raise
+
+ #dumb xml check;does it need to be smarter?
+ is_xml = config_file[-3:] == "xml"
+ if is_xml:
+ #TODO: should this prompt for pp start/stop years?
+ try:
+ _convert(config_file, platform, target, experiment, do_analysis=False)
+ #note: arg list for this function is a looooot longer, but all those
+ #args can be deduced from the xml when given default vals
+ except Exception as err:
+ raise
+ try:
+ _validate_subtool(experiment, platform, target)
+ #See notes in main() function
+ except Exception as err:
+ raise
+ else:
+ try:
+ _yamlInfo(config_file, experiment, platform, target)
+ except Exception as err:
+ raise
+
+ try:
+ _install_subtool(experiment, platform, target)
+ except:
+ raise
+
+ try:
+ _pp_run_subtool(experiment, platform, target)
+ except Exception as err:
+ raise
+
+ #send off a watcher script that reports on how it's going
+ for n in range(1,12):
+ try:
+ _status_subtool(experiment, platform, target)
+ except Exception as err:
+ raise
+ time.sleep(300)
+
+if __name__ == '__main__':
+ runFre2pp()
diff --git a/lib/fre/pp/wrapperscript b/lib/fre/pp/wrapperscript
new file mode 100755
index 00000000..f98d9cd6
--- /dev/null
+++ b/lib/fre/pp/wrapperscript
@@ -0,0 +1,320 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+# https://stackoverflow.com/questions/402377/using-getopts-to-process-long-and-short-command-line-options
+TEMP=$(getopt -o x:p:P:T:t:shvc:D:d: --long xml:,platform:,target:,time:,help,mppnccombine-opts:,mail-list: -n 'frepp' -- "$@")
+eval set -- "$TEMP"
+
+# defaults
+xml=
+platform=
+target=
+time=
+help=
+
+# arg parsing
+while true; do
+ case "$1" in
+ # required
+ -x | --xml ) xml="$2"; shift 2 ;;
+ -p | -P | --platform ) platform="$2"; shift 2 ;;
+ -T | --target ) target="$2"; shift 2 ;;
+ -t | --time ) time="$2"; shift 2 ;;
+
+ # optional
+ -h | --help ) help=true; shift ;;
+
+ # ignored
+ -v ) shift ;;
+ -c ) shift 2 ;;
+ -D ) shift 2 ;;
+ -d ) shift 2 ;;
+ -s ) shift ;;
+ --mppnccombine-opts ) shift 2 ;;
+ --mail-list ) shift 2 ;;
+
+ -- ) shift; break ;;
+ * ) break ;;
+ esac
+done
+if [[ -n ${1-} ]]; then
+ expname=$1
+else
+ expname=
+fi
+
+# If $FRE_DUALPP is set, then take two different actions
+# 1. Append "_canopy" to pp, analysis, and history_refined directories created through the XML converter
+# 2. Submit Bronx frepp as well
+set +u
+if [[ $FRE_DUALPP ]]; then
+ dual=true
+else
+ dual=false
+fi
+set -u
+
+# Help
+usage="Usage: frepp --xml=XML --platform=PLATFORM --target=TARGET --time=YYYY EXP"
+if [[ $help ]]; then
+ echo $usage
+ cat << EOF
+################################################################################
+FRE Canopy frepp wrapper to start Canopy postprocessing workflow with
+traditional Bronx frepp usage.
+
+Cylc implementation current settings used by this wrapper:
+1. Workflow name is ____
+e.g. use cylc commands such as:
+
+cylc workflow-state ____
+
+This is somewhat overly verbose and also not verbose enough
+(i.e. does not include FRE STEM).
+If you have suggestions please let the FRE team know.
+
+2. Will not use unique run directories.
+If the run directory exists you will need to remove it before re-installing.
+
+################################################################################
+What does this script do?
+1. If workflow run-dir was previously installed,
+ start postprocessing for a history file segment:
+
+- Check if the workflow is running
+- Check the task states
+- Start cylc scheduler
+- Trigger requested processing (-t YYYY)
+- Exit
+
+2. Otherwise, if workflow src-dir does not exist,
+ configure the postprocessing:
+
+- Checkout a fresh PP template
+- Run the XML converter
+
+3. Then, install and start the postprocessing for a history file segment
+- Run the validation scripts
+- Install the workflow
+- Start cylc scheduler
+- Trigger requested processing (-t YYYY)
+
+################################################################################
+Recovery steps and scenarios:
+1. Something is terribly wrong with PP and you want to reconfigure and try again
+- Stop cylc scheduler with "cylc stop --kill "
+- Remove run directory with "cylc clean "
+- Edit the configuration files in ~/cylc-src/
+- Run frepp again to reinstall and run the updated PP configuration.
+
+2. Something is terribly wrong and you want a complete fresh start,
+ or you want an update from the pp template repo.
+- Stop cylc scheduler with "cylc stop --kill"
+- Remove run directory with "cylc clean "
+- Remove src directory with "rm -rf ~/cylc-src/"
+- Run frepp again to recheckout pp template, run xml converter, and install/run
+
+################################################################################
+Specific suggestions to recover from task failures:
+
+1. refineDiag script failures are likely with a XML-converted configs
+ for two reasons, so you will probably need to either adjust or remove them.
+ To disable refineDiag,
+ - set DO_REFINEDIAG=False, and
+ - comment out HISTORY_DIR_REFINED
+
+a. It may use something in the XML, using an xml shell variable that does not
+ exist now. In these cases, you could rewrite the refineDiag script to
+ not use the xmlDir shell variable or not use the script.
+ For "refineDiag_atmos_cmip6.csh", it was included in the postprocessing
+ template checkout with a small modification. Use this location:
+ '\$CYLC_WORKFLOW_RUN_DIR/etc/refineDiag/refineDiag_atmos_cmip6.csh'.
+ - set REFINEDIAG_SCRIPTS to that location
+
+b. It may be a refineDiag script that does not generate .nc files
+ as it was expected to do. FRE Bronx allows these side-effect refineDiags,
+ and instead a new mechanism was invented for these scripts that
+ do not generate netcdf output:
+ - set DO_PREANALYSIS=True, and
+ - PREANALYSIS_SCRIPT="/paath/to/script".
+
+2. Many PP components in Bronx XMLs are doomed (in terms of failing to
+ produce output and job failing) caused by using history files that do not
+ exist, but do not cause problems for the other components. Currently,
+ the Canopy pp template is not robust in terms of this error mode,
+ so it's best to not process history files that do not exist.
+
+ In the future, diag manager metadata output will provide a catalog
+ of history output that the validators will check against. For now,
+ a simple checker exists, but you must manually generate the
+ history output list ("history-manifest" file).
+
+ Generate the file with a simple listing of the history tarfile.
+ You can append a history_refined tarfile as well. Then, the validator
+ will report on PP components you have specified
+ (PP_COMPONENTS) but that do not exist in the history-manifest file.
+
+ tar -tf /path/to/history/YYYYMMDD.nc.tar | sort > history-manifest
+
+ To run the configuration validation:
+
+cd ~/cylc-src/
+rose macro --validate
+
+ It is a good idea to not include pp components (PP_COMPONENTS) that
+ include history files that do not exist.
+
+ In all cases it is recommended to remove validation errors.
+ See README.md for general configuration instructions.
+EOF
+ exit 0
+fi
+
+# check for all options
+if [[ $xml ]]; then
+ xml=$(readlink -f $xml)
+ if [[ -f $xml ]]; then
+ echo "using $xml"
+ else
+ echo "XML '$xml' does not exist"
+ exit 1
+ fi
+else
+ echo $usage
+ exit 1
+fi
+
+if [[ $platform ]]; then
+ echo "using $platform"
+else
+ echo $usage
+ exit 1
+fi
+
+if [[ $target ]]; then
+ echo "using $target"
+else
+ echo $usage
+ exit 1
+fi
+
+if [[ $time ]]; then
+ echo "using $time"
+else
+ echo $usage
+ exit 1
+fi
+
+if [[ $expname ]]; then
+ echo "using $expname"
+else
+ echo $usage
+ exit 1
+fi
+
+cylc --version
+if cylc cycle-point $time; then
+ time_iso=$(cylc cycle-point $time --template CCYYMMDDT0000Z)
+else
+ echo "Time '$time' not a valid ISO8601 date"
+ exit 1
+fi
+
+# Start bronx dual-pp
+if [[ $dual == true ]]; then
+ $FRE_COMMANDS_HOME/bin/frepp -x $xml -P $platform -T $target -t $time -D '' $expname -v -s
+fi
+
+# Set the cylc workflow name to __
+# use the default workflow source convention
+name=${expname}__${platform}__$target
+rundir="$HOME/cylc-run/$name"
+srcdir="$HOME/cylc-src/$name"
+echo Workflow name: $name
+echo Run directory: $rundir
+echo Src directory: $srcdir
+
+# Start postprocessing for a history file segment (workflow was previously installed)
+if [[ -d $rundir ]]; then
+ echo "Run directory '$rundir' exists, so will now try to start it"
+ cylc scan
+ cylc workflow-state $name
+ if cylc workflow-state $name | grep failed; then
+ cat << EOF
+################################################################################
+Unfortunately, there are failed tasks, probably caused by refineDiag errors
+or try to use a history file that does not exist.
+
+While Cylc workflows can be configured to handle failure gracefully,
+this workflow is not yet set to do this, so currently it's recommended
+to reconfigure your postprocessing to remove task errors.
+
+For some suggestions to recover from the above most common errors, see:
+
+frepp --help
+################################################################################
+EOF
+ fi
+ # sometimes this hangs for unknown reasons
+ # So for now we'll add --debug to try to diagnose it, and
+ # use /bin/timeout to exit after 10 min
+ timeout 10m cylc play --debug $name
+ sleep 20
+ cylc trigger $name//$time_iso/pp-starter
+ exit 0
+fi
+
+# Checkout postprocessing template and configure
+if [[ ! -d $srcdir ]]; then
+ echo "Workflow source directory '$srcdir' does not exist, so will now try to checkout template"
+
+ # checkout
+ mkdir -p $HOME/cylc-src
+ cd $HOME/cylc-src
+ # try to reduce checkout size with depth=1
+ #git clone --depth=1 --recursive git@gitlab.gfdl.noaa.gov:fre2/workflows/postprocessing.git $name
+ git clone --depth=1 --recursive https://gitlab.gfdl.noaa.gov/fre2/workflows/postprocessing.git $name
+
+ # xml converter
+ cd $srcdir
+ if [[ $dual == true ]]; then
+ bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v --dual
+ else
+ bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v
+ fi
+fi
+
+# validate configuration
+cd $srcdir
+if ! rose macro --validate; then
+ cat << EOF
+################################################################################
+Configuration may not be valid.
+
+In general, Canopy configurations should pass all available validation scripts.
+To run them,
+
+cd $HOME/cylc-src/$name
+rose macro --validate
+
+Most validation errors reflect configurations problems that should be corrected.
+The exceptions are:
+1. PP_DIR will be created if it does not exist
+2. HISTORY_DIR_REFINED will be created if it does not exist,
+ assuming DO_REFINEDIAG is also set
+
+See README.md for general configuration instructions.
+################################################################################
+EOF
+fi
+cylc validate .
+
+# Install
+cylc install --no-run-name $name
+
+# Start
+cylc play $name
+sleep 20
+cylc trigger $name//$time_iso/pp-starter
+exit 0
diff --git a/lib/fre/pytest.ini b/lib/fre/pytest.ini
new file mode 100644
index 00000000..131a2523
--- /dev/null
+++ b/lib/fre/pytest.ini
@@ -0,0 +1,14 @@
+[pytest]
+testpaths =
+ fre/tests
+ fre/catalog/tests
+# fre/check/tests
+ fre/cmor/tests
+# fre/list/tests
+ fre/make/tests
+ fre/pp/tests
+# fre/run/tests
+# fre/test/tests
+# fre/yamltools/tests
+# fre/app/tests
+ fre/app/generate_time_averages/tests
diff --git a/lib/fre/run/__init__.py b/lib/fre/run/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/run/frerun.py b/lib/fre/run/frerun.py
new file mode 100644
index 00000000..c2c611c3
--- /dev/null
+++ b/lib/fre/run/frerun.py
@@ -0,0 +1,21 @@
+'''
+entry point for fre run subcommands
+'''
+
+import click
+from .frerunexample import run_test_function
+
+@click.group(help=click.style(" - access fre run subcommands", fg=(164,29,132)))
+def run_cli():
+ ''' entry point to fre run click commands '''
+
+@run_cli.command()
+@click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
+@click.pass_context
+def function(context, uppercase):
+ # pylint: disable=unused-argument
+ """ - Execute fre run test """
+ context.forward(run_test_function)
+
+if __name__ == "__main__":
+ run_cli()
diff --git a/lib/fre/run/frerunexample.py b/lib/fre/run/frerunexample.py
new file mode 100644
index 00000000..a4335144
--- /dev/null
+++ b/lib/fre/run/frerunexample.py
@@ -0,0 +1,18 @@
+"""
+experimentation file for integrating one file's functions into main prototype fre file
+authored by Bennett.Chang@noaa.gov | bcc2761
+NOAA | GFDL
+"""
+
+import click
+
+@click.command()
+def run_test_function(uppercase=None):
+ """Execute fre run run_test_function"""
+ statement = "testingtestingtestingtesting"
+ if uppercase:
+ statement = statement.upper()
+ click.echo(statement)
+
+if __name__ == '__main__':
+ run_test_function()
diff --git a/lib/fre/test/__init__.py b/lib/fre/test/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/test/fretest.py b/lib/fre/test/fretest.py
new file mode 100644
index 00000000..172ac855
--- /dev/null
+++ b/lib/fre/test/fretest.py
@@ -0,0 +1,21 @@
+'''
+entry point for fre test subcommands
+'''
+
+import click
+from .fretestexample import test_test_function
+
+@click.group(help=click.style(" - access fre test subcommands", fg=(92,164,29)))
+def test_cli():
+ ''' entry point to fre test click commands '''
+
+@test_cli.command()
+@click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
+@click.pass_context
+def function(context, uppercase):
+ # pylint: disable=unused-argument
+ """ - Execute fre test test """
+ context.forward(test_test_function)
+
+if __name__ == "__main__":
+ test_cli()
diff --git a/lib/fre/test/fretestexample.py b/lib/fre/test/fretestexample.py
new file mode 100644
index 00000000..86d0f6ce
--- /dev/null
+++ b/lib/fre/test/fretestexample.py
@@ -0,0 +1,18 @@
+"""
+experimentation file for integrating one file's functions into main prototype fre file
+authored by Bennett.Chang@noaa.gov | bcc2761
+NOAA | GFDL
+"""
+
+import click
+
+@click.command()
+def test_test_function(uppercase=None):
+ """Execute fre list test_test_function"""
+ statement = "testingtestingtestingtesting"
+ if uppercase:
+ statement = statement.upper()
+ click.echo(statement)
+
+if __name__ == '__main__':
+ test_test_function()
diff --git a/lib/fre/tests/README.md b/lib/fre/tests/README.md
new file mode 100644
index 00000000..31b2c2c6
--- /dev/null
+++ b/lib/fre/tests/README.md
@@ -0,0 +1,20 @@
+note- this directory is for unit tests assessing `fre` functionality, and NOT for code corresponding to the command `fre test`
+
+to run the tests as intended, simply call `pytest` from the root directory of this repository.
+
+From the root directory of this repository, if you want to...
+
+Invoke all tests (intended use):
+ `pytest fre/tests/`
+
+Invoke all tests in a single file:
+ `pytest fre/tests/test_fre_cli.py`
+
+Invoke a single test in a file:
+ `pytest fre/tests/test_fre_cli.py::test_cli_fre_option_dne`
+
+
+Note that pytest will not print stdout from individual tests. If you want to...
+
+Print stdout from an invoked test (debugging with print statements):
+ `pytest fre/tests/test_fre_cli.py::test_cli_fre_option_dne -s`
diff --git a/lib/fre/tests/__init__.py b/lib/fre/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/tests/test_files/CMOR_input_example.json b/lib/fre/tests/test_files/CMOR_input_example.json
new file mode 100644
index 00000000..3ce8a985
--- /dev/null
+++ b/lib/fre/tests/test_files/CMOR_input_example.json
@@ -0,0 +1,74 @@
+{
+ "#note": "explanation of what source_type is goes here",
+ "source_type": "AOGCM ISM AER",
+
+ "#note": "CMIP6 valid experiment_ids are found in CMIP6_CV.json",
+ "experiment_id": "piControl-withism",
+ "activity_id": "ISMIP6",
+ "sub_experiment_id": "none",
+
+ "realization_index": "3",
+ "initialization_index": "1",
+ "physics_index": "1",
+ "forcing_index": "1",
+
+ "#note": "Text stored in attribute variant_info (recommended, not required description of run variant)",
+ "run_variant": "3rd realization",
+
+ "parent_experiment_id": "no parent",
+ "parent_activity_id": "no parent",
+ "parent_source_id": "PCMDI-test-1-0",
+ "parent_variant_label": "r3i1p1f1",
+
+ "parent_time_units": "days since 1850-01-01",
+ "branch_method": "standard",
+ "branch_time_in_child": 59400.0,
+ "branch_time_in_parent": 59400.0,
+
+ "#note": "institution_id must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+ "institution_id": "PCMDI",
+
+ "#note": "source_id (model name) must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+ "source_id": "PCMDI-test-1-0",
+
+ "calendar": "360_day",
+
+ "grid": "native atmosphere regular grid (3x4 latxlon)",
+ "grid_label": "gn",
+ "nominal_resolution": "10000 km",
+
+ "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.",
+
+ "#output": "Root directory for output (can be either a relative or full path)",
+ "outpath": "CMIP6",
+
+ "#note": " **** The following descriptors are optional and may be set to an empty string ",
+
+ "contact ": "Python Coder (coder@a.b.c.com)",
+ "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.",
+ "comment": "",
+ "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'",
+
+ "#note": " **** The following will be obtained from the CV and do not need to be defined here",
+
+ "sub_experiment": "none",
+ "institution": "",
+ "source": "PCMDI-test 1.0 (1989)",
+
+ "#note": " **** The following are set correctly for CMIP6 and should not normally need editing",
+
+ "_controlled_vocabulary_file": "CMIP6_CV.json",
+ "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json",
+ "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json",
+ "_cmip6_option": "CMIP6",
+
+ "mip_era": "CMIP6",
+ "parent_mip_era": "CMIP6",
+
+ "tracking_prefix": "hdl:21.14100",
+ "_history_template": "%s ;rewrote data to be consistent with for variable found in table .",
+
+ "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax",
+ "output_path_template": "<_member_id>
+
+ `;
+
+ // Inject the generated flyout into the body HTML element.
+ document.body.insertAdjacentHTML("beforeend", flyout);
+
+ // Trigger the Read the Docs Addons Search modal when clicking on the "Search docs" input from inside the flyout.
+ document
+ .querySelector("#flyout-search-form")
+ .addEventListener("focusin", () => {
+ const event = new CustomEvent("readthedocs-search-show");
+ document.dispatchEvent(event);
+ });
+ })
+}
+
+if (themeLanguageSelector || themeVersionSelector) {
+ function onSelectorSwitch(event) {
+ const option = event.target.selectedIndex;
+ const item = event.target.options[option];
+ window.location.href = item.dataset.url;
+ }
+
+ document.addEventListener("readthedocs-addons-data-ready", function (event) {
+ const config = event.detail.data();
+
+ const versionSwitch = document.querySelector(
+ "div.switch-menus > div.version-switch",
+ );
+ if (themeVersionSelector) {
+ let versions = config.versions.active;
+ if (config.versions.current.hidden || config.versions.current.type === "external") {
+ versions.unshift(config.versions.current);
+ }
+ const versionSelect = `
+
+ `;
+
+ versionSwitch.innerHTML = versionSelect;
+ versionSwitch.firstElementChild.addEventListener("change", onSelectorSwitch);
+ }
+
+ const languageSwitch = document.querySelector(
+ "div.switch-menus > div.language-switch",
+ );
+
+ if (themeLanguageSelector) {
+ if (config.projects.translations.length) {
+ // Add the current language to the options on the selector
+ let languages = config.projects.translations.concat(
+ config.projects.current,
+ );
+ languages = languages.sort((a, b) =>
+ a.language.name.localeCompare(b.language.name),
+ );
+
+ const languageSelect = `
+
+ `;
+
+ languageSwitch.innerHTML = languageSelect;
+ languageSwitch.firstElementChild.addEventListener("change", onSelectorSwitch);
+ }
+ else {
+ languageSwitch.remove();
+ }
+ }
+ });
+}
+
+document.addEventListener("readthedocs-addons-data-ready", function (event) {
+ // Trigger the Read the Docs Addons Search modal when clicking on "Search docs" input from the topnav.
+ document
+ .querySelector("[role='search'] input")
+ .addEventListener("focusin", () => {
+ const event = new CustomEvent("readthedocs-search-show");
+ document.dispatchEvent(event);
+ });
+});
\ No newline at end of file
diff --git a/_static/language_data.js b/_static/language_data.js
new file mode 100644
index 00000000..367b8ed8
--- /dev/null
+++ b/_static/language_data.js
@@ -0,0 +1,199 @@
+/*
+ * language_data.js
+ * ~~~~~~~~~~~~~~~~
+ *
+ * This script contains the language-specific data used by searchtools.js,
+ * namely the list of stopwords, stemmer, scorer and splitter.
+ *
+ * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"];
+
+
+/* Non-minified version is copied as a separate JS file, if available */
+
+/**
+ * Porter Stemmer
+ */
+var Stemmer = function() {
+
+ var step2list = {
+ ational: 'ate',
+ tional: 'tion',
+ enci: 'ence',
+ anci: 'ance',
+ izer: 'ize',
+ bli: 'ble',
+ alli: 'al',
+ entli: 'ent',
+ eli: 'e',
+ ousli: 'ous',
+ ization: 'ize',
+ ation: 'ate',
+ ator: 'ate',
+ alism: 'al',
+ iveness: 'ive',
+ fulness: 'ful',
+ ousness: 'ous',
+ aliti: 'al',
+ iviti: 'ive',
+ biliti: 'ble',
+ logi: 'log'
+ };
+
+ var step3list = {
+ icate: 'ic',
+ ative: '',
+ alize: 'al',
+ iciti: 'ic',
+ ical: 'ic',
+ ful: '',
+ ness: ''
+ };
+
+ var c = "[^aeiou]"; // consonant
+ var v = "[aeiouy]"; // vowel
+ var C = c + "[^aeiouy]*"; // consonant sequence
+ var V = v + "[aeiou]*"; // vowel sequence
+
+ var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0
+ var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1
+ var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1
+ var s_v = "^(" + C + ")?" + v; // vowel in stem
+
+ this.stemWord = function (w) {
+ var stem;
+ var suffix;
+ var firstch;
+ var origword = w;
+
+ if (w.length < 3)
+ return w;
+
+ var re;
+ var re2;
+ var re3;
+ var re4;
+
+ firstch = w.substr(0,1);
+ if (firstch == "y")
+ w = firstch.toUpperCase() + w.substr(1);
+
+ // Step 1a
+ re = /^(.+?)(ss|i)es$/;
+ re2 = /^(.+?)([^s])s$/;
+
+ if (re.test(w))
+ w = w.replace(re,"$1$2");
+ else if (re2.test(w))
+ w = w.replace(re2,"$1$2");
+
+ // Step 1b
+ re = /^(.+?)eed$/;
+ re2 = /^(.+?)(ed|ing)$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ re = new RegExp(mgr0);
+ if (re.test(fp[1])) {
+ re = /.$/;
+ w = w.replace(re,"");
+ }
+ }
+ else if (re2.test(w)) {
+ var fp = re2.exec(w);
+ stem = fp[1];
+ re2 = new RegExp(s_v);
+ if (re2.test(stem)) {
+ w = stem;
+ re2 = /(at|bl|iz)$/;
+ re3 = new RegExp("([^aeiouylsz])\\1$");
+ re4 = new RegExp("^" + C + v + "[^aeiouwxy]$");
+ if (re2.test(w))
+ w = w + "e";
+ else if (re3.test(w)) {
+ re = /.$/;
+ w = w.replace(re,"");
+ }
+ else if (re4.test(w))
+ w = w + "e";
+ }
+ }
+
+ // Step 1c
+ re = /^(.+?)y$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ re = new RegExp(s_v);
+ if (re.test(stem))
+ w = stem + "i";
+ }
+
+ // Step 2
+ re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ suffix = fp[2];
+ re = new RegExp(mgr0);
+ if (re.test(stem))
+ w = stem + step2list[suffix];
+ }
+
+ // Step 3
+ re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ suffix = fp[2];
+ re = new RegExp(mgr0);
+ if (re.test(stem))
+ w = stem + step3list[suffix];
+ }
+
+ // Step 4
+ re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
+ re2 = /^(.+?)(s|t)(ion)$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ re = new RegExp(mgr1);
+ if (re.test(stem))
+ w = stem;
+ }
+ else if (re2.test(w)) {
+ var fp = re2.exec(w);
+ stem = fp[1] + fp[2];
+ re2 = new RegExp(mgr1);
+ if (re2.test(stem))
+ w = stem;
+ }
+
+ // Step 5
+ re = /^(.+?)e$/;
+ if (re.test(w)) {
+ var fp = re.exec(w);
+ stem = fp[1];
+ re = new RegExp(mgr1);
+ re2 = new RegExp(meq1);
+ re3 = new RegExp("^" + C + v + "[^aeiouwxy]$");
+ if (re.test(stem) || (re2.test(stem) && !(re3.test(stem))))
+ w = stem;
+ }
+ re = /ll$/;
+ re2 = new RegExp(mgr1);
+ if (re.test(w) && re2.test(w)) {
+ re = /.$/;
+ w = w.replace(re,"");
+ }
+
+ // and turn initial Y back to y
+ if (firstch == "y")
+ w = firstch.toLowerCase() + w.substr(1);
+ return w;
+ }
+}
+
diff --git a/_static/minus.png b/_static/minus.png
new file mode 100644
index 00000000..d96755fd
Binary files /dev/null and b/_static/minus.png differ
diff --git a/_static/plus.png b/_static/plus.png
new file mode 100644
index 00000000..7107cec9
Binary files /dev/null and b/_static/plus.png differ
diff --git a/_static/pygments.css b/_static/pygments.css
new file mode 100644
index 00000000..84ab3030
--- /dev/null
+++ b/_static/pygments.css
@@ -0,0 +1,75 @@
+pre { line-height: 125%; }
+td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
+span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
+td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
+span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
+.highlight .hll { background-color: #ffffcc }
+.highlight { background: #f8f8f8; }
+.highlight .c { color: #3D7B7B; font-style: italic } /* Comment */
+.highlight .err { border: 1px solid #FF0000 } /* Error */
+.highlight .k { color: #008000; font-weight: bold } /* Keyword */
+.highlight .o { color: #666666 } /* Operator */
+.highlight .ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */
+.highlight .cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */
+.highlight .cp { color: #9C6500 } /* Comment.Preproc */
+.highlight .cpf { color: #3D7B7B; font-style: italic } /* Comment.PreprocFile */
+.highlight .c1 { color: #3D7B7B; font-style: italic } /* Comment.Single */
+.highlight .cs { color: #3D7B7B; font-style: italic } /* Comment.Special */
+.highlight .gd { color: #A00000 } /* Generic.Deleted */
+.highlight .ge { font-style: italic } /* Generic.Emph */
+.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
+.highlight .gr { color: #E40000 } /* Generic.Error */
+.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.highlight .gi { color: #008400 } /* Generic.Inserted */
+.highlight .go { color: #717171 } /* Generic.Output */
+.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
+.highlight .gs { font-weight: bold } /* Generic.Strong */
+.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.highlight .gt { color: #0044DD } /* Generic.Traceback */
+.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
+.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
+.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
+.highlight .kp { color: #008000 } /* Keyword.Pseudo */
+.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
+.highlight .kt { color: #B00040 } /* Keyword.Type */
+.highlight .m { color: #666666 } /* Literal.Number */
+.highlight .s { color: #BA2121 } /* Literal.String */
+.highlight .na { color: #687822 } /* Name.Attribute */
+.highlight .nb { color: #008000 } /* Name.Builtin */
+.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
+.highlight .no { color: #880000 } /* Name.Constant */
+.highlight .nd { color: #AA22FF } /* Name.Decorator */
+.highlight .ni { color: #717171; font-weight: bold } /* Name.Entity */
+.highlight .ne { color: #CB3F38; font-weight: bold } /* Name.Exception */
+.highlight .nf { color: #0000FF } /* Name.Function */
+.highlight .nl { color: #767600 } /* Name.Label */
+.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
+.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
+.highlight .nv { color: #19177C } /* Name.Variable */
+.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
+.highlight .w { color: #bbbbbb } /* Text.Whitespace */
+.highlight .mb { color: #666666 } /* Literal.Number.Bin */
+.highlight .mf { color: #666666 } /* Literal.Number.Float */
+.highlight .mh { color: #666666 } /* Literal.Number.Hex */
+.highlight .mi { color: #666666 } /* Literal.Number.Integer */
+.highlight .mo { color: #666666 } /* Literal.Number.Oct */
+.highlight .sa { color: #BA2121 } /* Literal.String.Affix */
+.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
+.highlight .sc { color: #BA2121 } /* Literal.String.Char */
+.highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */
+.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
+.highlight .s2 { color: #BA2121 } /* Literal.String.Double */
+.highlight .se { color: #AA5D1F; font-weight: bold } /* Literal.String.Escape */
+.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */
+.highlight .si { color: #A45A77; font-weight: bold } /* Literal.String.Interpol */
+.highlight .sx { color: #008000 } /* Literal.String.Other */
+.highlight .sr { color: #A45A77 } /* Literal.String.Regex */
+.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
+.highlight .ss { color: #19177C } /* Literal.String.Symbol */
+.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
+.highlight .fm { color: #0000FF } /* Name.Function.Magic */
+.highlight .vc { color: #19177C } /* Name.Variable.Class */
+.highlight .vg { color: #19177C } /* Name.Variable.Global */
+.highlight .vi { color: #19177C } /* Name.Variable.Instance */
+.highlight .vm { color: #19177C } /* Name.Variable.Magic */
+.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
\ No newline at end of file
diff --git a/_static/searchtools.js b/_static/searchtools.js
new file mode 100644
index 00000000..b08d58c9
--- /dev/null
+++ b/_static/searchtools.js
@@ -0,0 +1,620 @@
+/*
+ * searchtools.js
+ * ~~~~~~~~~~~~~~~~
+ *
+ * Sphinx JavaScript utilities for the full-text search.
+ *
+ * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+"use strict";
+
+/**
+ * Simple result scoring code.
+ */
+if (typeof Scorer === "undefined") {
+ var Scorer = {
+ // Implement the following function to further tweak the score for each result
+ // The function takes a result array [docname, title, anchor, descr, score, filename]
+ // and returns the new score.
+ /*
+ score: result => {
+ const [docname, title, anchor, descr, score, filename] = result
+ return score
+ },
+ */
+
+ // query matches the full name of an object
+ objNameMatch: 11,
+ // or matches in the last dotted part of the object name
+ objPartialMatch: 6,
+ // Additive scores depending on the priority of the object
+ objPrio: {
+ 0: 15, // used to be importantResults
+ 1: 5, // used to be objectResults
+ 2: -5, // used to be unimportantResults
+ },
+ // Used when the priority is not in the mapping.
+ objPrioDefault: 0,
+
+ // query found in title
+ title: 15,
+ partialTitle: 7,
+ // query found in terms
+ term: 5,
+ partialTerm: 2,
+ };
+}
+
+const _removeChildren = (element) => {
+ while (element && element.lastChild) element.removeChild(element.lastChild);
+};
+
+/**
+ * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
+ */
+const _escapeRegExp = (string) =>
+ string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
+
+const _displayItem = (item, searchTerms, highlightTerms) => {
+ const docBuilder = DOCUMENTATION_OPTIONS.BUILDER;
+ const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX;
+ const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX;
+ const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY;
+ const contentRoot = document.documentElement.dataset.content_root;
+
+ const [docName, title, anchor, descr, score, _filename] = item;
+
+ let listItem = document.createElement("li");
+ let requestUrl;
+ let linkUrl;
+ if (docBuilder === "dirhtml") {
+ // dirhtml builder
+ let dirname = docName + "/";
+ if (dirname.match(/\/index\/$/))
+ dirname = dirname.substring(0, dirname.length - 6);
+ else if (dirname === "index/") dirname = "";
+ requestUrl = contentRoot + dirname;
+ linkUrl = requestUrl;
+ } else {
+ // normal html builders
+ requestUrl = contentRoot + docName + docFileSuffix;
+ linkUrl = docName + docLinkSuffix;
+ }
+ let linkEl = listItem.appendChild(document.createElement("a"));
+ linkEl.href = linkUrl + anchor;
+ linkEl.dataset.score = score;
+ linkEl.innerHTML = title;
+ if (descr) {
+ listItem.appendChild(document.createElement("span")).innerHTML =
+ " (" + descr + ")";
+ // highlight search terms in the description
+ if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
+ highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
+ }
+ else if (showSearchSummary)
+ fetch(requestUrl)
+ .then((responseData) => responseData.text())
+ .then((data) => {
+ if (data)
+ listItem.appendChild(
+ Search.makeSearchSummary(data, searchTerms, anchor)
+ );
+ // highlight search terms in the summary
+ if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
+ highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
+ });
+ Search.output.appendChild(listItem);
+};
+const _finishSearch = (resultCount) => {
+ Search.stopPulse();
+ Search.title.innerText = _("Search Results");
+ if (!resultCount)
+ Search.status.innerText = Documentation.gettext(
+ "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories."
+ );
+ else
+ Search.status.innerText = _(
+ "Search finished, found ${resultCount} page(s) matching the search query."
+ ).replace('${resultCount}', resultCount);
+};
+const _displayNextItem = (
+ results,
+ resultCount,
+ searchTerms,
+ highlightTerms,
+) => {
+ // results left, load the summary and display it
+ // this is intended to be dynamic (don't sub resultsCount)
+ if (results.length) {
+ _displayItem(results.pop(), searchTerms, highlightTerms);
+ setTimeout(
+ () => _displayNextItem(results, resultCount, searchTerms, highlightTerms),
+ 5
+ );
+ }
+ // search finished, update title and status message
+ else _finishSearch(resultCount);
+};
+// Helper function used by query() to order search results.
+// Each input is an array of [docname, title, anchor, descr, score, filename].
+// Order the results by score (in opposite order of appearance, since the
+// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically.
+const _orderResultsByScoreThenName = (a, b) => {
+ const leftScore = a[4];
+ const rightScore = b[4];
+ if (leftScore === rightScore) {
+ // same score: sort alphabetically
+ const leftTitle = a[1].toLowerCase();
+ const rightTitle = b[1].toLowerCase();
+ if (leftTitle === rightTitle) return 0;
+ return leftTitle > rightTitle ? -1 : 1; // inverted is intentional
+ }
+ return leftScore > rightScore ? 1 : -1;
+};
+
+/**
+ * Default splitQuery function. Can be overridden in ``sphinx.search`` with a
+ * custom function per language.
+ *
+ * The regular expression works by splitting the string on consecutive characters
+ * that are not Unicode letters, numbers, underscores, or emoji characters.
+ * This is the same as ``\W+`` in Python, preserving the surrogate pair area.
+ */
+if (typeof splitQuery === "undefined") {
+ var splitQuery = (query) => query
+ .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu)
+ .filter(term => term) // remove remaining empty strings
+}
+
+/**
+ * Search Module
+ */
+const Search = {
+ _index: null,
+ _queued_query: null,
+ _pulse_status: -1,
+
+ htmlToText: (htmlString, anchor) => {
+ const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html');
+ for (const removalQuery of [".headerlink", "script", "style"]) {
+ htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() });
+ }
+ if (anchor) {
+ const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`);
+ if (anchorContent) return anchorContent.textContent;
+
+ console.warn(
+ `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.`
+ );
+ }
+
+ // if anchor not specified or not found, fall back to main content
+ const docContent = htmlElement.querySelector('[role="main"]');
+ if (docContent) return docContent.textContent;
+
+ console.warn(
+ "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template."
+ );
+ return "";
+ },
+
+ init: () => {
+ const query = new URLSearchParams(window.location.search).get("q");
+ document
+ .querySelectorAll('input[name="q"]')
+ .forEach((el) => (el.value = query));
+ if (query) Search.performSearch(query);
+ },
+
+ loadIndex: (url) =>
+ (document.body.appendChild(document.createElement("script")).src = url),
+
+ setIndex: (index) => {
+ Search._index = index;
+ if (Search._queued_query !== null) {
+ const query = Search._queued_query;
+ Search._queued_query = null;
+ Search.query(query);
+ }
+ },
+
+ hasIndex: () => Search._index !== null,
+
+ deferQuery: (query) => (Search._queued_query = query),
+
+ stopPulse: () => (Search._pulse_status = -1),
+
+ startPulse: () => {
+ if (Search._pulse_status >= 0) return;
+
+ const pulse = () => {
+ Search._pulse_status = (Search._pulse_status + 1) % 4;
+ Search.dots.innerText = ".".repeat(Search._pulse_status);
+ if (Search._pulse_status >= 0) window.setTimeout(pulse, 500);
+ };
+ pulse();
+ },
+
+ /**
+ * perform a search for something (or wait until index is loaded)
+ */
+ performSearch: (query) => {
+ // create the required interface elements
+ const searchText = document.createElement("h2");
+ searchText.textContent = _("Searching");
+ const searchSummary = document.createElement("p");
+ searchSummary.classList.add("search-summary");
+ searchSummary.innerText = "";
+ const searchList = document.createElement("ul");
+ searchList.classList.add("search");
+
+ const out = document.getElementById("search-results");
+ Search.title = out.appendChild(searchText);
+ Search.dots = Search.title.appendChild(document.createElement("span"));
+ Search.status = out.appendChild(searchSummary);
+ Search.output = out.appendChild(searchList);
+
+ const searchProgress = document.getElementById("search-progress");
+ // Some themes don't use the search progress node
+ if (searchProgress) {
+ searchProgress.innerText = _("Preparing search...");
+ }
+ Search.startPulse();
+
+ // index already loaded, the browser was quick!
+ if (Search.hasIndex()) Search.query(query);
+ else Search.deferQuery(query);
+ },
+
+ _parseQuery: (query) => {
+ // stem the search terms and add them to the correct list
+ const stemmer = new Stemmer();
+ const searchTerms = new Set();
+ const excludedTerms = new Set();
+ const highlightTerms = new Set();
+ const objectTerms = new Set(splitQuery(query.toLowerCase().trim()));
+ splitQuery(query.trim()).forEach((queryTerm) => {
+ const queryTermLower = queryTerm.toLowerCase();
+
+ // maybe skip this "word"
+ // stopwords array is from language_data.js
+ if (
+ stopwords.indexOf(queryTermLower) !== -1 ||
+ queryTerm.match(/^\d+$/)
+ )
+ return;
+
+ // stem the word
+ let word = stemmer.stemWord(queryTermLower);
+ // select the correct list
+ if (word[0] === "-") excludedTerms.add(word.substr(1));
+ else {
+ searchTerms.add(word);
+ highlightTerms.add(queryTermLower);
+ }
+ });
+
+ if (SPHINX_HIGHLIGHT_ENABLED) { // set in sphinx_highlight.js
+ localStorage.setItem("sphinx_highlight_terms", [...highlightTerms].join(" "))
+ }
+
+ // console.debug("SEARCH: searching for:");
+ // console.info("required: ", [...searchTerms]);
+ // console.info("excluded: ", [...excludedTerms]);
+
+ return [query, searchTerms, excludedTerms, highlightTerms, objectTerms];
+ },
+
+ /**
+ * execute search (requires search index to be loaded)
+ */
+ _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => {
+ const filenames = Search._index.filenames;
+ const docNames = Search._index.docnames;
+ const titles = Search._index.titles;
+ const allTitles = Search._index.alltitles;
+ const indexEntries = Search._index.indexentries;
+
+ // Collect multiple result groups to be sorted separately and then ordered.
+ // Each is an array of [docname, title, anchor, descr, score, filename].
+ const normalResults = [];
+ const nonMainIndexResults = [];
+
+ _removeChildren(document.getElementById("search-progress"));
+
+ const queryLower = query.toLowerCase().trim();
+ for (const [title, foundTitles] of Object.entries(allTitles)) {
+ if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) {
+ for (const [file, id] of foundTitles) {
+ const score = Math.round(Scorer.title * queryLower.length / title.length);
+ const boost = titles[file] === title ? 1 : 0; // add a boost for document titles
+ normalResults.push([
+ docNames[file],
+ titles[file] !== title ? `${titles[file]} > ${title}` : title,
+ id !== null ? "#" + id : "",
+ null,
+ score + boost,
+ filenames[file],
+ ]);
+ }
+ }
+ }
+
+ // search for explicit entries in index directives
+ for (const [entry, foundEntries] of Object.entries(indexEntries)) {
+ if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) {
+ for (const [file, id, isMain] of foundEntries) {
+ const score = Math.round(100 * queryLower.length / entry.length);
+ const result = [
+ docNames[file],
+ titles[file],
+ id ? "#" + id : "",
+ null,
+ score,
+ filenames[file],
+ ];
+ if (isMain) {
+ normalResults.push(result);
+ } else {
+ nonMainIndexResults.push(result);
+ }
+ }
+ }
+ }
+
+ // lookup as object
+ objectTerms.forEach((term) =>
+ normalResults.push(...Search.performObjectSearch(term, objectTerms))
+ );
+
+ // lookup as search terms in fulltext
+ normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms));
+
+ // let the scorer override scores with a custom scoring function
+ if (Scorer.score) {
+ normalResults.forEach((item) => (item[4] = Scorer.score(item)));
+ nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item)));
+ }
+
+ // Sort each group of results by score and then alphabetically by name.
+ normalResults.sort(_orderResultsByScoreThenName);
+ nonMainIndexResults.sort(_orderResultsByScoreThenName);
+
+ // Combine the result groups in (reverse) order.
+ // Non-main index entries are typically arbitrary cross-references,
+ // so display them after other results.
+ let results = [...nonMainIndexResults, ...normalResults];
+
+ // remove duplicate search results
+ // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept
+ let seen = new Set();
+ results = results.reverse().reduce((acc, result) => {
+ let resultStr = result.slice(0, 4).concat([result[5]]).map(v => String(v)).join(',');
+ if (!seen.has(resultStr)) {
+ acc.push(result);
+ seen.add(resultStr);
+ }
+ return acc;
+ }, []);
+
+ return results.reverse();
+ },
+
+ query: (query) => {
+ const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query);
+ const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms);
+
+ // for debugging
+ //Search.lastresults = results.slice(); // a copy
+ // console.info("search results:", Search.lastresults);
+
+ // print the results
+ _displayNextItem(results, results.length, searchTerms, highlightTerms);
+ },
+
+ /**
+ * search for object names
+ */
+ performObjectSearch: (object, objectTerms) => {
+ const filenames = Search._index.filenames;
+ const docNames = Search._index.docnames;
+ const objects = Search._index.objects;
+ const objNames = Search._index.objnames;
+ const titles = Search._index.titles;
+
+ const results = [];
+
+ const objectSearchCallback = (prefix, match) => {
+ const name = match[4]
+ const fullname = (prefix ? prefix + "." : "") + name;
+ const fullnameLower = fullname.toLowerCase();
+ if (fullnameLower.indexOf(object) < 0) return;
+
+ let score = 0;
+ const parts = fullnameLower.split(".");
+
+ // check for different match types: exact matches of full name or
+ // "last name" (i.e. last dotted part)
+ if (fullnameLower === object || parts.slice(-1)[0] === object)
+ score += Scorer.objNameMatch;
+ else if (parts.slice(-1)[0].indexOf(object) > -1)
+ score += Scorer.objPartialMatch; // matches in last name
+
+ const objName = objNames[match[1]][2];
+ const title = titles[match[0]];
+
+ // If more than one term searched for, we require other words to be
+ // found in the name/title/description
+ const otherTerms = new Set(objectTerms);
+ otherTerms.delete(object);
+ if (otherTerms.size > 0) {
+ const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase();
+ if (
+ [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0)
+ )
+ return;
+ }
+
+ let anchor = match[3];
+ if (anchor === "") anchor = fullname;
+ else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname;
+
+ const descr = objName + _(", in ") + title;
+
+ // add custom score for some objects according to scorer
+ if (Scorer.objPrio.hasOwnProperty(match[2]))
+ score += Scorer.objPrio[match[2]];
+ else score += Scorer.objPrioDefault;
+
+ results.push([
+ docNames[match[0]],
+ fullname,
+ "#" + anchor,
+ descr,
+ score,
+ filenames[match[0]],
+ ]);
+ };
+ Object.keys(objects).forEach((prefix) =>
+ objects[prefix].forEach((array) =>
+ objectSearchCallback(prefix, array)
+ )
+ );
+ return results;
+ },
+
+ /**
+ * search for full-text terms in the index
+ */
+ performTermsSearch: (searchTerms, excludedTerms) => {
+ // prepare search
+ const terms = Search._index.terms;
+ const titleTerms = Search._index.titleterms;
+ const filenames = Search._index.filenames;
+ const docNames = Search._index.docnames;
+ const titles = Search._index.titles;
+
+ const scoreMap = new Map();
+ const fileMap = new Map();
+
+ // perform the search on the required terms
+ searchTerms.forEach((word) => {
+ const files = [];
+ const arr = [
+ { files: terms[word], score: Scorer.term },
+ { files: titleTerms[word], score: Scorer.title },
+ ];
+ // add support for partial matches
+ if (word.length > 2) {
+ const escapedWord = _escapeRegExp(word);
+ if (!terms.hasOwnProperty(word)) {
+ Object.keys(terms).forEach((term) => {
+ if (term.match(escapedWord))
+ arr.push({ files: terms[term], score: Scorer.partialTerm });
+ });
+ }
+ if (!titleTerms.hasOwnProperty(word)) {
+ Object.keys(titleTerms).forEach((term) => {
+ if (term.match(escapedWord))
+ arr.push({ files: titleTerms[term], score: Scorer.partialTitle });
+ });
+ }
+ }
+
+ // no match but word was a required one
+ if (arr.every((record) => record.files === undefined)) return;
+
+ // found search word in contents
+ arr.forEach((record) => {
+ if (record.files === undefined) return;
+
+ let recordFiles = record.files;
+ if (recordFiles.length === undefined) recordFiles = [recordFiles];
+ files.push(...recordFiles);
+
+ // set score for the word in each file
+ recordFiles.forEach((file) => {
+ if (!scoreMap.has(file)) scoreMap.set(file, {});
+ scoreMap.get(file)[word] = record.score;
+ });
+ });
+
+ // create the mapping
+ files.forEach((file) => {
+ if (!fileMap.has(file)) fileMap.set(file, [word]);
+ else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word);
+ });
+ });
+
+ // now check if the files don't contain excluded terms
+ const results = [];
+ for (const [file, wordList] of fileMap) {
+ // check if all requirements are matched
+
+ // as search terms with length < 3 are discarded
+ const filteredTermCount = [...searchTerms].filter(
+ (term) => term.length > 2
+ ).length;
+ if (
+ wordList.length !== searchTerms.size &&
+ wordList.length !== filteredTermCount
+ )
+ continue;
+
+ // ensure that none of the excluded terms is in the search result
+ if (
+ [...excludedTerms].some(
+ (term) =>
+ terms[term] === file ||
+ titleTerms[term] === file ||
+ (terms[term] || []).includes(file) ||
+ (titleTerms[term] || []).includes(file)
+ )
+ )
+ break;
+
+ // select one (max) score for the file.
+ const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w]));
+ // add result to the result list
+ results.push([
+ docNames[file],
+ titles[file],
+ "",
+ null,
+ score,
+ filenames[file],
+ ]);
+ }
+ return results;
+ },
+
+ /**
+ * helper function to return a node containing the
+ * search summary for a given text. keywords is a list
+ * of stemmed words.
+ */
+ makeSearchSummary: (htmlText, keywords, anchor) => {
+ const text = Search.htmlToText(htmlText, anchor);
+ if (text === "") return null;
+
+ const textLower = text.toLowerCase();
+ const actualStartPosition = [...keywords]
+ .map((k) => textLower.indexOf(k.toLowerCase()))
+ .filter((i) => i > -1)
+ .slice(-1)[0];
+ const startWithContext = Math.max(actualStartPosition - 120, 0);
+
+ const top = startWithContext === 0 ? "" : "...";
+ const tail = startWithContext + 240 < text.length ? "..." : "";
+
+ let summary = document.createElement("p");
+ summary.classList.add("context");
+ summary.textContent = top + text.substr(startWithContext, 240).trim() + tail;
+
+ return summary;
+ },
+};
+
+_ready(Search.init);
diff --git a/_static/sphinx_highlight.js b/_static/sphinx_highlight.js
new file mode 100644
index 00000000..8a96c69a
--- /dev/null
+++ b/_static/sphinx_highlight.js
@@ -0,0 +1,154 @@
+/* Highlighting utilities for Sphinx HTML documentation. */
+"use strict";
+
+const SPHINX_HIGHLIGHT_ENABLED = true
+
+/**
+ * highlight a given string on a node by wrapping it in
+ * span elements with the given class name.
+ */
+const _highlight = (node, addItems, text, className) => {
+ if (node.nodeType === Node.TEXT_NODE) {
+ const val = node.nodeValue;
+ const parent = node.parentNode;
+ const pos = val.toLowerCase().indexOf(text);
+ if (
+ pos >= 0 &&
+ !parent.classList.contains(className) &&
+ !parent.classList.contains("nohighlight")
+ ) {
+ let span;
+
+ const closestNode = parent.closest("body, svg, foreignObject");
+ const isInSVG = closestNode && closestNode.matches("svg");
+ if (isInSVG) {
+ span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+ } else {
+ span = document.createElement("span");
+ span.classList.add(className);
+ }
+
+ span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+ const rest = document.createTextNode(val.substr(pos + text.length));
+ parent.insertBefore(
+ span,
+ parent.insertBefore(
+ rest,
+ node.nextSibling
+ )
+ );
+ node.nodeValue = val.substr(0, pos);
+ /* There may be more occurrences of search term in this node. So call this
+ * function recursively on the remaining fragment.
+ */
+ _highlight(rest, addItems, text, className);
+
+ if (isInSVG) {
+ const rect = document.createElementNS(
+ "http://www.w3.org/2000/svg",
+ "rect"
+ );
+ const bbox = parent.getBBox();
+ rect.x.baseVal.value = bbox.x;
+ rect.y.baseVal.value = bbox.y;
+ rect.width.baseVal.value = bbox.width;
+ rect.height.baseVal.value = bbox.height;
+ rect.setAttribute("class", className);
+ addItems.push({ parent: parent, target: rect });
+ }
+ }
+ } else if (node.matches && !node.matches("button, select, textarea")) {
+ node.childNodes.forEach((el) => _highlight(el, addItems, text, className));
+ }
+};
+const _highlightText = (thisNode, text, className) => {
+ let addItems = [];
+ _highlight(thisNode, addItems, text, className);
+ addItems.forEach((obj) =>
+ obj.parent.insertAdjacentElement("beforebegin", obj.target)
+ );
+};
+
+/**
+ * Small JavaScript module for the documentation.
+ */
+const SphinxHighlight = {
+
+ /**
+ * highlight the search words provided in localstorage in the text
+ */
+ highlightSearchWords: () => {
+ if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight
+
+ // get and clear terms from localstorage
+ const url = new URL(window.location);
+ const highlight =
+ localStorage.getItem("sphinx_highlight_terms")
+ || url.searchParams.get("highlight")
+ || "";
+ localStorage.removeItem("sphinx_highlight_terms")
+ url.searchParams.delete("highlight");
+ window.history.replaceState({}, "", url);
+
+ // get individual terms from highlight string
+ const terms = highlight.toLowerCase().split(/\s+/).filter(x => x);
+ if (terms.length === 0) return; // nothing to do
+
+ // There should never be more than one element matching "div.body"
+ const divBody = document.querySelectorAll("div.body");
+ const body = divBody.length ? divBody[0] : document.querySelector("body");
+ window.setTimeout(() => {
+ terms.forEach((term) => _highlightText(body, term, "highlighted"));
+ }, 10);
+
+ const searchBox = document.getElementById("searchbox");
+ if (searchBox === null) return;
+ searchBox.appendChild(
+ document
+ .createRange()
+ .createContextualFragment(
+ '
+ ${renderLanguages(config)}
+ ${renderVersions(config)}
+ ${renderDownloads(config)}
+
+ + Hosted by Read the Docs + +
+ -
+
- On Read the Docs +
- + Project Home + +
- + Builds + +
- + Downloads + +
-
+
- Search +
- + + +
+ + Hosted by Read the Docs + +
' + + '' + + _("Hide Search Matches") + + "
" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms") + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; + if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/badges.html b/badges.html new file mode 100644 index 00000000..b2ce99f1 --- /dev/null +++ b/badges.html @@ -0,0 +1,115 @@ + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/contributing_to_doc.html b/contributing_to_doc.html
new file mode 100644
index 00000000..fdf47740
--- /dev/null
+++ b/contributing_to_doc.html
@@ -0,0 +1,165 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/developer_usage.html b/developer_usage.html
new file mode 100644
index 00000000..bb0d6da8
--- /dev/null
+++ b/developer_usage.html
@@ -0,0 +1,232 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+How to Contribute to
+
+
+
+
+
+ Documentation-Documentation
+Welcome to fre-cli’s Documentation-documentation- where we document how the documentation is
+documented
How to Contribute to fre-cli’s documentation
+-
+
fork and poke at the settings
+-
+
Fork
fre-clion github
+On github, navigate to your
fre-clifork, and click “settings”
+In “settings”, click “pages”
+In “pages”, under “build and deployment”, make sure “source” is set to “Deploy from a branch”
+Under that, find “Branch”, make sure the branch selected is
gh-pages.
+-
+
- The branch
gh-pagesis “automagic”- i.e. do not change anything about it nor create a new one, nor interact with anything in that branch directly.
+
+
+- The branch
+enable workflows for your fork
+-
+
note: this step may depend on user-specific settings!
+Back on top where you found “settings”, find and click “actions” to the left
+Enable running the workflow actions assoc with the
fre-clirepo under.github/workflows
+
+run your forks first workflow
+-
+
-
+
- The documentation builds as the last steps to
create_test_conda_env.ymlwhen theres a push tomain. To get your first workflow run on your fork, comment out the
+github.ref == ‘refs/heads/main’bit +so that it runs when you push to any branch, and make a small, trivial, commit somewhere to your +remote fork.
+
+- The documentation builds as the last steps to
-
+
- You should be able to find the deployed webpage from a successful workflow at
https://your_username.github.io/fre-cli (if you did not change the fork’s name from
+fre-cli, that is).
+
+If you’re only editing docs, you can make the turn-around time on your workflow ~3 min faster by
+
commenting-out the
+pylintandpyteststeps increate_test_conda_env.yml, and disable running the +build_conda.ymlworkflow
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.freapp.html b/fre.app.freapp.html
new file mode 100644
index 00000000..dbbabc3b
--- /dev/null
+++ b/fre.app.freapp.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Example
+
+
+
+
+
+
+
+
+
+
+
+ Developer Usage
+Developers are free to use the user guide above to familiarize with the CLI and save time from +having to install any dependencies, but development within a Conda environment is heavily +recommended regardless.
+Gain access to the repository with git clone --recursive git@github.com:NOAA-GFDL/fre-cli.git or your fork’s
+link (recommended) and an SSH RSA key. Once inside the repository, developers can test local changes
+by running a pip install . inside of the root directory to install the fre-cli package locally
+with the newest local changes. Test as a normal user would use the CLI.
Adding New Tools
+From Other Repositories
+Currently, the solution to this task is to approach it using Conda packages. The tool that is being
+added must reside within a repository that contains a meta.yaml that includes Conda dependencies
+like the one in this repository and ideally a setup.py (may be subject to change due to deprecation)
+that may include any potentially needed pip dependencies
-
+
Once published as a Conda package, ideally on the NOAA-GFDL conda channel, +an addition can be made to the
runsection underrequirementsinmeta.yamlof thefre-cli+following the syntaxchannel::package
+On pushes to the main branch, the package located at https://anaconda.org/NOAA-GFDL/fre-cli will automatically +be updated using by the workflow defined in
.github/workflows/publish_conda.yml
+
Checklist
+For the new tool you are trying to develop, there are a few criteria to satisfy
+-
+
Create a subdirectory for the tool group inside the
fre/directory; i.e.fre/<tool>
+Add an
__init__.pyinside offre/<tool>
+
-
+
typically this file should be empty, but it depends on the
<tool>’s needs
+even if empty, the file facillitates module importability and must be present
+
-
+
Add a file named
fre/<tool>/fre<tool>.py. This will serve as the main entry point forfre+into the<tool>’s functionality
+Add a
clickgroup named after<tool>withinfre/<tool>/fre<tool>.py
+
-
+
This
clickgroup will contain all the functionality under the<tool>
+
-
+
Create separate files as needed for different commands; do not code out the full +implemetation of
<tool>inside of aclickcommand withinfre/<tool>/fre<tool>.py.
+
-
+
better yet, consider what structure your tool may need in the future for maintainability’s sake
+if you need, specify a
<subtool>likefre/<tool>/<subtool>.fre/appcurrently has +this structure
+
-
+
Be sure to import the contents of the needed subcommand scripts inside of
fre<tool>.py
+
-
+
i.e. from
fre.<tool>.toolCommandScript import *
+
-
+
At this point, you can copy and paste the parts of your main
clickcommand from its script +intofre<tool>.pywhen implementing the function reflective of the command function
+
-
+
Everything will remain the same; i.e. arguments, options, etc.
+However, this new function within
fre<tool>.pymust a new line after the arguments, options, +and other command components;@click.pass_context
+Along with this, a new argument
contextmust now be added to the parameters of the command +(preferably at the beginning, but it won’t break it if it’s not)
+
-
+
From here, all that needs to be added after defining the command with a name is +
context.forward(mainFunctionOfToolCommand), and done!
+-
+
- The last step is to replicate the command in the same way as done in
fre<tool>.py inside of
+fre.py, but make sure to addfrom fre import <tool>and +from fre.<tool> import *
+
+- The last step is to replicate the command in the same way as done in
Please refer to this issue when encountering naming issues: +NOAA-GFDL#31
+Example fre/ Directory Structure
+fre/
+├── __init__.py
+├── fre.py
+├── fre<tool>
+│ ├── __init__.py
+│ ├── toolCommandScript.py
+│ └── fre<tool>.py
MANIFEST.in
+In the case where non-python files like templates, examples, and outputs are to be included in the fre-cli package,
+MANIFEST.in can provide the solution. Ensure that the file exists within the correct folder, and add a line to the
+MANIFEST.in file saying something like include fre/fre<tool>/fileName.fileExtension
-
+
For more efficiency, if there are multiple files of the same type needed, the
MANIFEST.inaddition can be something +likerecursive-include fre/fre<tool> *.fileExtensionwhich would recursively include every file matching that +fileExtensionwithin the specified directory and its respective subdirectories.
+
Adding Documentation
+see section “Documentation-Documentation”
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.cdoTimeAverager.html b/fre.app.generate_time_averages.cdoTimeAverager.html
new file mode 100644
index 00000000..6b317378
--- /dev/null
+++ b/fre.app.generate_time_averages.cdoTimeAverager.html
@@ -0,0 +1,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.freapp module
+fre app calls
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.frenctoolsTimeAverager.html b/fre.app.generate_time_averages.frenctoolsTimeAverager.html
new file mode 100644
index 00000000..eab53648
--- /dev/null
+++ b/fre.app.generate_time_averages.frenctoolsTimeAverager.html
@@ -0,0 +1,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.cdoTimeAverager module
+class using (mostly) cdo functions for time-averages
+-
+
- +class fre.app.generate_time_averages.cdoTimeAverager.cdoTimeAverager(pkg, var, unwgt, avg_type, stddev_type) +
Bases:
+timeAveragerclass inheriting from abstract base class timeAverager +generates time-averages using cdo (mostly, see weighted approach)
+-
+
- +generate_timavg(infile=None, outfile=None) +
use cdo package routines via python bindings
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.frepytoolsTimeAverager.html b/fre.app.generate_time_averages.frepytoolsTimeAverager.html
new file mode 100644
index 00000000..39d055f4
--- /dev/null
+++ b/fre.app.generate_time_averages.frepytoolsTimeAverager.html
@@ -0,0 +1,127 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.frenctoolsTimeAverager module
+class for utilizing timavg.csh (aka script to TAVG fortran exe) in frenc-tools
+-
+
- +class fre.app.generate_time_averages.frenctoolsTimeAverager.frenctoolsTimeAverager(pkg, var, unwgt, avg_type, stddev_type) +
Bases:
+timeAveragerclass inheriting from abstract base class timeAverager +generates time-averages using fre-nctools
+-
+
- +generate_timavg(infile=None, outfile=None) +
use fre-nctool’s CLI timavg.csh with subprocess call
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.generate_time_averages.html b/fre.app.generate_time_averages.generate_time_averages.html
new file mode 100644
index 00000000..567c0363
--- /dev/null
+++ b/fre.app.generate_time_averages.generate_time_averages.html
@@ -0,0 +1,117 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.frepytoolsTimeAverager module
+class for python-native routine usuing netCDF4 and numpy to crunch time-averages
+-
+
- +class fre.app.generate_time_averages.frepytoolsTimeAverager.frepytoolsTimeAverager(pkg, var, unwgt, avg_type, stddev_type) +
Bases:
+timeAveragerclass inheriting from abstract base class timeAverager +generates time-averages using a python-native approach +avoids using other third party statistics functions by design.
+-
+
- +generate_timavg(infile=None, outfile=None) +
frepytools approach in a python-native manner. +deliberately avoids pre-packaged routines.
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.html b/fre.app.generate_time_averages.html
new file mode 100644
index 00000000..4c2dc3a2
--- /dev/null
+++ b/fre.app.generate_time_averages.html
@@ -0,0 +1,194 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.generate_time_averages module
+tools for generating time averages from various packages
+-
+
- +fre.app.generate_time_averages.generate_time_averages.generate_time_average(infile=None, outfile=None, pkg=None, var=None, unwgt=False, avg_type=None, stddev_type=None) +
steering function to various averaging functions above
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.tests.html b/fre.app.generate_time_averages.tests.html
new file mode 100644
index 00000000..5f07a024
--- /dev/null
+++ b/fre.app.generate_time_averages.tests.html
@@ -0,0 +1,141 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages package
+Subpackages
+
+
+-
+
- fre.app.generate_time_averages.tests package
-
+
- Submodules
-
+
- fre.app.generate_time_averages.tests.test_generate_time_averages module
-
+
run_avgtype_pkg_calculations()
+test_cdo_time_avgs()
+test_cdo_time_unwgt_avgs()
+test_cdo_time_unwgt_stddevs()
+test_compare_cdo_to_fre_nctools()
+test_compare_fre_python_tools_to_cdo()
+test_compare_fre_python_tools_to_fre_nctools()
+test_compare_unwgt_fre_python_tools_to_unwgt_cdo()
+test_fre_python_tools_time_avgs()
+test_fre_python_tools_time_avgs_stddevs()
+test_fre_python_tools_time_unwgt_avgs()
+test_fre_python_tools_time_unwgt_avgs_stddevs()
+test_monthly_cdo_time_unwgt_avgs()
+test_monthly_cdo_time_unwgt_stddevs()
+test_seasonal_cdo_time_unwgt_avgs()
+test_seasonal_cdo_time_unwgt_stddevs()
+test_time_avg_file_dir_exists()
+test_time_avg_input_file_exists()
+
+
+ - fre.app.generate_time_averages.tests.test_generate_time_averages module
- Module contents +
+ - Submodules
Submodules
+
+
+-
+
- fre.app.generate_time_averages.cdoTimeAverager module + +
- fre.app.generate_time_averages.frenctoolsTimeAverager module + +
- fre.app.generate_time_averages.frepytoolsTimeAverager module + +
- fre.app.generate_time_averages.generate_time_averages module + +
- fre.app.generate_time_averages.timeAverager module + +
Module contents
+required for generate_time_averages module import functionality
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.tests.test_generate_time_averages.html b/fre.app.generate_time_averages.tests.test_generate_time_averages.html
new file mode 100644
index 00000000..18d9b5c1
--- /dev/null
+++ b/fre.app.generate_time_averages.tests.test_generate_time_averages.html
@@ -0,0 +1,219 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.tests package
+Submodules
+
+
+-
+
- fre.app.generate_time_averages.tests.test_generate_time_averages module
-
+
run_avgtype_pkg_calculations()
+test_cdo_time_avgs()
+test_cdo_time_unwgt_avgs()
+test_cdo_time_unwgt_stddevs()
+test_compare_cdo_to_fre_nctools()
+test_compare_fre_python_tools_to_cdo()
+test_compare_fre_python_tools_to_fre_nctools()
+test_compare_unwgt_fre_python_tools_to_unwgt_cdo()
+test_fre_python_tools_time_avgs()
+test_fre_python_tools_time_avgs_stddevs()
+test_fre_python_tools_time_unwgt_avgs()
+test_fre_python_tools_time_unwgt_avgs_stddevs()
+test_monthly_cdo_time_unwgt_avgs()
+test_monthly_cdo_time_unwgt_stddevs()
+test_seasonal_cdo_time_unwgt_avgs()
+test_seasonal_cdo_time_unwgt_stddevs()
+test_time_avg_file_dir_exists()
+test_time_avg_input_file_exists()
+
+
Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.generate_time_averages.timeAverager.html b/fre.app.generate_time_averages.timeAverager.html
new file mode 100644
index 00000000..b1eb2363
--- /dev/null
+++ b/fre.app.generate_time_averages.timeAverager.html
@@ -0,0 +1,157 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.tests.test_generate_time_averages module
+for testing fre app generate-time-averages
+-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.run_avgtype_pkg_calculations(infile=None, outfile=None, pkg=None, avg_type=None, unwgt=None, stddev_type=None) +
test-harness function, called by other test functions.
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_cdo_time_avgs() +
generates a weighted time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_cdo_time_unwgt_avgs() +
generates an unweighted time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_cdo_time_unwgt_stddevs() +
generates a time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_compare_cdo_to_fre_nctools() +
compares cdo pkg answer to fre_nctools pkg answer
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_compare_fre_python_tools_to_cdo() +
compares fre_python_tools pkg answer to cdo pkg answer
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_compare_fre_python_tools_to_fre_nctools() +
compares fre_python_tools pkg answer to fre_nctools pkg answer
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_compare_unwgt_fre_python_tools_to_unwgt_cdo() +
compares fre_python_tools pkg answer to cdo pkg answer
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_fre_python_tools_time_avgs() +
generates a time averaged file using fre_python_tools’s version
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_fre_python_tools_time_avgs_stddevs() +
generates a time averaged file using fre_python_tools’s version
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_fre_python_tools_time_unwgt_avgs() +
generates a time averaged file using fre_python_tools’s version
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_fre_python_tools_time_unwgt_avgs_stddevs() +
generates a time averaged file using fre_python_tools’s version
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_monthly_cdo_time_unwgt_avgs() +
generates an unweighted monthly time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_monthly_cdo_time_unwgt_stddevs() +
generates a monthly time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_seasonal_cdo_time_unwgt_avgs() +
generates an unweighted seasonal time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_seasonal_cdo_time_unwgt_stddevs() +
generates a seasonal time averaged file using cdo
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_time_avg_file_dir_exists() +
look for input test file directory
+
-
+
- +fre.app.generate_time_averages.tests.test_generate_time_averages.test_time_avg_input_file_exists() +
look for input test file
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.html b/fre.app.html
new file mode 100644
index 00000000..20e7ff27
--- /dev/null
+++ b/fre.app.html
@@ -0,0 +1,194 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.generate_time_averages.timeAverager module
+core class structure for this module.
+-
+
- +class fre.app.generate_time_averages.timeAverager.timeAverager(pkg, var, unwgt, avg_type, stddev_type) +
Bases:
+objectabstract base class for generating time averages + related statistical quantities +this class must be inherited by another for functionality.
+-
+
- +avg_type: str +
-
+
- +generate_timavg(infile=None, outfile=None) +
# this is a hint: this is to be defined by classes inheriting from the abstract one +this function is never to be fully defined here by design.
+
-
+
- +pkg: str +
-
+
- +stddev_type: str +
-
+
- +unwgt: bool +
-
+
- +var: str +
-
+
- +var_has_time_units(an_nc_var=None) +
checks if variable’s units are of time
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.mask_atmos_plevel.html b/fre.app.mask_atmos_plevel.html
new file mode 100644
index 00000000..15b2197c
--- /dev/null
+++ b/fre.app.mask_atmos_plevel.html
@@ -0,0 +1,153 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app package
+Subpackages
+
+
+-
+
- fre.app.generate_time_averages package
-
+
- Subpackages + +
- Submodules
-
+
- fre.app.generate_time_averages.cdoTimeAverager module + +
- fre.app.generate_time_averages.frenctoolsTimeAverager module + +
- fre.app.generate_time_averages.frepytoolsTimeAverager module + +
- fre.app.generate_time_averages.generate_time_averages module + +
- fre.app.generate_time_averages.timeAverager module
-
+
timeAverager
+
+
+ - Module contents +
+ - fre.app.regrid_xy package
-
+
- Submodules + +
- Module contents +
+
Submodules
+ +Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.regrid_xy.html b/fre.app.regrid_xy.html
new file mode 100644
index 00000000..f09744aa
--- /dev/null
+++ b/fre.app.regrid_xy.html
@@ -0,0 +1,135 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.mask_atmos_plevel module
+This script contains the refineDiags that produce data at the same +frequency as the input data (no reduction) such as surface albedo, +masking fields,… +It can accept any file and will only compute refineDiags in fields +are present.
+-
+
- +fre.app.mask_atmos_plevel.mask_field_above_surface_pressure(ds, var, ds_ps) +
mask data with pressure larger than surface pressure
+
-
+
- +fre.app.mask_atmos_plevel.post_write(filename, var_with_bounds, bounds_variables) +
fix a posteriori attributes that xarray.to_netcdf +did not do properly using low level netcdf lib
+
-
+
- +fre.app.mask_atmos_plevel.preprocess(ds) +
add needs_atmos_masking attribute if var ends with _unmsk
+
-
+
- +fre.app.mask_atmos_plevel.pressure_coordinate(ds, varname) +
check if dataArray has pressure coordinate fitting requirements +and return it
+
-
+
- +fre.app.mask_atmos_plevel.set_netcdf_encoding(ds, pressure_vars) +
set preferred options for netcdf encoding
+
-
+
- +fre.app.mask_atmos_plevel.write_dataset(ds, template, outfile) +
prepare the dataset and dump into netcdf file
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.app.regrid_xy.regrid_xy.html b/fre.app.regrid_xy.regrid_xy.html
new file mode 100644
index 00000000..7515f9ab
--- /dev/null
+++ b/fre.app.regrid_xy.regrid_xy.html
@@ -0,0 +1,188 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.catalog.frecatalog.html b/fre.catalog.frecatalog.html
new file mode 100644
index 00000000..9d711a62
--- /dev/null
+++ b/fre.catalog.frecatalog.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.app.regrid_xy.regrid_xy module
+remaps scalar and/or vector fields. It is capable of remapping +from a spherical grid onto a different one, e.g. spherical, +or tripolar. By default, it does so using a conservative scheme. +most valid input args to fregrid are valid in this script
+-
+
- +fre.app.regrid_xy.regrid_xy.check_interp_method(nc_variable, interp_method) +
print warning if optional interp_method clashes with nc file attribute field, if present
+
-
+
- +fre.app.regrid_xy.regrid_xy.check_per_component_settings(component_list, rose_app_cfg) +
for a source file ref’d by multiple components check per-component +settings for uniqueness. output list of bools of same length to check +in componenet loop
+
-
+
- +fre.app.regrid_xy.regrid_xy.freq_to_date_format(iso_freq) +
Print legacy Bronx-like date template format given a frequency (ISO 8601 duration)
+
-
+
- +fre.app.regrid_xy.regrid_xy.get_mosaic_file_name(grid_spec_file, mosaic_type) +
read string from a numpy masked array WHY
+
-
+
- +fre.app.regrid_xy.regrid_xy.get_mosaic_grid_file_name(input_mosaic) +
get mosaic grid file name from NESTED numpy masked array WHY
+
-
+
- +fre.app.regrid_xy.regrid_xy.main() +
steering, local test/debug
+
-
+
- +fre.app.regrid_xy.regrid_xy.make_component_list(config, source) +
make list of relevant component names where source file appears in sources
+
-
+
- +fre.app.regrid_xy.regrid_xy.make_regrid_var_list(target_file, interp_method=None) +
create default list of variables to be regridded within target file.
+
-
+
- +fre.app.regrid_xy.regrid_xy.regrid_xy(input_dir=None, output_dir=None, begin=None, tmp_dir=None, remap_dir=None, source=None, grid_spec=None, def_xy_interp=None) +
calls fre-nctools’ fregrid to regrid net cdf files
+
-
+
- +fre.app.regrid_xy.regrid_xy.safe_rose_config_get(config, section, field) +
read optional variables from rose configuration, and don’t error on None value
+
-
+
- +fre.app.regrid_xy.regrid_xy.test_import() +
for quickly testing import within pytest
+
-
+
- +fre.app.regrid_xy.regrid_xy.truncate_date(date, freq) +
truncates iso freq to iso date time
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.catalog.html b/fre.catalog.html
new file mode 100644
index 00000000..49c1db7a
--- /dev/null
+++ b/fre.catalog.html
@@ -0,0 +1,139 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.catalog.frecatalog module
+entry point for fre catalog subcommands
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.catalog.tests.html b/fre.catalog.tests.html
new file mode 100644
index 00000000..2446eff6
--- /dev/null
+++ b/fre.catalog.tests.html
@@ -0,0 +1,124 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.catalog.tests.test_fre_catalog.html b/fre.catalog.tests.test_fre_catalog.html
new file mode 100644
index 00000000..5e5bcd04
--- /dev/null
+++ b/fre.catalog.tests.test_fre_catalog.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.check.frecheck.html b/fre.check.frecheck.html
new file mode 100644
index 00000000..5ebc9507
--- /dev/null
+++ b/fre.check.frecheck.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.check.frecheckexample.html b/fre.check.frecheckexample.html
new file mode 100644
index 00000000..d05d7010
--- /dev/null
+++ b/fre.check.frecheckexample.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.check.frecheck module
+fre check
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.check.html b/fre.check.html
new file mode 100644
index 00000000..f3c9c7e9
--- /dev/null
+++ b/fre.check.html
@@ -0,0 +1,122 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.check.frecheckexample module
+experimentation file for integrating one file’s functions into main prototype fre file +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.cmor.cmor_mixer.html b/fre.cmor.cmor_mixer.html
new file mode 100644
index 00000000..4f82939e
--- /dev/null
+++ b/fre.cmor.cmor_mixer.html
@@ -0,0 +1,269 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.cmor.frecmor.html b/fre.cmor.frecmor.html
new file mode 100644
index 00000000..117261f7
--- /dev/null
+++ b/fre.cmor.frecmor.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.cmor_mixer module
+python module housing the metadata processing routines utilizing the cmor module, in addition to +click API entry points +see README.md for additional information on fre cmor run (cmor_mixer.py) usage
+-
+
- +fre.cmor.cmor_mixer.check_dataset_for_ocean_grid(ds) +
checks netCDF4.Dataset ds for ocean grid origin, and throws an error if it finds one. accepts +one argument. this function has no return.
++
+ds: netCDF4.Dataset object containing variables with associated dimensional information.
+
-
+
- +fre.cmor.cmor_mixer.cmor_run_subtool(indir=None, json_var_list=None, json_table_config=None, json_exp_config=None, outdir=None, opt_var_name=None) +
-
+
- primary steering function for the cmor_mixer tool, i.e essentially main. Accepts six args:
-
+
- indir: string, directory containing netCDF files. keys specified in json_var_list are local
variable names used for targeting specific files
+
+- json_var_list: string, path pointing to a json file containing directory of key/value
pairs. the keys are the “local” names used in the filename, and the +values pointed to by those keys are strings representing the name of the +variable contained in targeted files. the key and value are often the same, +but it is not required.
+
+- json_table_config: json file containing CMIP-compliant per-variable/metadata for specific
MIP table. The MIP table can generally be identified by the specific +filename (e.g. “Omon”)
+
+- json_exp_config: json file containing metadata dictionary for CMORization. this metadata is effectively
appended to the final output file’s header
+
+- outdir: string, directory root that will contain the full output and output directory
structure generated by the cmor module upon request.
+
+- opt_var_name: string, optional, specify a variable name to specifically process only filenames matching
that variable name. I.e., this string help target local_vars, not target_vars.
+
+
+
-
+
- +fre.cmor.cmor_mixer.cmorize_target_var_files(indir=None, target_var=None, local_var=None, iso_datetime_arr=None, name_of_set=None, json_exp_config=None, outdir=None, proj_table_vars=None, json_table_config=None) +
processes a target directory/file +this routine is almost entirely exposed data movement before/after calling +rewrite_netcdf_file_var it is also the most hopelessly opaque routine in this entire dang macro. +this badboy right here accepts… lord help us… !!!NINE!!! arguments, NINE.
++
+indir: string, path to target directories containing netcdf files to cmorize +target_var: string, name of variable inside the netcdf file to cmorize +local_var: string, value of the variable name in the filename, right before the .nc
++
+extension. often identical to target_var but not always.
+-
+
- iso_datetime_arr: list of strings, each one a unique ISO datetime string found in targeted
netcdf filenames
+
+- name_of_set: string, representing the post-processing component (GFDL convention) of the
targeted files.
+
+
json_exp_config: see cmor_run_subtool arg desc +outdir: string, path to output directory root to move the cmor module output to, including
++
+the whole directory structure
+proj_table_vars: an opened json file object, read from json_table_config +json_table_config: see cmor_run_subtool arg desc
+
-
+
- +fre.cmor.cmor_mixer.copy_nc(in_nc, out_nc) +
copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy +operation, as if it were, using shutil’s copy would be sufficient. accepts two arguments
++
+in_nc: string, path to an input netcdf file we wish to copy +out_nc: string, an output path to copy the targeted input netcdf file to
+
-
+
- +fre.cmor.cmor_mixer.create_tmp_dir(outdir) +
creates a tmp_dir based on targeted output directory root. returns the name of the tmp dir. +accepts one argument:
++
+-
+
- outdir: string, representing the final output directory root for the cmor modules netcdf
file output. tmp_dir will be slightly different depending on the output directory +targeted
+
+
-
+
- +fre.cmor.cmor_mixer.get_iso_datetimes(var_filenames, iso_datetime_arr=None) +
-
+
- appends iso datetime strings found amongst filenames to iso_datetime_arr.
-
+
- var_filenames: non-empty list of strings representing filenames. some of which presumably
contain datetime strings
+
+- iso_datetime_arr: list of strings, empty or non-empty, representing datetimes found in
var_filenames entries. the objet pointed to by the reference +iso_datetime_arr is manipulated, and so need-not be returned
+
+
+
-
+
- +fre.cmor.cmor_mixer.get_var_filenames(indir, var_filenames=None, local_var=None) +
-
+
- appends files ending in .nc located within indir to list var_filenames accepts three arguments
indir: string, representing a path to a directory containing files ending in .nc extension +var_filenames: list of strings, empty or non-empty, to append discovered filenames to. the
++
+object pointed to by the reference var_filenames is manipulated, and so need +not be returned.
+local_var: string, optional, if not None, will be used for ruling out filename targets
+
+
-
+
- +fre.cmor.cmor_mixer.get_vertical_dimension(ds, target_var) +
determines the vertical dimensionality of target_var within netCDF4 Dataset ds. accepts two +arguments and returns an object represnting the vertical dimensions assoc with the target_var.
++
+ds: netCDF4.Dataset object containing variables with associated dimensional information. +target_var: string, representating a variable contained within the netCDF4.Dataset ds
+
-
+
- +fre.cmor.cmor_mixer.rewrite_netcdf_file_var(proj_table_vars=None, local_var=None, netcdf_file=None, target_var=None, json_exp_config=None, json_table_config=None) +
rewrite the input netcdf file nc_fl containing target_var in a CMIP-compliant manner. +accepts six arguments, all required:
++
+proj_table_vars: json dictionary object, variable table read from json_table_config. +local_var: string, variable name used for finding files locally containing target_var,
++
+this argument is often equal to target_var.
+netcdf_file: string, representing path to intput netcdf file. +target_var: string, representing the variable name attached to the data object in the netcdf file. +json_exp_config: string, representing path to json configuration file holding metadata for appending to output
++
+this argument is most used for making sure the right grid label is getting attached to the right output
+-
+
- json_table_config: string, representing path to json configuration file holding variable names for a given table.
proj_table_vars is read from this file, but both are passed anyways.
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.cmor.html b/fre.cmor.html
new file mode 100644
index 00000000..957ad5f1
--- /dev/null
+++ b/fre.cmor.html
@@ -0,0 +1,160 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.frecmor module
+fre cmor
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.cmor.tests.html b/fre.cmor.tests.html
new file mode 100644
index 00000000..fa71b390
--- /dev/null
+++ b/fre.cmor.tests.html
@@ -0,0 +1,132 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor package
+Subpackages
+
+
+-
+
- fre.cmor.tests package
-
+
- Submodules
-
+
- fre.cmor.tests.test_cmor_run_subtool module
-
+
test_fre_cmor_run_subtool_case1()
+test_fre_cmor_run_subtool_case1_output_compare_data()
+test_fre_cmor_run_subtool_case1_output_compare_metadata()
+test_fre_cmor_run_subtool_case2()
+test_fre_cmor_run_subtool_case2_output_compare_data()
+test_fre_cmor_run_subtool_case2_output_compare_metadata()
+test_setup_cmor_cmip_table_repo()
+test_setup_fre_cmor_run_subtool()
+test_setup_fre_cmor_run_subtool_case2()
+
+
+ - fre.cmor.tests.test_cmor_run_subtool module
- Module contents +
+ - Submodules
Submodules
+ +Module contents
+for fre.cmor imports
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.cmor.tests.test_cmor_run_subtool.html b/fre.cmor.tests.test_cmor_run_subtool.html
new file mode 100644
index 00000000..d6e1719c
--- /dev/null
+++ b/fre.cmor.tests.test_cmor_run_subtool.html
@@ -0,0 +1,168 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.tests package
+Submodules
+
+
+-
+
- fre.cmor.tests.test_cmor_run_subtool module
-
+
test_fre_cmor_run_subtool_case1()
+test_fre_cmor_run_subtool_case1_output_compare_data()
+test_fre_cmor_run_subtool_case1_output_compare_metadata()
+test_fre_cmor_run_subtool_case2()
+test_fre_cmor_run_subtool_case2_output_compare_data()
+test_fre_cmor_run_subtool_case2_output_compare_metadata()
+test_setup_cmor_cmip_table_repo()
+test_setup_fre_cmor_run_subtool()
+test_setup_fre_cmor_run_subtool_case2()
+
+
Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.fre.html b/fre.fre.html
new file mode 100644
index 00000000..29a8f630
--- /dev/null
+++ b/fre.fre.html
@@ -0,0 +1,116 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.cmor.tests.test_cmor_run_subtool module
+tests for fre.cmor.cmor_run_subtool
+-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case1(capfd) +
fre cmor run, test-use case
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case1_output_compare_data(capfd) +
I/O data-only comparison of test case1
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd) +
I/O metadata-only comparison of test case1
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case2(capfd) +
fre cmor run, test-use case2
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case2_output_compare_data(capfd) +
I/O data-only comparison of test case2
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_fre_cmor_run_subtool_case2_output_compare_metadata(capfd) +
I/O metadata-only comparison of test case2
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_setup_cmor_cmip_table_repo() +
setup routine, if it doesnt exist, clone the repo holding CMOR/CMIP6 tables
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_setup_fre_cmor_run_subtool(capfd) +
checks for outputfile from prev pytest runs, removes it if it’s present. +this routine also checks to make sure the desired input file is present
+
-
+
- +fre.cmor.tests.test_cmor_run_subtool.test_setup_fre_cmor_run_subtool_case2(capfd) +
make a copy of the input file to the slightly different name. +checks for outputfile from prev pytest runs, removes it if it’s present. +this routine also checks to make sure the desired input file is present
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.html b/fre.html
new file mode 100644
index 00000000..50e621f4
--- /dev/null
+++ b/fre.html
@@ -0,0 +1,468 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.fre module
+Main host file for FRE-CLI program scripts +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL +2023-2024 +principal click group for main/fre allows for subgroup functions to +be called via this script. I.e. ‘fre’ is the entry point
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.lazy_group.html b/fre.lazy_group.html
new file mode 100644
index 00000000..f276b6de
--- /dev/null
+++ b/fre.lazy_group.html
@@ -0,0 +1,133 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre package
+Subpackages
+
+
+-
+
- fre.app package
-
+
- Subpackages + +
- Submodules + +
- Module contents +
+ - fre.catalog package + +
- fre.check package + +
- fre.cmor package
-
+
- Subpackages + +
- Submodules + +
- Module contents +
+ - fre.list package + +
- fre.make package
-
+
- Subpackages + +
- Submodules + +
- Module contents +
+ - fre.pp package + +
- fre.run package + +
- fre.test package + +
- fre.tests package
-
+
- Submodules
-
+
- fre.tests.test_fre_app_cli module
-
+
test_cli_fre_app()
+test_cli_fre_app_gen_time_averages()
+test_cli_fre_app_gen_time_averages_help()
+test_cli_fre_app_gen_time_averages_opt_dne()
+test_cli_fre_app_help()
+test_cli_fre_app_opt_dne()
+test_cli_fre_app_regrid()
+test_cli_fre_app_regrid_help()
+test_cli_fre_app_regrid_opt_dne()
+test_cli_fre_app_regrid_test_case_1()
+
+ - fre.tests.test_fre_catalog_cli module + +
- fre.tests.test_fre_check_cli module + +
- fre.tests.test_fre_cli module + +
- fre.tests.test_fre_cmor_cli module + +
- fre.tests.test_fre_list_cli module + +
- fre.tests.test_fre_make_cli module + +
- fre.tests.test_fre_pp_cli module
-
+
test_cli_fre_pp()
+test_cli_fre_pp_checkout()
+test_cli_fre_pp_checkout_help()
+test_cli_fre_pp_checkout_opt_dne()
+test_cli_fre_pp_configure_xml()
+test_cli_fre_pp_configure_xml_help()
+test_cli_fre_pp_configure_xml_opt_dne()
+test_cli_fre_pp_configure_yaml()
+test_cli_fre_pp_configure_yaml_help()
+test_cli_fre_pp_configure_yaml_opt_dne()
+test_cli_fre_pp_help()
+test_cli_fre_pp_install()
+test_cli_fre_pp_install_help()
+test_cli_fre_pp_install_opt_dne()
+test_cli_fre_pp_opt_dne()
+test_cli_fre_pp_run()
+test_cli_fre_pp_run_help()
+test_cli_fre_pp_run_opt_dne()
+test_cli_fre_pp_status()
+test_cli_fre_pp_status_help()
+test_cli_fre_pp_status_opt_dne()
+test_cli_fre_pp_validate()
+test_cli_fre_pp_validate_help()
+test_cli_fre_pp_validate_opt_dne()
+test_cli_fre_pp_wrapper()
+test_cli_fre_pp_wrapper_help()
+test_cli_fre_pp_wrapper_opt_dne()
+
+ - fre.tests.test_fre_run_cli module + +
- fre.tests.test_fre_test_cli module + +
- fre.tests.test_fre_yamltools_cli module + +
+ - fre.tests.test_fre_app_cli module
- Module contents +
+ - Submodules
- fre.yamltools package + +
Submodules
+
+
+
+Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.list.frelist.html b/fre.list.frelist.html
new file mode 100644
index 00000000..f18ca3bb
--- /dev/null
+++ b/fre.list.frelist.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.lazy_group module
+for lazy-style loading of commands and subcommands while using click +see https://click.palletsprojects.com/en/8.1.x/complex/
+-
+
- +class fre.lazy_group.LazyGroup(*args, lazy_subcommands=None, **kwargs) +
Bases:
+Groupclass defining lazygroup command/subcommand loading
+-
+
- +get_command(ctx, cmd_name) +
Given a context and a command name, this returns a +
+Commandobject if it exists or returns None.
-
+
- +list_commands(ctx) +
Returns a list of subcommand names in the order they should +appear.
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.list.frelistexample.html b/fre.list.frelistexample.html
new file mode 100644
index 00000000..c31f5faa
--- /dev/null
+++ b/fre.list.frelistexample.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.list.frelist module
+fre list
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.list.html b/fre.list.html
new file mode 100644
index 00000000..73130f32
--- /dev/null
+++ b/fre.list.html
@@ -0,0 +1,122 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.list.frelistexample module
+experimentation file for integrating one file’s functions into main prototype fre file +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.createCheckout.html b/fre.make.createCheckout.html
new file mode 100644
index 00000000..dc9496e5
--- /dev/null
+++ b/fre.make.createCheckout.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.createCompile.html b/fre.make.createCompile.html
new file mode 100644
index 00000000..80cf35ac
--- /dev/null
+++ b/fre.make.createCompile.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.createDocker.html b/fre.make.createDocker.html
new file mode 100644
index 00000000..b74b1042
--- /dev/null
+++ b/fre.make.createDocker.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.createMakefile.html b/fre.make.createMakefile.html
new file mode 100644
index 00000000..264b39a1
--- /dev/null
+++ b/fre.make.createMakefile.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.fremake.html b/fre.make.fremake.html
new file mode 100644
index 00000000..0df622e8
--- /dev/null
+++ b/fre.make.fremake.html
@@ -0,0 +1,110 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.html b/fre.make.html
new file mode 100644
index 00000000..237ac419
--- /dev/null
+++ b/fre.make.html
@@ -0,0 +1,163 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.fremake module
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.runFremake.html b/fre.make.runFremake.html
new file mode 100644
index 00000000..38060ab8
--- /dev/null
+++ b/fre.make.runFremake.html
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.tests.html b/fre.make.tests.html
new file mode 100644
index 00000000..fa3f19a3
--- /dev/null
+++ b/fre.make.tests.html
@@ -0,0 +1,128 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.runFremake module
+date 2023 +author(s): Tom Robinson, Dana Singh, Bennett Chang +fre make is used to create, run and checkout code, and compile a model.
+-
+
- +fre.make.runFremake.fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose) +
run fremake via click
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.make.tests.test_create_makefile.html b/fre.make.tests.test_create_makefile.html
new file mode 100644
index 00000000..8c8f2752
--- /dev/null
+++ b/fre.make.tests.test_create_makefile.html
@@ -0,0 +1,141 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.checkoutScript.html b/fre.pp.checkoutScript.html
new file mode 100644
index 00000000..58c57e13
--- /dev/null
+++ b/fre.pp.checkoutScript.html
@@ -0,0 +1,110 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.make.tests.test_create_makefile module
+Test fre make create-makefile
+-
+
- +fre.make.tests.test_create_makefile.test_bm_makefile_creation() +
Check the makefile is created when a bare-metal platform is used
+
-
+
- +fre.make.tests.test_create_makefile.test_compileyaml_exists() +
Check the compile yaml exists
+
-
+
- +fre.make.tests.test_create_makefile.test_container_makefile_creation() +
Check the makefile is created when the container platform is used
+
-
+
- +fre.make.tests.test_create_makefile.test_modelyaml_exists() +
Check the model yaml exists
+
-
+
- +fre.make.tests.test_create_makefile.test_platformyaml_exists() +
Check the platform yaml exists
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.configure_script_xml.html b/fre.pp.configure_script_xml.html
new file mode 100644
index 00000000..53103afb
--- /dev/null
+++ b/fre.pp.configure_script_xml.html
@@ -0,0 +1,183 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.checkoutScript module
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.configure_script_yaml.html b/fre.pp.configure_script_yaml.html
new file mode 100644
index 00000000..c649e3d0
--- /dev/null
+++ b/fre.pp.configure_script_yaml.html
@@ -0,0 +1,158 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.configure_script_xml module
+Primary Usage: fre-bronx-to-canopy -x XML -e EXP -p PLATFORM -t TARGET
+The Bronx-to-Canopy XML converter overwrites 3 files: +- rose-suite.conf +- app/remap-pp-components/rose-app.conf +- app/regrid-xy/rose-app.conf
+-
+
- +fre.pp.configure_script_xml.chunk_from_legacy(legacy_chunk) +
Return ISO8601 duration given Bronx-style chunk
+-
+
- Arguments:
-
+
legacy_chunk[str]: The Bronx chunk
+
+
-
+
- +fre.pp.configure_script_xml.duration_to_seconds(duration) +
Returns the conversion of a chunk duration to seconds
+-
+
- Arguments:
-
+
duration[str]: The original chunk duration
+
+
-
+
- +fre.pp.configure_script_xml.frelist_xpath(xml, platform, target, experiment, xpath) +
-
+
- Returns filepaths of FRE XML elements that use X-path notation
using Bronx’s ‘frelist’ command via subprocess module
+
+- Arguments:
-
+
args[str]: Argparse user-input arguments
+xpath[str]: X-path (XML) notation required by ‘frelist’
+
+
-
+
- +fre.pp.configure_script_xml.freq_from_legacy(legacy_freq) +
Return ISO8601 duration given Bronx-style frequencies
+-
+
- Arguments:
-
+
legacy_freq[str]: The Bronx frequency
+
+
-
+
- +fre.pp.configure_script_xml.main(xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual) +
The meat of the converter
+-
+
- Arguments:
-
+
args[argparse Namespace]: Arguments given at the command line
+
+- Tasks:
-
+
Generate key-value pairs for the rose-suite.conf.
+Generate content for the regrid-xy app
+Generate content for the remap-pp-components app
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.frepp.html b/fre.pp.frepp.html
new file mode 100644
index 00000000..5131ef8f
--- /dev/null
+++ b/fre.pp.frepp.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.configure_script_yaml module
+Script creates rose-apps and rose-suite +files for the workflow from the pp yaml.
+-
+
- +fre.pp.configure_script_yaml.quote_rose_values(value) +
rose-suite.conf template variables must be quoted unless they are +boolean, in which case do not quote them.
+
-
+
- +fre.pp.configure_script_yaml.rose_init(experiment, platform, target) +
Initialize the rose suite and app configurations.
+
-
+
- +fre.pp.configure_script_yaml.set_rose_apps(yamlfile, rose_regrid, rose_remap) +
Set items in the regrid and remap rose app configurations.
+
-
+
- +fre.pp.configure_script_yaml.set_rose_suite(yamlfile, rose_suite) +
Set items in the rose suite configuration.
+
-
+
- +fre.pp.configure_script_yaml.validate_yaml(yamlfile) +
Using the schema.json file, the yaml format is validated.
+
-
+
- +fre.pp.configure_script_yaml.yamlInfo(yamlfile, experiment, platform, target) +
Using a valid pp.yaml, the rose-app and rose-suite +configuration files are created in the cylc-src +directory. The pp.yaml is also copied to the +cylc-src directory.
+
-
+
- +fre.pp.configure_script_yaml.yaml_load(yamlfile) +
Load the given yaml and validate.
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.html b/fre.pp.html
new file mode 100644
index 00000000..9d6684c0
--- /dev/null
+++ b/fre.pp.html
@@ -0,0 +1,169 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.frepp module
+fre pp
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.install.html b/fre.pp.install.html
new file mode 100644
index 00000000..fe82319b
--- /dev/null
+++ b/fre.pp.install.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp package
+Subpackages
+ +Submodules
+ +Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.run.html b/fre.pp.run.html
new file mode 100644
index 00000000..018e10df
--- /dev/null
+++ b/fre.pp.run.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.install module
+fre pp install
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.status.html b/fre.pp.status.html
new file mode 100644
index 00000000..cd65b433
--- /dev/null
+++ b/fre.pp.status.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.run module
+fre pp run
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.tests.html b/fre.pp.tests.html
new file mode 100644
index 00000000..f964a896
--- /dev/null
+++ b/fre.pp.tests.html
@@ -0,0 +1,130 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.status module
+fre pp status
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.tests.test_configure_script_yaml.html b/fre.pp.tests.test_configure_script_yaml.html
new file mode 100644
index 00000000..f87ac527
--- /dev/null
+++ b/fre.pp.tests.test_configure_script_yaml.html
@@ -0,0 +1,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.tests.test_rose_quoting.html b/fre.pp.tests.test_rose_quoting.html
new file mode 100644
index 00000000..057091c5
--- /dev/null
+++ b/fre.pp.tests.test_rose_quoting.html
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.tests.test_configure_script_yaml module
+Test configure_script_yaml
+-
+
- +fre.pp.tests.test_configure_script_yaml.test_combinedyaml_exists() +
Make sure combined yaml file exists
+
-
+
- +fre.pp.tests.test_configure_script_yaml.test_configure_script() +
Tests success of confgure yaml script +Creates rose-suite, regrid rose-app, remap rose-app +TO-DO: will break this up for better tests
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.validate.html b/fre.pp.validate.html
new file mode 100644
index 00000000..16202280
--- /dev/null
+++ b/fre.pp.validate.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.pp.wrapper.html b/fre.pp.wrapper.html
new file mode 100644
index 00000000..ff38405c
--- /dev/null
+++ b/fre.pp.wrapper.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.validate module
+fre pp validate
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.run.frerun.html b/fre.run.frerun.html
new file mode 100644
index 00000000..c7aa3306
--- /dev/null
+++ b/fre.run.frerun.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.pp.wrapper module
+frepp.py, a replacement for the frepp bash script located at: +https://gitlab.gfdl.noaa.gov/fre2/system-settings/-/blob/main/bin/frepp +Author: Carolyn.Whitlock
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.run.frerunexample.html b/fre.run.frerunexample.html
new file mode 100644
index 00000000..fb15e4e6
--- /dev/null
+++ b/fre.run.frerunexample.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.run.frerun module
+entry point for fre run subcommands
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.run.html b/fre.run.html
new file mode 100644
index 00000000..4de56794
--- /dev/null
+++ b/fre.run.html
@@ -0,0 +1,122 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.run.frerunexample module
+experimentation file for integrating one file’s functions into main prototype fre file +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.test.fretest.html b/fre.test.fretest.html
new file mode 100644
index 00000000..59729a4a
--- /dev/null
+++ b/fre.test.fretest.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.test.fretestexample.html b/fre.test.fretestexample.html
new file mode 100644
index 00000000..cf915a27
--- /dev/null
+++ b/fre.test.fretestexample.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.test.fretest module
+entry point for fre test subcommands
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.test.html b/fre.test.html
new file mode 100644
index 00000000..a3bd3d25
--- /dev/null
+++ b/fre.test.html
@@ -0,0 +1,122 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.test.fretestexample module
+experimentation file for integrating one file’s functions into main prototype fre file +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.html b/fre.tests.html
new file mode 100644
index 00000000..688cbeb5
--- /dev/null
+++ b/fre.tests.html
@@ -0,0 +1,228 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_app_cli.html b/fre.tests.test_fre_app_cli.html
new file mode 100644
index 00000000..07c42362
--- /dev/null
+++ b/fre.tests.test_fre_app_cli.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests package
+Submodules
+
+
+-
+
- fre.tests.test_fre_app_cli module
-
+
test_cli_fre_app()
+test_cli_fre_app_gen_time_averages()
+test_cli_fre_app_gen_time_averages_help()
+test_cli_fre_app_gen_time_averages_opt_dne()
+test_cli_fre_app_help()
+test_cli_fre_app_opt_dne()
+test_cli_fre_app_regrid()
+test_cli_fre_app_regrid_help()
+test_cli_fre_app_regrid_opt_dne()
+test_cli_fre_app_regrid_test_case_1()
+
+ - fre.tests.test_fre_catalog_cli module + +
- fre.tests.test_fre_check_cli module + +
- fre.tests.test_fre_cli module + +
- fre.tests.test_fre_cmor_cli module + +
- fre.tests.test_fre_list_cli module + +
- fre.tests.test_fre_make_cli module + +
- fre.tests.test_fre_pp_cli module
-
+
test_cli_fre_pp()
+test_cli_fre_pp_checkout()
+test_cli_fre_pp_checkout_help()
+test_cli_fre_pp_checkout_opt_dne()
+test_cli_fre_pp_configure_xml()
+test_cli_fre_pp_configure_xml_help()
+test_cli_fre_pp_configure_xml_opt_dne()
+test_cli_fre_pp_configure_yaml()
+test_cli_fre_pp_configure_yaml_help()
+test_cli_fre_pp_configure_yaml_opt_dne()
+test_cli_fre_pp_help()
+test_cli_fre_pp_install()
+test_cli_fre_pp_install_help()
+test_cli_fre_pp_install_opt_dne()
+test_cli_fre_pp_opt_dne()
+test_cli_fre_pp_run()
+test_cli_fre_pp_run_help()
+test_cli_fre_pp_run_opt_dne()
+test_cli_fre_pp_status()
+test_cli_fre_pp_status_help()
+test_cli_fre_pp_status_opt_dne()
+test_cli_fre_pp_validate()
+test_cli_fre_pp_validate_help()
+test_cli_fre_pp_validate_opt_dne()
+test_cli_fre_pp_wrapper()
+test_cli_fre_pp_wrapper_help()
+test_cli_fre_pp_wrapper_opt_dne()
+
+ - fre.tests.test_fre_run_cli module + +
- fre.tests.test_fre_test_cli module + +
- fre.tests.test_fre_yamltools_cli module + +
Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_catalog_cli.html b/fre.tests.test_fre_catalog_cli.html
new file mode 100644
index 00000000..9c8c9bf2
--- /dev/null
+++ b/fre.tests.test_fre_catalog_cli.html
@@ -0,0 +1,151 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_app_cli module
+test “fre app” calls
+-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app(capfd) +
fre app
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_gen_time_averages(capfd) +
fre cmor run
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_gen_time_averages_help(capfd) +
fre cmor run –help
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_gen_time_averages_opt_dne(capfd) +
fre cmor run optionDNE
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_help(capfd) +
fre app –help
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_opt_dne(capfd) +
fre app optionDNE
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_regrid(capfd) +
fre cmor run
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_regrid_help(capfd) +
fre cmor run –help
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_regrid_opt_dne(capfd) +
fre cmor run optionDNE
+
-
+
- +fre.tests.test_fre_app_cli.test_cli_fre_app_regrid_test_case_1(capfd) +
fre cmor run –help
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_check_cli.html b/fre.tests.test_fre_check_cli.html
new file mode 100644
index 00000000..9e19e93e
--- /dev/null
+++ b/fre.tests.test_fre_check_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_catalog_cli module
+test “fre catalog” calls
+-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog() +
fre catalog
+
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_builder() +
fre catalog builder
+
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_builder_help() +
fre catalog builder –help
+
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_help() +
fre catalog –help
+
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_merge() +
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_merge_help() +
-
+
- +fre.tests.test_fre_catalog_cli.test_cli_fre_catalog_opt_dne() +
fre catalog optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_cli.html b/fre.tests.test_fre_cli.html
new file mode 100644
index 00000000..a10d1de9
--- /dev/null
+++ b/fre.tests.test_fre_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_check_cli module
+test “fre check” calls
+-
+
- +fre.tests.test_fre_check_cli.test_cli_fre_check() +
fre check
+
-
+
- +fre.tests.test_fre_check_cli.test_cli_fre_check_help() +
fre check –help
+
-
+
- +fre.tests.test_fre_check_cli.test_cli_fre_check_opt_dne() +
fre check optionDNE
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_cmor_cli.html b/fre.tests.test_fre_cmor_cli.html
new file mode 100644
index 00000000..6a1233a5
--- /dev/null
+++ b/fre.tests.test_fre_cmor_cli.html
@@ -0,0 +1,159 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_list_cli.html b/fre.tests.test_fre_list_cli.html
new file mode 100644
index 00000000..941944ec
--- /dev/null
+++ b/fre.tests.test_fre_list_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_cmor_cli module
+test “fre cmor” calls
+-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor() +
fre cmor
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_help() +
fre cmor –help
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_opt_dne() +
fre cmor optionDNE
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_run() +
fre cmor run
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_run_case1() +
fre cmor run, test-use case
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_run_case2() +
fre cmor run, test-use case
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_run_help() +
fre cmor run –help
+
-
+
- +fre.tests.test_fre_cmor_cli.test_cli_fre_cmor_run_opt_dne() +
fre cmor run optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_make_cli.html b/fre.tests.test_fre_make_cli.html
new file mode 100644
index 00000000..209e7bdc
--- /dev/null
+++ b/fre.tests.test_fre_make_cli.html
@@ -0,0 +1,141 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_list_cli module
+test “fre list” calls
+-
+
- +fre.tests.test_fre_list_cli.test_cli_fre_list() +
fre list
+
-
+
- +fre.tests.test_fre_list_cli.test_cli_fre_list_help() +
fre list –help
+
-
+
- +fre.tests.test_fre_list_cli.test_cli_fre_list_opt_dne() +
fre list optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_pp_cli.html b/fre.tests.test_fre_pp_cli.html
new file mode 100644
index 00000000..259f61bc
--- /dev/null
+++ b/fre.tests.test_fre_pp_cli.html
@@ -0,0 +1,273 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_make_cli module
+test “fre make” calls
+-
+
- +fre.tests.test_fre_make_cli.test_cli_fre_make() +
fre make
+
-
+
- +fre.tests.test_fre_make_cli.test_cli_fre_make_create_checkout_baremetal() +
fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t debug
+
-
+
- +fre.tests.test_fre_make_cli.test_cli_fre_make_create_checkout_container() +
fre make create-checkout -y am5.yaml -p hpcme.2023 -t debug
+
-
+
- +fre.tests.test_fre_make_cli.test_cli_fre_make_help() +
fre make –help
+
-
+
- +fre.tests.test_fre_make_cli.test_cli_fre_make_opt_dne() +
fre make optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_run_cli.html b/fre.tests.test_fre_run_cli.html
new file mode 100644
index 00000000..c682341c
--- /dev/null
+++ b/fre.tests.test_fre_run_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_pp_cli module
+test “fre pp” calls
+-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp() +
fre pp
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_checkout() +
fre pp checkout
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_checkout_help() +
fre pp checkout –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_checkout_opt_dne() +
fre pp checkout optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_xml() +
fre pp configure-xml
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_xml_help() +
fre pp configure-xml –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_xml_opt_dne() +
fre pp configure-xml optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_yaml() +
fre pp configure-yaml
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_yaml_help() +
fre pp configure-yaml –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_configure_yaml_opt_dne() +
fre pp configure-yaml optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_help() +
fre pp –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_install() +
fre pp install
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_install_help() +
fre pp install –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_install_opt_dne() +
fre pp install optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_opt_dne() +
fre pp optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_run() +
fre pp run
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_run_help() +
fre pp run –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_run_opt_dne() +
fre pp run optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_status() +
fre pp status
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_status_help() +
fre pp status –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_status_opt_dne() +
fre pp status optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_validate() +
fre pp validate
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_validate_help() +
fre pp validate –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_validate_opt_dne() +
fre pp validate optionDNE
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_wrapper() +
fre pp wrapper
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_wrapper_help() +
fre pp wrapper –help
+
-
+
- +fre.tests.test_fre_pp_cli.test_cli_fre_pp_wrapper_opt_dne() +
fre pp wrapper optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_test_cli.html b/fre.tests.test_fre_test_cli.html
new file mode 100644
index 00000000..a11a5988
--- /dev/null
+++ b/fre.tests.test_fre_test_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.tests.test_fre_yamltools_cli.html b/fre.tests.test_fre_yamltools_cli.html
new file mode 100644
index 00000000..b21be134
--- /dev/null
+++ b/fre.tests.test_fre_yamltools_cli.html
@@ -0,0 +1,129 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_test_cli module
+test “fre test” calls
+-
+
- +fre.tests.test_fre_test_cli.test_cli_fre_test() +
fre test
+
-
+
- +fre.tests.test_fre_test_cli.test_cli_fre_test_help() +
fre test –help
+
-
+
- +fre.tests.test_fre_test_cli.test_cli_fre_test_opt_dne() +
fre test optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.combine_yamls.html b/fre.yamltools.combine_yamls.html
new file mode 100644
index 00000000..8d6716a7
--- /dev/null
+++ b/fre.yamltools.combine_yamls.html
@@ -0,0 +1,231 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.tests.test_fre_yamltools_cli module
+test “fre yamltools” calls
+-
+
- +fre.tests.test_fre_yamltools_cli.test_cli_fre_yamltools() +
fre yamltools
+
-
+
- +fre.tests.test_fre_yamltools_cli.test_cli_fre_yamltools_help() +
fre yamltools –help
+
-
+
- +fre.tests.test_fre_yamltools_cli.test_cli_fre_yamltools_opt_dne() +
fre yamltools optionDNE
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.freyamltools.html b/fre.yamltools.freyamltools.html
new file mode 100644
index 00000000..b680817b
--- /dev/null
+++ b/fre.yamltools.freyamltools.html
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.combine_yamls module
+Script combines the model yaml with the compile, platform, and experiment yamls.
+-
+
- +fre.yamltools.combine_yamls.combined_compile_existcheck(combined, yml, platform, target) +
Checks for if combined compile yaml exists already. +If not, combine model, compile, and platform yamls.
+
-
+
- +fre.yamltools.combine_yamls.consolidate_yamls(yamlfile, experiment, platform, target, use) +
-
+
- +fre.yamltools.combine_yamls.experiment_check(mainyaml_dir, comb, experiment) +
Check that the experiment given is an experiment listed in the model yaml. +Extract experiment specific information and file paths. +Arguments: +mainyaml_dir : model yaml file +comb : combined yaml file name +experiment : experiment name
+
-
+
- +fre.yamltools.combine_yamls.get_combined_compileyaml(comb) +
Combine the model, compile, and platform yamls +Arguments: +comb : combined yaml object
+
-
+
- +fre.yamltools.combine_yamls.get_combined_ppyaml(comb) +
Combine the model, experiment, and analysis yamls +Arguments: +comb : combined yaml object
+
-
+
- +fre.yamltools.combine_yamls.get_compile_paths(mainyaml_dir, comb) +
Extract compile and platform paths from model yaml
+
-
+
- +class fre.yamltools.combine_yamls.init_compile_yaml(yamlfile, platform, target) +
Bases:
+object-
+
- +clean_yaml() +
Clean the yaml; remove unnecessary sections in +final combined yaml.
+
-
+
- +combine_compile() +
Combine compile yaml with the defined combined.yaml
+
-
+
- +combine_model() +
Create the combined.yaml and merge it with the model yaml
+
-
+
- +combine_platforms() +
Combine platforms yaml with the defined combined.yaml
+
-
+
- +class fre.yamltools.combine_yamls.init_pp_yaml(yamlfile, experiment, platform, target) +
Bases:
+object-
+
- +clean_yaml() +
Clean the yaml; remove unnecessary sections in +final combined yaml.
+
-
+
- +combine_analysis() +
Combine analysis yamls with the defined combined.yaml
+
-
+
- +combine_experiment() +
Combine experiment yamls with the defined combined.yaml
+
-
+
- +combine_model() +
Create the combined.yaml and merge it with the model yaml
+
-
+
- +fre.yamltools.combine_yamls.join_constructor(loader, node) +
Allows FRE properties defined +in main yaml to be concatenated.
+
-
+
- +fre.yamltools.combine_yamls.yaml_load(yamlfile) +
Load the yamlfile
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.freyamltoolsexample.html b/fre.yamltools.freyamltoolsexample.html
new file mode 100644
index 00000000..74fde119
--- /dev/null
+++ b/fre.yamltools.freyamltoolsexample.html
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.freyamltools module
+fre yamltools
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.html b/fre.yamltools.html
new file mode 100644
index 00000000..08e37f91
--- /dev/null
+++ b/fre.yamltools.html
@@ -0,0 +1,175 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.freyamltoolsexample module
+experimentation file for integrating one file’s functions into main prototype fre file +authored by Bennett.Chang@noaa.gov | bcc2761 +NOAA | GFDL
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.tests.html b/fre.yamltools.tests.html
new file mode 100644
index 00000000..fa7ba26b
--- /dev/null
+++ b/fre.yamltools.tests.html
@@ -0,0 +1,134 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools package
+Subpackages
+
+
+-
+
- fre.yamltools.tests package
-
+
- Submodules
-
+
- fre.yamltools.tests.test_combine_yamls module
-
+
test_analysisyaml_exists()
+test_combined_compileyaml_combinefail()
+test_combined_compileyaml_validatefail()
+test_combined_compileyaml_validation()
+test_combined_ppyaml_validation()
+test_compileyaml_exists()
+test_expyaml_exists()
+test_merged_compile_yamls()
+test_merged_pp_yamls()
+test_modelyaml_exists()
+test_platformyaml_exists()
+
+
+ - fre.yamltools.tests.test_combine_yamls module
- Module contents +
+ - Submodules
Submodules
+
+
+-
+
- fre.yamltools.combine_yamls module + +
- fre.yamltools.freyamltools module +
- fre.yamltools.freyamltoolsexample module +
Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/fre.yamltools.tests.test_combine_yamls.html b/fre.yamltools.tests.test_combine_yamls.html
new file mode 100644
index 00000000..ac6c2851
--- /dev/null
+++ b/fre.yamltools.tests.test_combine_yamls.html
@@ -0,0 +1,181 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.tests package
+Submodules
+
+
+-
+
- fre.yamltools.tests.test_combine_yamls module
-
+
test_analysisyaml_exists()
+test_combined_compileyaml_combinefail()
+test_combined_compileyaml_validatefail()
+test_combined_compileyaml_validation()
+test_combined_ppyaml_validation()
+test_compileyaml_exists()
+test_expyaml_exists()
+test_merged_compile_yamls()
+test_merged_pp_yamls()
+test_modelyaml_exists()
+test_platformyaml_exists()
+
+
Module contents
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/genindex.html b/genindex.html
new file mode 100644
index 00000000..ae99beff
--- /dev/null
+++ b/genindex.html
@@ -0,0 +1,1388 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fre.yamltools.tests.test_combine_yamls module
+tests routines in fre.yamltools.combine_yamls
+-
+
- +fre.yamltools.tests.test_combine_yamls.test_analysisyaml_exists() +
Make sure experiment yaml file exists
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_combined_compileyaml_combinefail() +
Check to test if compile yaml is incorrect/does not exist, +the combine fails. (compile yaml path misspelled)
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_combined_compileyaml_validatefail() +
Check if the schema is validating correctly +Branch should be string
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_combined_compileyaml_validation() +
Validate the combined compile yaml
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_combined_ppyaml_validation() +
Validate the combined compile yaml
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_compileyaml_exists() +
Make sure experiment yaml file exists
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_expyaml_exists() +
Make sure experiment yaml file exists
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_merged_compile_yamls() +
Check for the creation of the combined-[experiment] yaml +Check that the model yaml was merged into the combined yaml
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_merged_pp_yamls() +
Check for the creation of the combined-[experiment] yaml +Check that the model yaml was merged into the combined yaml
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_modelyaml_exists() +
Make sure main yaml file exists
+
-
+
- +fre.yamltools.tests.test_combine_yamls.test_platformyaml_exists() +
Make sure experiment yaml file exists
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/index.html b/index.html
new file mode 100644
index 00000000..3eaf7d65
--- /dev/null
+++ b/index.html
@@ -0,0 +1,164 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Index
+ +
+ A
+ | C
+ | D
+ | E
+ | F
+ | G
+ | I
+ | J
+ | L
+ | M
+ | P
+ | Q
+ | R
+ | S
+ | T
+ | U
+ | V
+ | W
+ | Y
+
+
+A
+| + |
C
+D
+| + | + |
E
+| + |
F
+
|
+
|
+
G
+I
+| + | + |
J
+| + |
L
+| + | + |
M
+P
+| + | + |
Q
+| + |
R
+| + | + |
S
+| + | + |
T
+U
+| + |
V
+| + | + |
W
+| + |
Y
+| + | + |
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/lib/docs/conf.py b/lib/docs/conf.py
new file mode 100644
index 00000000..32399782
--- /dev/null
+++ b/lib/docs/conf.py
@@ -0,0 +1,29 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Fre-Cli'
+copyright = '2024, Bennett Chang'
+author = 'Bennett Chang'
+release = '1.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['sphinx.ext.autodoc']
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'renku'
+#html_theme = 'sphinx_rtd_theme'
+html_static_path = ['_static']
diff --git a/lib/fre/README.md b/lib/fre/README.md
new file mode 100644
index 00000000..4570b317
--- /dev/null
+++ b/lib/fre/README.md
@@ -0,0 +1,43 @@
+
+* Refer to fre-cli [README.md](https://github.com/NOAA-GFDL/fre-cli/blob/main/README.md) for usage and tips.
+
+
diff --git a/lib/fre/__init__.py b/lib/fre/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/app/__init__.py b/lib/fre/app/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/app/freapp.py b/lib/fre/app/freapp.py
new file mode 100644
index 00000000..545356e4
--- /dev/null
+++ b/lib/fre/app/freapp.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+''' fre app calls '''
+
+import time
+
+import click
+
+from .mask_atmos_plevel import mask_atmos_plevel_subtool
+from .generate_time_averages.generate_time_averages import generate
+from .regrid_xy.regrid_xy import _regrid_xy
+
+@click.group(help=click.style(" - access fre app subcommands", fg=(250,154,90)))
+def app_cli():
+ ''' entry point to fre app click commands '''
+
+@app_cli.command()
+@click.option("-i", "--input_dir",
+ type = str,
+ help = "`inputDir` / `input_dir` (env var) specifies input directory to regrid, " + \
+ "typically an untarredv history file archive" ,
+ required = True)
+@click.option("-o", "--output_dir",
+ type = str,
+ help = "`outputDir` / `output_dir` (env var) specifies target location for output" + \
+ " regridded files",
+ required = True)
+@click.option("-b", "--begin",
+ type = str,
+ help = "`begin` / `begin` (env var) ISO8601 datetime format specification for" + \
+ " starting date of data, part of input target file name",
+ required = True)
+@click.option("-tmp", "--tmp_dir",
+ type = str,
+ help = "`TMPDIR` / `tmp_dir` (env var) temp directory for location of file " + \
+ "read/writes",
+ required = True)
+@click.option("-rd", "--remap_dir",
+ type = str,
+ help = "`fregridRemapDir` / `remap_dir` (env var) directory containing remap file" + \
+ " for regridding",
+ required = True)
+@click.option("-s", "--source",
+ type = str,
+ help = "`source` / `source` (env var) source name for input target file name " + \
+ "within input directory to target for regridding. the value for `source` " + \
+ "must be present in at least one component's configuration fields",
+ required = True)
+@click.option("-g", "--grid_spec",
+ type = str,
+ help = "`gridSpec` / `grid_spec` (env var) file containing mosaic for regridding",
+ required = True)
+@click.option("-xy", "--def_xy_interp",
+ type = str,
+ help = "`defaultxyInterp` / `def_xy_interp` (env var) default lat/lon resolution " + \
+ "for output regridding. (change me? TODO)",
+ required = True)
+@click.pass_context
+def regrid(context,
+ input_dir, output_dir, begin, tmp_dir,
+ remap_dir, source, grid_spec, def_xy_interp ):
+ ''' regrid target netcdf file '''
+ context.forward(_regrid_xy)
+
+@app_cli.command()
+@click.option("-i", "--infile",
+ type = str,
+ help = "Input NetCDF file containing pressure-level output to be masked",
+ required = True)
+@click.option("-o", "--outfile",
+ type = str,
+ help = "Output file",
+ required = True)
+@click.option("-p", "--psfile", # surface pressure... ps? TODO
+ help = "Input NetCDF file containing surface pressure (ps)",
+ required = True)
+@click.pass_context
+def mask_atmos_plevel(context, infile, outfile, psfile):
+ # pylint: disable = unused-argument
+ """Mask out pressure level diagnostic output below land surface"""
+ context.forward(mask_atmos_plevel_subtool)
+
+
+@app_cli.command()
+@click.option("-i", "--inf",
+ type = str,
+ required = True,
+ help = "Input file name")
+@click.option("-o", "--outf",
+ type = str,
+ required = True,
+ help = "Output file name")
+@click.option("-p", "--pkg",
+ type = click.Choice(["cdo","fre-nctools","fre-python-tools"]),
+ default = "cdo",
+ help = "Time average approach")
+@click.option("-v", "--var",
+ type = str,
+ default = None,
+ help = "Specify variable to average")
+@click.option("-u", "--unwgt",
+ is_flag = True,
+ default = False,
+ help = "Request unweighted statistics")
+@click.option("-a", "--avg_type",
+ type = click.Choice(["month","seas","all"]),
+ default = "all",
+ help = "Type of time average to generate. \n \
+ currently, fre-nctools and fre-python-tools pkg options\n \
+ do not support seasonal and monthly averaging.\n")
+@click.option("-s", "--stddev_type",
+ type = click.Choice(["samp","pop","samp_mean","pop_mean"]),
+ default = "samp",
+ help = "Compute standard deviations for time-averages as well")
+@click.pass_context
+def gen_time_averages(context, inf, outf, pkg, var, unwgt, avg_type, stddev_type):
+ # pylint: disable = unused-argument
+ """
+ generate time averages for specified set of netCDF files.
+ Example: generate-time-averages.py /path/to/your/files/
+ """
+ start_time = time.perf_counter()
+ context.forward(generate)
+ # need to change to a click.echo, not sure if echo supports f strings
+ print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
+
+if __name__ == "__main__":
+ app_cli()
diff --git a/lib/fre/app/generate_time_averages/README.md b/lib/fre/app/generate_time_averages/README.md
new file mode 100644
index 00000000..49991e13
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/README.md
@@ -0,0 +1,8 @@
+From an input netCDF file, output a time-averaged netCDF file.
+
+Will average all available information on a given variable across time,
+but can also handle monthly+seasonal averaging.
+
+To run time-averaging tests, return to root directory and call just those
+tests with `python -m pytest tests/test_generate_time_averages.py`, or run
+all tests with `python -m pytests tests/`
\ No newline at end of file
diff --git a/lib/fre/app/generate_time_averages/__init__.py b/lib/fre/app/generate_time_averages/__init__.py
new file mode 100644
index 00000000..37410fef
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/__init__.py
@@ -0,0 +1,3 @@
+'''required for generate_time_averages module import functionality'''
+__all__ = ['generate_time_averages', 'timeAverager',
+ 'frenctoolsTimeAverager', 'cdoTimeAverager', 'frepytoolsTimeAverager']
diff --git a/lib/fre/app/generate_time_averages/cdoTimeAverager.py b/lib/fre/app/generate_time_averages/cdoTimeAverager.py
new file mode 100644
index 00000000..d288bebb
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/cdoTimeAverager.py
@@ -0,0 +1,85 @@
+''' class using (mostly) cdo functions for time-averages '''
+from .timeAverager import timeAverager
+
+class cdoTimeAverager(timeAverager):
+ '''
+ class inheriting from abstract base class timeAverager
+ generates time-averages using cdo (mostly, see weighted approach)
+ '''
+
+ def generate_timavg(self, infile=None, outfile=None):
+ ''' use cdo package routines via python bindings '''
+ assert self.pkg=="cdo"
+ if __debug__:
+ print(locals()) #input argument details
+
+ if all([self.avg_type!='all',self.avg_type!='seas',self.avg_type!='month',
+ self.avg_type is not None]):
+ print('ERROR, avg_type requested unknown.')
+ return 1
+
+ if self.var is not None:
+ print(f'WARNING: variable specification (var={self.var})' + \
+ f' not currently supported for cdo time averaging. ignoring!')
+
+ import cdo
+ print(f'python-cdo version is {cdo.__version__}')
+ from cdo import Cdo
+
+ _cdo=Cdo()
+
+ wgts_sum=0
+ if not self.unwgt: #weighted case, cdo ops alone don't support a weighted time-average.
+ from netCDF4 import Dataset
+ import numpy
+
+ nc_fin = Dataset(infile, 'r')
+
+ time_bnds=nc_fin['time_bnds'][:].copy()
+ wgts = ( numpy.moveaxis(time_bnds,0,-1)[1][:].copy() - \
+ numpy.moveaxis(time_bnds,0,-1)[0][:].copy() )
+ wgts_sum=sum(wgts)
+ if __debug__:
+ print(f'wgts_sum={wgts_sum}')
+
+ if self.avg_type == 'all':
+ if self.stddev_type is None:
+ print('time average over all time requested.')
+ if self.unwgt:
+ _cdo.timmean(input=infile, output=outfile, returnCdf=True)
+ else:
+ _cdo.divc( str(wgts_sum), input="-timsum -muldpm "+infile, output=outfile)
+
+ print('done averaging over all time.')
+ else:
+ print('time standard-deviation (N-1) over all time requested.')
+ _cdo.timstd1(input=infile, output=outfile, returnCdf=True)
+ print('done computing standard-deviation over all time.')
+
+ elif self.avg_type == 'seas':
+ if self.stddev_type is None:
+ print('seasonal time-averages requested.')
+ _cdo.yseasmean(input=infile, output=outfile, returnCdf=True)
+ print('done averaging over seasons.')
+ else:
+ print('seasonal time standard-deviation (N-1) requested.')
+ _cdo.yseasstd1(input=infile, output=outfile, returnCdf=True)
+ print('done averaging over seasons.')
+
+ elif self.avg_type == 'month':
+
+ if self.stddev_type is None:
+ print('monthly time-averages requested.')
+ _cdo.ymonmean(input=infile, output=outfile, returnCdf=True)
+ print('done averaging over months.')
+ else:
+ print('monthly time standard-deviation (N-1) requested.')
+ _cdo.ymonstd1(input=infile, output=outfile, returnCdf=True)
+ print('done averaging over months.')
+
+ else:
+ print(f'problem: unknown avg_type={self.avg_type}')
+ return 1
+
+ print('done averaging')
+ return 0
diff --git a/lib/fre/app/generate_time_averages/frenctoolsTimeAverager.py b/lib/fre/app/generate_time_averages/frenctoolsTimeAverager.py
new file mode 100644
index 00000000..32b9c2bb
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/frenctoolsTimeAverager.py
@@ -0,0 +1,55 @@
+''' class for utilizing timavg.csh (aka script to TAVG fortran exe) in frenc-tools '''
+from .timeAverager import timeAverager
+
+class frenctoolsTimeAverager(timeAverager):
+ '''
+ class inheriting from abstract base class timeAverager
+ generates time-averages using fre-nctools
+ '''
+
+ def generate_timavg(self, infile=None, outfile=None):
+ ''' use fre-nctool's CLI timavg.csh with subprocess call '''
+ assert self.pkg=="fre-nctools"
+ if __debug__:
+ print(locals()) #input argument details
+
+ exitstatus=1
+ if self.avg_type!='all':
+ print(f'ERROR: avg_type={self.avg_type} not supported by this class at this time.')
+ return exitstatus
+
+ if self.unwgt:
+ print('WARNING: unwgt=True unsupported by frenctoolsAverager. ignoring!!!')
+
+ if self.stddev_type is not None:
+ print('WARNING: stddev_type arg unsupported by frenctoolsTimeAverager. ignoring!!!')
+
+ if self.var is not None:
+ print(f'WARNING: variable specification (var={self.var})' + \
+ f' not currently supported for frenctols time averaging. ignoring!')
+
+ if infile is None:
+ print('ERROR: I need an input file, specify a value for the infile argument')
+ return exitstatus
+
+ if outfile is None:
+ outfile='frenctoolsTimeAverage_'+infile
+ print(f'WARNING: no output filename given, setting outfile={outfile}')
+
+ from subprocess import Popen, PIPE
+
+ precision='-r8'
+ timavgcsh_command=['timavg.csh', precision, '-mb','-o', outfile, infile]
+ exitstatus=1
+ with Popen(timavgcsh_command,
+ stdout=PIPE, stderr=PIPE, shell=False) as subp:
+ output=subp.communicate()[0]
+ print(f'output={output}')
+
+ if subp.returncode < 0:
+ print('error')
+ else:
+ print('success')
+ exitstatus=0
+
+ return exitstatus
diff --git a/lib/fre/app/generate_time_averages/frepytoolsTimeAverager.py b/lib/fre/app/generate_time_averages/frepytoolsTimeAverager.py
new file mode 100644
index 00000000..1e1e6638
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/frepytoolsTimeAverager.py
@@ -0,0 +1,249 @@
+''' class for python-native routine usuing netCDF4 and numpy to crunch time-averages '''
+
+import math
+import numpy
+from netCDF4 import Dataset
+
+from .timeAverager import timeAverager
+
+class frepytoolsTimeAverager(timeAverager):
+ '''
+ class inheriting from abstract base class timeAverager
+ generates time-averages using a python-native approach
+ avoids using other third party statistics functions by design.
+ '''
+
+ def generate_timavg(self, infile=None, outfile=None):
+ ''' frepytools approach in a python-native manner.
+ deliberately avoids pre-packaged routines. '''
+ assert self.pkg=="fre-python-tools"
+ if __debug__:
+ print(locals()) #input argument details
+
+ if __debug__:
+ print('calling generate_frepythontools_timavg for file: ' + infile)
+
+ if self.avg_type != 'all':
+ print(f'ERROR: avg_type={self.avg_type} not supported at this time.')
+ return 1
+
+ # (TODO) file I/O should be a sep function, no? make tests, extend
+ nc_fin = Dataset(infile, 'r')
+ if nc_fin.file_format != 'NETCDF4':
+ print(f'INFO: input file is not netCDF4 format, is {nc_fin.file_format}')
+
+ # (TODO) make this a sep function, make tests, extend
+ # identifying the input variable, two approaches
+ # user inputs target variable OR
+ # attempt to determine target var w/ bronx convention
+ if self.var is not None:
+ targ_var = self.var
+ else: # this can be replaced w/ a regex search maybe
+ targ_var = infile.split('/').pop().split('.')[-2]
+
+ if __debug__:
+ print(f'targ_var={targ_var}')
+
+ # (TODO) make this a sep function, make tests, extend
+ # check for the variable we're hoping is in the file
+ time_bnds = None
+ nc_fin_vars = nc_fin.variables
+ for key in nc_fin_vars:
+ if str(key) == targ_var: # found our variable, grab bounds
+ time_bnds = nc_fin['time_bnds'][:].copy()
+ break
+ if time_bnds is None:
+ print('requested variable not found. exit.')
+ return 1
+
+
+ # (TODO) determine what we need to worry about with masks and account for it
+ # check for mask, adjust accordingly
+ #is_masked = ma.is_masked(val_array)
+
+ # (TODO) make this a sep function, make tests, extend
+ # read in sizes of specific axes / compute weights
+ # weights can be encoded as a class member, whose existence
+ # depends on the user specifying unwgt=True, if vect_wgts=None, set the avg
+ # and stddev gen functions to the appropriate behavior (TODO)
+ fin_dims = nc_fin.dimensions
+ num_time_bnds = fin_dims['time'].size
+ if not self.unwgt: #compute sum of weights
+ wgts = ( numpy.moveaxis( time_bnds,0,-1 )[1][:].copy() - \
+ numpy.moveaxis( time_bnds,0,-1 )[0][:].copy() )
+ wgts_sum=sum(wgts)
+ if __debug__:
+ print(f'wgts_sum={wgts_sum}')
+
+
+ # initialize arrays, is there better practice for reserving the memory necessary
+ # for holding the day? is something that does more write-on-demand possible like
+ # reading data on-demand? (TODO)
+ num_lat_bnds=fin_dims['lat'].size
+ print(f'num_lat_bnds={num_lat_bnds}')
+ num_lon_bnds=fin_dims['lon'].size
+ print(f'num_lon_bnds={num_lon_bnds}')
+ avgvals=numpy.zeros((1,num_lat_bnds,num_lon_bnds),dtype=float)
+ if self.stddev_type is not None:
+ print('computing std. deviations')
+ stddevs=numpy.zeros((1,num_lat_bnds,num_lon_bnds),dtype=float)
+
+ # this loop behavior 100% should be re-factored into generator functions.
+ # they should be slightly faster, and much more readable. (TODO)
+ # the average/stddev cpu settings should also be genfunctions, their behavior
+ # (e.g. stddev_pop v stddev_samp) should be set given user inputs. (TODO)
+ # the computations can lean on numpy.stat more- i imagine it's faster (TODO)
+ # parallelism via multiprocessing shouldn't be too bad- explore an alt [dask] too (TODO)
+ # compute average, for each lat/lon coordinate over time record in file
+ if not self.unwgt: #weighted case
+ print('computing weighted statistics')
+ for lat in range(num_lat_bnds):
+ lon_val_array=numpy.moveaxis( nc_fin[targ_var][:],0,-1)[lat].copy()
+
+ for lon in range(num_lon_bnds):
+ tim_val_array= lon_val_array[lon].copy()
+ avgvals[0][lat][lon]=sum( (tim_val_array[tim] * wgts[tim] )
+ for tim in range(num_time_bnds) ) / wgts_sum
+
+ if self.stddev_type is not None: # implement stddeviation types TO DO
+ stddevs[0][lat][lon]=math.sqrt(
+ sum( wgts[tim] *
+ (tim_val_array[tim]-avgvals[0][lat][lon]) ** 2
+ for tim in range(num_time_bnds) )
+ / wgts_sum )
+ del tim_val_array
+ del lon_val_array
+ else: #unweighted case
+ print('computing unweighted statistics')
+ for lat in range(num_lat_bnds):
+ lon_val_array=numpy.moveaxis( nc_fin[targ_var][:],0,-1)[lat].copy()
+
+ for lon in range(num_lon_bnds):
+ tim_val_array= lon_val_array[lon].copy()
+ avgvals[0][lat][lon]=sum( # no time sum needed here, b.c. unweighted, so sum
+ tim_val_array[tim] for tim in range(num_time_bnds)
+ ) / num_time_bnds
+
+ if self.stddev_type is not None:
+
+ stddevs[0][lat][lon]=math.sqrt(
+ sum(
+ (tim_val_array[tim]-avgvals[0][lat][lon]) ** 2
+ for tim in range(num_time_bnds)
+ ) / ( (num_time_bnds - 1. ) )
+ )
+ del tim_val_array
+ del lon_val_array
+
+
+
+ # write output file
+ # (TODO) make this a sep function, make tests, extend,
+ # (TODO) consider compression particular;y for NETCDF file writing
+ # consider this approach instead:
+ # with Dataset( outfile, 'w', format='NETCDF4', persist=True ) as nc_fout:
+ nc_fout= Dataset( outfile, 'w', format=nc_fin.file_format, persist=True )
+
+ # (TODO) make this a sep function, make tests, extend
+ # write file global attributes
+ print('------- writing output attributes. --------')
+ unwritten_ncattr_list=[]
+ try:
+ nc_fout.setncatts(nc_fin.__dict__) #this copies the global attributes exactly.
+ except: # if the first way doesn't work...
+ print('could not copy ncatts from input file. trying to copy one-by-one')
+ fin_ncattrs=nc_fin.ncattrs()
+ for ncattr in fin_ncattrs:
+ print(f'\n_________\nncattr={ncattr}')
+ try:
+ nc_fout.setncattr(ncattr, nc_fin.getncattr(ncattr))
+ except:
+ print(f'could not get nc file attribute: {ncattr}. moving on.')
+ unwritten_ncattr_list.append(ncattr)
+ if len(unwritten_ncattr_list)>0:
+ print(f'WARNING: Some global attributes ({unwritten_ncattr_list}) were not written.')
+ print('------- DONE writing output attributes. --------')
+ ##
+
+ # (TODO) make this a sep function, make tests, extend
+ # write file dimensions
+ print('\n ------ writing output dimensions. ------ ')
+ unwritten_dims_list=[]
+ for key in fin_dims:
+ try:
+ if key=='time':
+ # this strongly influences the final data structure shape of the averages.
+ # if set to None, and lets say you try to write
+ # e.g. the original 'time_bnds' (which has 60 time steps)
+ # the array holding the avg. value will suddently have 60 time steps
+ # even though only 1 is needed, 59 time steps will have no data
+ #nc_fout.createDimension( dimname=key, size=None )
+ nc_fout.createDimension( dimname=key, size=1)
+ else:
+ nc_fout.createDimension( dimname=key, size=fin_dims[key].size )
+ except:
+ print(f'problem. cannot read/write dimension {key}')
+ unwritten_dims_list.append(key)
+ if len(unwritten_dims_list)>0:
+ print(f'WARNING: Some dimensions ({unwritten_dims_list}) were not written.')
+ print('------ DONE writing output dimensions. ------- \n')
+ ##
+
+
+ # (TODO) make this a sep function, make tests, extend
+ # first write the data we care most about- those we computed
+ # copying metadata, not fully correct
+ # but not far from wrong according to CF
+ # cell_methods must be changed TO DO
+ print(f'\n------- writing data for data {targ_var} -------- ')
+ nc_fout.createVariable(targ_var, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions)
+ nc_fout.variables[targ_var].setncatts(nc_fin[targ_var].__dict__)
+
+
+ nc_fout.variables[targ_var][:]=avgvals
+ if self.stddev_type is not None:
+ stddev_varname=targ_var+'_'+self.stddev_type+'_stddev'
+ nc_fout.createVariable(
+ stddev_varname, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions )
+ nc_fout.variables[stddev_varname].setncatts(nc_fin[targ_var].__dict__)
+ nc_fout.variables[stddev_varname][:]=stddevs
+ print('---------- DONE writing output variables. ---------')
+ ##
+
+ # (TODO) make this a sep function, make tests, extend
+ # write OTHER output variables (aka data) #prev code.
+ print('\n------- writing other output variables. -------- ')
+ unwritten_var_list=[]
+ unwritten_var_ncattr_dict={}
+ for var in nc_fin_vars:
+ if var != targ_var:
+ print(f'\nattempting to create output variable: {var}')
+ #print(f'is it a time variable? {self.var_is_time(nc_fin.variables[var])}')
+ nc_fout.createVariable(var, nc_fin[var].dtype, nc_fin[var].dimensions)
+ nc_fout.variables[var].setncatts(nc_fin[var].__dict__)
+ try:
+ nc_fout.variables[var][:] = nc_fin[var][:]
+ except:
+ print(f'could not write var={var}. i bet its the shape!')
+ print(f'nc_fin[var].shape={nc_fin[var].shape}')
+ #print(f'len(nc_fout.variables[{var}])={len(nc_fout.variables[var])}')
+ nc_fout.variables[var][:] = [ nc_fin[var][0] ]
+ print(f'time variable? {self.var_has_time_units(nc_fin.variables[var])}')
+ else:
+ continue
+
+ if len(unwritten_var_list)>0:
+ print(f'WARNING: some variables\' data ({unwritten_var_list}) was not written.')
+ if len(unwritten_var_ncattr_dict)>0:
+ print('WARNING: some variables\' metadata was not successfully written.')
+ print(f'WARNING: relevant variable/attr pairs: \n{unwritten_var_ncattr_dict}')
+ print('---------- DONE writing output variables. ---------')
+ ##
+
+
+ nc_fout.close()
+ #close input file
+ nc_fin.close()
+ print(f'wrote ouput file: {outfile}')
+
+ return 0
diff --git a/lib/fre/app/generate_time_averages/generate_time_averages.py b/lib/fre/app/generate_time_averages/generate_time_averages.py
new file mode 100644
index 00000000..1c4bbf69
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/generate_time_averages.py
@@ -0,0 +1,61 @@
+''' tools for generating time averages from various packages '''
+import click
+
+def generate_time_average(infile=None, outfile=None,
+ pkg=None, var=None,
+ unwgt=False, avg_type=None, stddev_type=None):
+ ''' steering function to various averaging functions above'''
+ if __debug__:
+ print(locals()) #input argument details
+ exitstatus=1
+
+ #needs a case statement. better yet, smarter way to do this? (TODO)
+ myavger=None
+ if pkg == 'cdo' :
+ from .cdoTimeAverager import cdoTimeAverager
+ myavger=cdoTimeAverager(pkg = pkg, var=var,
+ unwgt = unwgt,
+ avg_type = avg_type, stddev_type = stddev_type)
+
+ elif pkg == 'fre-nctools' :
+ from .frenctoolsTimeAverager import frenctoolsTimeAverager
+ myavger=frenctoolsTimeAverager(pkg = pkg, var=var,
+ unwgt = unwgt ,
+ avg_type = avg_type, stddev_type = stddev_type)
+
+ elif pkg == 'fre-python-tools':
+ from .frepytoolsTimeAverager import frepytoolsTimeAverager
+ myavger=frepytoolsTimeAverager(pkg = pkg, var=var,
+ unwgt = unwgt,
+ avg_type = avg_type, stddev_type = stddev_type)
+
+ else :
+ print('requested package unknown. exit.')
+ return exitstatus
+
+ if __debug__:
+ print(f'myavger.__repr__={myavger.__repr__}')
+ if myavger is not None:
+ exitstatus=myavger.generate_timavg(infile=infile, outfile=outfile)
+ else:
+ print('ERROR: averager is None, check generate_time_average in generate_time_averages.py!')
+
+ return exitstatus
+
+@click.command()
+def generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type):
+ ''' click entrypoint to time averaging routine '''
+ exitstatus=generate_time_average( inf, outf,
+ pkg, var,
+ unwgt,
+ avg_type, stddev_type)
+ if exitstatus!=0:
+ print(f'WARNING: exitstatus={exitstatus} != 0. Something exited poorly!')
+ else:
+ print('time averaging finished successfully')
+
+if __name__ == '__main__':
+ import time
+ start_time=time.perf_counter()
+ generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type)
+ print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
diff --git a/lib/fre/app/generate_time_averages/tests/.unused_tests/test_multiply_duration.py b/lib/fre/app/generate_time_averages/tests/.unused_tests/test_multiply_duration.py
new file mode 100644
index 00000000..704907b5
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/tests/.unused_tests/test_multiply_duration.py
@@ -0,0 +1,17 @@
+from fre_python_tools.utilities.multiply_duration import multiply_duration
+import metomi.isodatetime.parsers as parse
+
+def test_month():
+ '''1 month x 2 = 2 months'''
+ two_months = parse.DurationParser().parse('P2M')
+ assert multiply_duration('P1M', 2) == two_months
+
+def test_minutes():
+ '''12 minutes x 5 = 1 hour'''
+ hour = parse.DurationParser().parse('PT1H')
+ assert multiply_duration('PT12M', 5) == hour
+
+def test_fail():
+ '''10 minutes x 5 != 1 hour'''
+ hour = parse.DurationParser().parse('PT1H')
+ assert multiply_duration('PT10M', 5) != hour
diff --git a/lib/fre/app/generate_time_averages/tests/.unused_tests/test_subtract_durations.py b/lib/fre/app/generate_time_averages/tests/.unused_tests/test_subtract_durations.py
new file mode 100644
index 00000000..f495745d
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/tests/.unused_tests/test_subtract_durations.py
@@ -0,0 +1,17 @@
+from fre_python_tools.utilities.subtract_durations import subtract_durations
+import metomi.isodatetime.parsers as parse
+
+def test_months():
+ '''13 months - 3 months = 10 months'''
+ ten_months = parse.DurationParser().parse('P10M')
+ assert subtract_durations('P13M', 'P3M') == ten_months
+
+def test_hour():
+ '''2 hours minus 30 minutes = 90 minutes'''
+ ninety_mins = parse.DurationParser().parse('PT90M')
+ assert subtract_durations('PT2H', 'PT30M') == ninety_mins
+
+def test_fail():
+ '''2 hours minus 60 minutes != 90 minutes'''
+ ninety_mins = parse.DurationParser().parse('PT90M')
+ assert subtract_durations('PT2H', 'PT60M') != ninety_mins
diff --git a/lib/fre/app/generate_time_averages/tests/__init__.py b/lib/fre/app/generate_time_averages/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/app/generate_time_averages/tests/test_generate_time_averages.py b/lib/fre/app/generate_time_averages/tests/test_generate_time_averages.py
new file mode 100644
index 00000000..4ce3963d
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/tests/test_generate_time_averages.py
@@ -0,0 +1,308 @@
+''' for testing fre app generate-time-averages '''
+import pathlib as pl
+import pytest
+
+def run_avgtype_pkg_calculations(infile=None,outfile=None, pkg=None, avg_type=None, unwgt=None,
+ stddev_type=None):
+ ''' test-harness function, called by other test functions. '''
+ assert all( [infile is not None, outfile is not None,
+ pkg is not None, avg_type is not None,
+ unwgt is not None] )
+ if pl.Path(outfile).exists():
+ print('output test file exists. deleting before remaking.')
+ pl.Path(outfile).unlink() #delete file so we check that it can be recreated
+ from fre.app.generate_time_averages import generate_time_averages as gtas
+ gtas.generate_time_average(infile = infile, outfile = outfile,
+ pkg = pkg, unwgt = unwgt,
+ avg_type = avg_type, stddev_type = stddev_type)
+ return pl.Path(outfile).exists()
+
+### preamble tests. if these fail, none of the others will succeed. -----------------
+time_avg_file_dir=str(pl.Path.cwd())+'/fre/app/generate_time_averages/tests/time_avg_test_files/'
+test_file_name='atmos.197901-198312.LWP.nc'
+
+def test_time_avg_file_dir_exists():
+ ''' look for input test file directory '''
+ assert pl.Path(time_avg_file_dir).exists()
+
+def test_time_avg_input_file_exists():
+ ''' look for input test file '''
+ assert pl.Path( time_avg_file_dir + test_file_name ).exists()
+
+### cdo avgs, unweighted, all/seasonal/monthly ------------------------
+def test_monthly_cdo_time_unwgt_avgs():
+ ''' generates an unweighted monthly time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'ymonmean_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='month',unwgt=True )
+
+def test_seasonal_cdo_time_unwgt_avgs():
+ ''' generates an unweighted seasonal time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='seas',unwgt=True )
+
+def test_cdo_time_unwgt_avgs():
+ ''' generates an unweighted time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'timmean_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='all',unwgt=True )
+
+
+#### cdo avgs, weighted, all/seasonal/monthly ------------------------
+## (TODO) WRITE THESE VERSIONS FOR CDOTIMEAVERAGER CLASS THEN MAKE THESE TESTS
+##def test_monthly_cdo_time_avgs():
+##def test_seasonal_cdo_time_avgs():
+
+def test_cdo_time_avgs():
+ ''' generates a weighted time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'timmean_'+test_file_name),
+ pkg='cdo',avg_type='all',unwgt=False )
+
+
+### cdo stddevs, unweighted, all/seasonal/monthly ------------------------
+def test_monthly_cdo_time_unwgt_stddevs():
+ ''' generates a monthly time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'ymonstddev1_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='month',stddev_type='samp', unwgt=True )
+
+def test_seasonal_cdo_time_unwgt_stddevs():
+ ''' generates a seasonal time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'yseasstddev1_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='seas',stddev_type='samp',unwgt=True )
+
+def test_cdo_time_unwgt_stddevs():
+ ''' generates a time averaged file using cdo '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name),
+ pkg='cdo',avg_type='all',stddev_type='samp', unwgt=True )
+
+
+#### cdo stddevs, weighted, all/seasonal/monthly -----------------------
+## (TODO) WRITE THESE VERSIONS FOR CDOTIMEAVERAGER CLASS THEN MAKE THESE TESTS
+#def test_monthly_cdo_time_stddevs():
+#def test_seasonal_cdo_time_stddevs():
+#def test_cdo_time_stddevs():
+
+## frepythontools avgs+stddevs, weighted+unweighted, all ------------------------
+def test_fre_python_tools_time_avgs():
+ ''' generates a time averaged file using fre_python_tools's version '''
+ ''' weighted average, no std deviation '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'frepytools_timavg_'+test_file_name),
+ pkg='fre-python-tools',avg_type='all', unwgt=False )
+
+def test_fre_python_tools_time_unwgt_avgs():
+ ''' generates a time averaged file using fre_python_tools's version '''
+ ''' weighted average, no std deviation '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'frepytools_unwgt_timavg_'+test_file_name),
+ pkg='fre-python-tools',avg_type='all', unwgt=True )
+
+def test_fre_python_tools_time_avgs_stddevs():
+ ''' generates a time averaged file using fre_python_tools's version '''
+ ''' weighted average, no std deviation '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'frepytools_stddev_'+test_file_name),
+ pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=False )
+
+def test_fre_python_tools_time_unwgt_avgs_stddevs():
+ ''' generates a time averaged file using fre_python_tools's version '''
+ ''' weighted average, no std deviation '''
+ assert run_avgtype_pkg_calculations(
+ infile = (time_avg_file_dir+test_file_name),
+ outfile = (time_avg_file_dir+'frepytools_unwgt_stddev_'+test_file_name),
+ pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=True )
+
+## (TODO) WRITE THESE VERSIONS FOR FREPYTOOLSTIMEAVERAGER CLASS THEN MAKE THESE TESTS
+#def test_monthly_fre_python_tools_time_avgs():
+#def test_monthly_fre_python_tools_time_unwgt_avgs():
+#def test_monthly_fre_python_tools_time_avgs_stddevs():
+#def test_monthly_fre_python_tools_time_unwgt_avgs_stddevs():
+#
+#def test_seasonal_fre_python_tools_time_avgs():
+#def test_seasonal_fre_python_tools_time_unwgt_avgs():
+#def test_seasonal_fre_python_tools_time_avgs_stddevs():
+#def test_seasonal_fre_python_tools_time_unwgt_avgs_stddevs(:)
+
+
+
+## this will only work at GFDL. dev convenience only.
+#alt_str_fre_nctools_inf= \
+# 'tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc'
+#def test_fre_nctools_time_avgs():
+# ''' generates a time averaged file using fre_python_tools's version '''
+# ''' weighted average, no std deviation '''
+# infile =time_avg_file_dir+test_file_name
+# all_outfile=time_avg_file_dir+'frenctools_timavg_'+test_file_name
+#
+# if pl.Path(all_outfile).exists():
+# print('output test file exists. deleting before remaking.')
+# pl.Path(all_outfile).unlink() #delete file so we check that it can be recreated
+#
+# from fre_python_tools.generate_time_averages import generate_time_averages as gtas
+# gtas.generate_time_average(infile = infile, outfile = all_outfile,
+# pkg='fre-nctools', unwgt=False, avg_type='all')
+# assert pl.Path(all_outfile).exists()
+
+
+# Numerics-based tests. these have room for improvemnent for sure (TODO)
+# compare frepytools, cdo time-average output to fre-nctools where possible
+var='LWP'
+str_fre_nctools_inf=time_avg_file_dir+'frenctools_timavg_'+test_file_name # this is now in the repo
+str_fre_pytools_inf=time_avg_file_dir+'frepytools_timavg_'+test_file_name
+str_cdo_inf=time_avg_file_dir+'timmean_'+test_file_name
+str_unwgt_fre_pytools_inf=time_avg_file_dir+'frepytools_unwgt_timavg_'+test_file_name
+str_unwgt_cdo_inf=time_avg_file_dir+'timmean_unwgt_'+test_file_name
+
+
+def test_compare_fre_python_tools_to_fre_nctools():
+ ''' compares fre_python_tools pkg answer to fre_nctools pkg answer '''
+ import numpy as np
+ import netCDF4 as nc
+ fre_pytools_inf=nc.Dataset(str_fre_pytools_inf,'r')
+
+ try:
+ fre_nctools_inf=nc.Dataset(str_fre_nctools_inf,'r')
+ except:
+ print('fre-nctools input file not found. \
+ probably because you are not at GFDL! run the shell script \
+ example if you would like to see this pass. otherwise, \
+ i will error right after this message.')
+ try:
+ fre_nctools_inf=nc.Dataset(alt_str_fre_nctools_inf,'r')
+ except:
+ print('fre-nctools output does not exist. create it first!')
+ assert False
+
+ fre_pytools_timavg=fre_pytools_inf[var][:].copy()
+ fre_nctools_timavg=fre_nctools_inf[var][:].copy()
+ assert all([ len(fre_pytools_timavg)==len(fre_nctools_timavg),
+ len(fre_pytools_timavg[0])==len(fre_nctools_timavg[0]),
+ len(fre_pytools_timavg[0][0])==len(fre_nctools_timavg[0][0]) ])
+
+ diff_pytools_nctools_timavg=fre_pytools_timavg-fre_nctools_timavg
+ for lat in range(0,len(diff_pytools_nctools_timavg[0])):
+ for lon in range(0,len(diff_pytools_nctools_timavg[0][0])):
+ print(f'lat={lat},lon={lon}')
+ #diff_at_latlon=diff_pytools_nctools_timavg[0][lat][lon]
+ #print(f'diff_pytools_nctools_timavg[0][lat][lon]={diff_at_latlon}')
+ if lon>10: break
+ break
+
+ non_zero_count=np.count_nonzero(diff_pytools_nctools_timavg[:])
+ #assert (non_zero_count == 0.) # bad way to check for zero.
+ assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+
+@pytest.mark.skip(reason='test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer')
+def test_compare_fre_python_tools_to_cdo():
+ ''' compares fre_python_tools pkg answer to cdo pkg answer '''
+ import numpy as np
+ import netCDF4 as nc
+ fre_pytools_inf=nc.Dataset(str_fre_pytools_inf,'r')
+
+ try:
+ cdo_inf=nc.Dataset(str_cdo_inf,'r')
+ except:
+ print('cdo input file not found. run cdo tests first.')
+ assert False
+
+ fre_pytools_timavg=fre_pytools_inf[var][:].copy()
+ cdo_timavg=cdo_inf[var][:].copy()
+
+ assert all([ len(fre_pytools_timavg)==len(cdo_timavg),
+ len(fre_pytools_timavg[0])==len(cdo_timavg[0]),
+ len(fre_pytools_timavg[0][0])==len(cdo_timavg[0][0]) ])
+
+ diff_pytools_cdo_timavg=fre_pytools_timavg-cdo_timavg
+ for lat in range(0,len(diff_pytools_cdo_timavg[0])):
+ for lon in range(0,len(diff_pytools_cdo_timavg[0][0])):
+ print(f'lat={lat},lon={lon}')
+ print(f'diff_pytools_cdo_timavg[0][lat][lon]={diff_pytools_cdo_timavg[0][lat][lon]}')
+ if lon>10: break
+ break
+
+ non_zero_count=np.count_nonzero(diff_pytools_cdo_timavg[:])
+ assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+
+
+def test_compare_unwgt_fre_python_tools_to_unwgt_cdo():
+ ''' compares fre_python_tools pkg answer to cdo pkg answer '''
+ import numpy as np
+ import netCDF4 as nc
+ fre_pytools_inf=nc.Dataset(str_unwgt_fre_pytools_inf,'r')
+
+ try:
+ cdo_inf=nc.Dataset(str_unwgt_cdo_inf,'r')
+ except:
+ print('cdo input file not found. run cdo tests first.')
+ assert False
+
+ fre_pytools_timavg=fre_pytools_inf[var][:].copy()
+ cdo_timavg=cdo_inf[var][:].copy()
+ assert all([ len(fre_pytools_timavg)==len(cdo_timavg),
+ len(fre_pytools_timavg[0])==len(cdo_timavg[0]),
+ len(fre_pytools_timavg[0][0])==len(cdo_timavg[0][0]) ])
+
+ diff_pytools_cdo_timavg=fre_pytools_timavg-cdo_timavg
+ for lat in range(0,len(diff_pytools_cdo_timavg[0])):
+ for lon in range(0,len(diff_pytools_cdo_timavg[0][0])):
+ print(f'lat={lat},lon={lon}')
+ print(f'diff_pytools_cdo_timavg[0][lat][lon]={diff_pytools_cdo_timavg[0][lat][lon]}')
+ if lon>10: break
+ break
+
+ non_zero_count=np.count_nonzero(diff_pytools_cdo_timavg[:])
+ assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+
+@pytest.mark.skip(reason='test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer')
+def test_compare_cdo_to_fre_nctools():
+ ''' compares cdo pkg answer to fre_nctools pkg answer '''
+ import numpy as np
+ import netCDF4 as nc
+ cdo_inf=nc.Dataset(str_cdo_inf,'r')
+
+ try:
+ fre_nctools_inf=nc.Dataset(str_fre_nctools_inf,'r')
+ except:
+ print('fre-nctools input file not found. \
+ probably because you are not at GFDL! run the shell script \
+ example if you would like to see this pass. otherwise, \
+ i will error right after this message.')
+ alt_str_fre_nctools_inf = \
+ 'tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc'
+ try:
+ fre_nctools_inf=nc.Dataset(alt_str_fre_nctools_inf,'r')
+ except:
+ print('fre-nctools output does not exist. create it first!')
+ assert False
+
+ cdo_timavg=cdo_inf[var][:].copy()
+ fre_nctools_timavg=fre_nctools_inf[var][:].copy()
+ assert all([ len(cdo_timavg)==len(fre_nctools_timavg),
+ len(cdo_timavg[0])==len(fre_nctools_timavg[0]),
+ len(cdo_timavg[0][0])==len(fre_nctools_timavg[0][0]) ])
+
+ diff_cdo_nctools_timavg=cdo_timavg-fre_nctools_timavg
+ for lat in range(0,len(diff_cdo_nctools_timavg[0])):
+ for lon in range(0,len(diff_cdo_nctools_timavg[0][0])):
+ print(f'lat={lat},lon={lon}')
+ print(f'diff_cdo_nctools_timavg[0][lat][lon]={diff_cdo_nctools_timavg[0][lat][lon]}')
+ if lon>10: break
+ break
+
+ non_zero_count=np.count_nonzero(diff_cdo_nctools_timavg[:])
+ assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
diff --git a/lib/fre/app/generate_time_averages/tests/time_avg_test_files/README.md b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/README.md
new file mode 100644
index 00000000..e25c76f4
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/README.md
@@ -0,0 +1 @@
+this directory is for tests/examples/sample files, etc.
diff --git a/lib/fre/app/generate_time_averages/tests/time_avg_test_files/atmos.197901-198312.LWP.nc b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/atmos.197901-198312.LWP.nc
new file mode 100644
index 00000000..e2aa57b2
Binary files /dev/null and b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/atmos.197901-198312.LWP.nc differ
diff --git a/lib/fre/app/generate_time_averages/tests/time_avg_test_files/frenctools_timavg_atmos.197901-198312.LWP.nc b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/frenctools_timavg_atmos.197901-198312.LWP.nc
new file mode 100644
index 00000000..049d668e
Binary files /dev/null and b/lib/fre/app/generate_time_averages/tests/time_avg_test_files/frenctools_timavg_atmos.197901-198312.LWP.nc differ
diff --git a/lib/fre/app/generate_time_averages/timeAverager.py b/lib/fre/app/generate_time_averages/timeAverager.py
new file mode 100644
index 00000000..803febfe
--- /dev/null
+++ b/lib/fre/app/generate_time_averages/timeAverager.py
@@ -0,0 +1,68 @@
+''' core class structure for this module.'''
+
+class timeAverager:
+ '''
+ abstract base class for generating time averages + related statistical quantities
+ this class must be inherited by another for functionality.
+ '''
+ pkg: str
+ var: str
+ unwgt: bool
+ avg_type: str
+ stddev_type: str
+
+ def __init__(self):
+ ''' init method 1, no inputs given '''
+ self.pkg = None
+ self.var = None
+ self.unwgt = False
+ self.avg_type = "all" #see argparser for options
+ self.stddev_type = None #see argparser for options
+
+ def __init__(self, pkg, var, unwgt,
+ avg_type, stddev_type):
+ ''' init method 2, all inputs specified '''
+ self.pkg = pkg
+ self.var = var
+ self.unwgt = unwgt
+ self.avg_type = avg_type
+ self.stddev_type = stddev_type
+
+ def __repr__(self):
+ ''' return text representation of object '''
+ return f'{type(self).__name__}( pkg={self.pkg}, \
+ unwgt={self.unwgt}, \
+ var={self.var}, \
+ avg_type={self.avg_type}, \
+ stddev_type={self.stddev_type})'
+
+ def var_has_time_units(self, an_nc_var=None):
+ ''' checks if variable's units are of time '''
+ try:
+ var_units=an_nc_var.units
+ units_is_time = False
+ units_is_time = any( [ var_units == 'seconds' , 'seconds since' in var_units ,
+ var_units == 'minutes' , 'minutes since' in var_units ,
+ var_units == 'hours' , 'hours since' in var_units ,
+ var_units == 'days' , 'days since' in var_units ,
+ var_units == 'months' , 'months since' in var_units ,
+ var_units == 'years' , 'years since' in var_units ] )
+ return units_is_time
+ except:
+ print('variable does not have units')
+ print('PROBABLY not time.')
+ return False
+
+ #def var_has_time_dims(self, an_nc_var=None):
+ #try:
+ # var_dims=an_nc_var.dimensions
+ #for dim in an_nc_var.dimensions:
+ # if dim ==
+
+
+
+
+ def generate_timavg(self, infile=None, outfile=None):
+ '''# this is a hint: this is to be defined by classes inheriting from the abstract one
+ this function is never to be fully defined here by design.'''
+ raise NotImplementedError()
diff --git a/lib/fre/app/mask_atmos_plevel.py b/lib/fre/app/mask_atmos_plevel.py
new file mode 100644
index 00000000..0e7b2c28
--- /dev/null
+++ b/lib/fre/app/mask_atmos_plevel.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+'''
+This script contains the refineDiags that produce data at the same
+frequency as the input data (no reduction) such as surface albedo,
+masking fields,...
+It can accept any file and will only compute refineDiags in fields
+are present.
+'''
+
+import os
+import netCDF4 as nc
+import xarray as xr
+import click
+
+@click.command()
+def mask_atmos_plevel_subtool(infile, outfile, psfile):
+ ''' click entry point to fre cmor mask-atmos-plevel'''
+
+ # Error if outfile exists
+ if os.path.exists(outfile):
+ raise FileExistsError(f"ERROR: Output file {outfile} already exists")
+
+ # Open ps dataset
+ if not os.path.exists(psfile):
+ raise FileNotFoundError(f"ERROR: Input surface pressure file {psfile} does not exist")
+ ds_ps = xr.open_dataset(psfile)
+
+ # Exit with message if "ps" not available
+ if "ps" not in list(ds_ps.variables):
+ raise ValueError(f"ERROR: File {infile} does not contain surface pressure. exit.")
+
+ # Open input dataset
+ if not os.path.exists(infile):
+ raise FileNotFoundError(f"ERROR: Input file {infile} does not exist")
+ ds_in = xr.open_dataset(infile)
+
+ # The trigger for atmos masking is a variable attribute "needs_atmos_masking = True".
+ # In the future this will be set within the model, but for now and testing,
+ # we'll add the attribute for variables that end with "_unmsk".
+ ds_in = preprocess(ds_in)
+
+ ds_out = xr.Dataset()
+
+ # Process all variables with attribute "needs_atmos_masking = True"
+ for var in list(ds_in.variables):
+ if 'needs_atmos_masking' in ds_in[var].attrs:
+ del ds_in[var].attrs['needs_atmos_masking']
+ ds_out[var] = mask_field_above_surface_pressure(ds_in, var, ds_ps)
+ else:
+ continue
+
+ # Write the output file if anything was done
+ if ds_out.variables:
+ print(f"Modifying variables: {list(ds_out.variables)}\n appending into new file {outfile}")
+ write_dataset(ds_out, ds_in, outfile)
+ else:
+ print(f"No variables modified, so not writing output file {outfile}")
+
+
+
+def preprocess(ds):
+ """add needs_atmos_masking attribute if var ends with _unmsk"""
+ for var in list(ds.variables):
+ if var.endswith('_unmsk'):
+ ds[var].attrs['needs_atmos_masking'] = True
+ return ds
+
+
+def mask_field_above_surface_pressure(ds, var, ds_ps):
+ """mask data with pressure larger than surface pressure"""
+ plev = pressure_coordinate(ds, var)
+
+ # broadcast pressure coordinate and surface pressure to
+ # the dimensions of the variable to mask
+ plev_extended, _ = xr.broadcast(plev, ds[var])
+ ps_extended, _ = xr.broadcast(ds_ps["ps"], ds[var])
+
+ # masking do not need looping
+ masked = xr.where(plev_extended > ps_extended, 1.0e20, ds[var])
+
+ # copy attributes, but it doesn't include the missing values
+ attrs = ds[var].attrs.copy()
+
+ # add the missing values back
+ attrs['missing_value'] = 1.0e20
+ attrs['_FillValue'] = 1.0e20
+ masked.attrs = attrs
+
+ # transpose dims like the original array
+ masked = masked.transpose(*ds[var].dims)
+ print(f"Processed {var}")
+
+ return masked
+
+
+def pressure_coordinate(ds, varname):#, verbose=False):
+ """check if dataArray has pressure coordinate fitting requirements
+ and return it"""
+
+ pressure_coord = None
+ for dim in list(ds[varname].dims):
+ if dim in list(ds.variables): # dim needs to have values in file
+ if ds[dim].attrs["long_name"] == "pressure":
+ pressure_coord = ds[dim]
+ elif ("coordinates" in ds.attrs) and (ds[dim].attrs["units"] == "Pa"):
+ pressure_coord = ds[dim]
+ return pressure_coord
+
+
+def write_dataset(ds, template, outfile):
+ """prepare the dataset and dump into netcdf file"""
+
+ # copy global attributes
+ ds.attrs = template.attrs.copy()
+
+ # copy all variables and their attributes
+ # except those already processed
+ for var in list(template.variables):
+ if var in list(ds.variables):
+ continue
+ ds[var] = template[var]
+ ds[var].attrs = template[var].attrs.copy()
+ # write to file
+ ds.to_netcdf(outfile, unlimited_dims="time")
+
+
+
+def set_netcdf_encoding(ds, pressure_vars):
+ """set preferred options for netcdf encoding"""
+ all_vars = list(ds.variables)
+ encoding = {}
+ #for var in do_not_encode_vars + pressure_vars: #what was here in first place
+ for var in pressure_vars: #remove unused variable
+ if var in all_vars:
+ encoding.update( { var :
+ { '_FillValue':None} } )
+ return encoding
+
+
+def post_write(filename, var_with_bounds, bounds_variables):
+ """fix a posteriori attributes that xarray.to_netcdf
+ did not do properly using low level netcdf lib"""
+ f = nc.Dataset(filename, "a")
+ for var, bndvar in zip(var_with_bounds, bounds_variables):
+ f.variables[var].setncattr("bounds", bndvar)
+ f.close()
diff --git a/lib/fre/app/regrid_xy/README.md b/lib/fre/app/regrid_xy/README.md
new file mode 100644
index 00000000..30bf9b22
--- /dev/null
+++ b/lib/fre/app/regrid_xy/README.md
@@ -0,0 +1,55 @@
+# ABOUT
+
+`regrid_xy.py` remaps scalar and/or vector fields from one kind of lat/lon grid to another. It can remap between different grids of the same type (e.g. spherical), and between grids of different types (e.g. spherical to tripolar). By default, it uses an O(1) conservative interpolation scheme to accomplish the regridding, except under certain conditions [defined within `fregrid`](https://github.com/NOAA-GFDL/FRE-NCtools/blob/master/tools/fregrid/fregrid.c#L915-L920) the underlying CLI tool which does the heavy lifting.
+
+requires `fre-nctools` and `fregrid` to be in one's `PATH` variable, and `python3` (tested/developed with python 3.9.16). there should be `netCDF4` and `metomi` python modules in one's python environment for imports. `pytest` and `nccmp` is required for tests. `pylint` recommended for future developers working on this tool.
+
+
+# INPUT PARAMETERS (mandatory, env vars)
+format here is:
+config field name / python variable name (type) explanation
+
+the following are required to be specified:
+_______________________________________
+`inputDir` / `input_dir` (env var) specifies input directory to regrid, typically an untarredv history file archive
+_______________________________________
+`source` / `source` (env var) source name for input target file name within input directory to target for regridding. the value for `source` must be present in at least one component's configuration fields
+_______________________________________
+`begin` / `begin` (env var) ISO8601 datetime format specification for starting date of data, part of input target file name
+_______________________________________
+`outputDir` / `output_dir` (env var) specifies target location for output regridded files
+_______________________________________
+`TMPDIR` / `tmp_dir` (env var) temp directory for location of file read/writes
+_______________________________________
+`fregridRemapDir` / `remap_dir` (env var) directory containing remap file for regridding
+_______________________________________
+`gridSpec` / `grid_spec` (env var) file containing mosaic for regridding
+_______________________________________
+`defaultxyInterp` / `def_xy_interp` (env var) default lat/lon resolution for output regridding. (change me? TODO)
+
+
+# INPUT PARAMETERS (configuration fields, mandatory)
+the following parameters are REQUIRED to be specified on a per-component basis wtihin `app/regrid-xy/rose-app.conf`. A component's input parameters are delineated with a `[component_name]`
+_______________________________________
+`inputRealm` / `input_realm` (config field) realm within model from which the input component/source files are derived
+_______________________________________
+`interpMethod` / `interp_method` (config field) interpolation method to use for regridding, it may be changed if it is specified in the target source file's attributes
+_______________________________________
+`inputGrid` / `input_grid` (config field) current grid type of input source files.
+_______________________________________
+`sources` / N/A (config field)
+
+# INPUT PARAMETERS (configuration fields, optional)
+the following parameters are OPTIONAL for specifying on a per-component basis within `app/regrid-xy/rose-app.conf`.
+_______________________________________
+`outputGridType` / `output_grid_type` (config field) used only for output dir to specify grid type, but does not determine actual output grid type for regridding to fregrid
+_______________________________________
+`fregridRemapFile` / `fregrid_remap_file` (config field) remap file name to use for regridding
+_______________________________________
+`fregridMoreOptions` / `more_options` (config field) field for specifying additional options to `fregrid` that have not-yet been officially implemented. use with caution!
+_______________________________________
+`variables` / `regrid_vars` (config field) list of variable data to regrid within target source files. if unspecified, all variables within the target file of dimension 2 or greater will be regridded.
+_______________________________________
+`outputGridLat` / `output_grid_lat` (config field) latitude resolution for regridded output, also used for remap file targeting if there is no remap file specified.
+_______________________________________
+`outputGridLon` / `output_grid_lon` (config field) latitude resolution for regridded output, also used for remap file targeting if there is no remap file specified.
diff --git a/lib/fre/app/regrid_xy/__init__.py b/lib/fre/app/regrid_xy/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/app/regrid_xy/regrid_xy.py b/lib/fre/app/regrid_xy/regrid_xy.py
new file mode 100644
index 00000000..8721b36a
--- /dev/null
+++ b/lib/fre/app/regrid_xy/regrid_xy.py
@@ -0,0 +1,486 @@
+"""
+ remaps scalar and/or vector fields. It is capable of remapping
+ from a spherical grid onto a different one, e.g. spherical,
+ or tripolar. By default, it does so using a conservative scheme.
+ most valid input args to fregrid are valid in this script
+"""
+
+import subprocess
+import shutil
+import os
+from pathlib import Path
+
+#3rd party
+import metomi.rose.config as rose_cfg
+from netCDF4 import Dataset
+import click
+
+FREGRID_SHARED_FILES='/home/fms/shared_fregrid_remap_files'
+
+def truncate_date(date, freq):
+ """ truncates iso freq to iso date time """
+ freq_in_date_format=freq_to_date_format(freq)
+
+ #in the shell version, this line simply gets run.
+ #we will simply print this command to screen for now. TO DO (maybe we can work around it?)
+ #not clear to me why piping to tr is necessary, doesnt seem to change output at all
+ #print('cylc date --template '+freq_in_date_format+' '+date+' | tr -d T')
+ #output =subprocess.Popen(["cylc", "date", "--template", freq_in_date_format, date,
+ # "|","tr","-d","T"],
+ # stdout=subprocess.PIPE)
+ output =subprocess.Popen(["cylc", "cycle-point", "--template", freq_in_date_format, date],
+ stdout=subprocess.PIPE)
+ bytedate = output.communicate()[0]
+ date=str(bytedate.decode())
+
+ #remove trailing newline
+ date=date[:(len(date)-1)]
+
+ #check for and remove 'T' if present
+ if not date.isnumeric():
+ date=date[:8]+date[-2:]
+ return date
+
+def freq_to_date_format(iso_freq):
+ """
+ Print legacy Bronx-like date template format given a frequency (ISO 8601 duration)
+ """
+ if iso_freq=='P1Y':
+ return 'CCYY'
+ if iso_freq=='P1M':
+ return 'CCYYMM'
+ if iso_freq=='P1D':
+ return 'CCYYMMDD'
+ if (iso_freq[:2]=='PT') and (iso_freq[-1:]=='H'):
+ return 'CCYYMMDDThh'
+ raise ValueError(f'ERROR: Unknown Frequency {iso_freq}')
+
+def test_import():
+ """for quickly testing import within pytest"""
+ return 1
+
+def safe_rose_config_get(config, section, field):
+ """read optional variables from rose configuration, and don't error on None value"""
+ config_dict = config.get( [section,field] )
+ return None if config_dict is None else config_dict.get_value()
+
+
+def get_mosaic_file_name(grid_spec_file, mosaic_type):
+ """read string from a numpy masked array WHY"""
+ grid_spec_nc = Dataset(grid_spec_file, 'r')
+ masked_data = grid_spec_nc[mosaic_type][:].copy() # maskedArray
+ grid_spec_nc.close()
+ unmasked_data = masked_data[~masked_data.mask]
+ file_name = ''.join([ one_char.decode() for one_char in unmasked_data ])
+ return file_name
+
+
+def get_mosaic_grid_file_name(input_mosaic):
+ """get mosaic grid file name from NESTED numpy masked array WHY"""
+ input_mosaic_nc = Dataset(input_mosaic,'r')
+ masked_data = input_mosaic_nc['gridfiles'][0].copy()
+ input_mosaic_nc.close()
+ unmasked_data = masked_data[~masked_data.mask]
+ grid_file_name = ''.join([ one_char.decode() for one_char in unmasked_data ])
+ return grid_file_name
+
+
+def check_interp_method( nc_variable, interp_method):
+ """print warning if optional interp_method clashes with nc file attribute field, if present"""
+ attr_list=nc_variable.ncattrs()
+ if 'interp_method' not in attr_list:
+ pass
+ elif nc_variable.interp_method != interp_method:
+ print(f'WARNING: interp_method={interp_method} differs from what is in target_file')
+ print(f'WARNING: interp_method used may not be {interp_method}!')
+
+
+def check_per_component_settings(component_list, rose_app_cfg):
+ """for a source file ref'd by multiple components check per-component
+ settings for uniqueness. output list of bools of same length to check
+ in componenet loop"""
+ do_regridding = [True] #first component will always be run
+ curr_out_grid_type_list = [safe_rose_config_get( \
+ rose_app_cfg, component_list[0], 'outputGridType')]
+ for i in range( 1, len(component_list) ):
+ next_comp=component_list[i]
+ next_out_grid_type=safe_rose_config_get( rose_app_cfg, next_comp, 'outputGridType')
+ if next_out_grid_type not in curr_out_grid_type_list:
+ do_regridding.append(True)
+ curr_out_grid_type_list.append(next_out_grid_type)
+ else:
+ do_regridding.append(False)
+ if len(do_regridding) != len(component_list) :
+ raise ValueError('problem with checking per-component settings for uniqueness')
+ return do_regridding
+
+
+
+
+def make_component_list(config, source):
+ """make list of relevant component names where source file appears in sources"""
+ comp_list=[] #will not contain env, or command
+ try:
+ for keys, sub_node in config.walk():
+ sources = sub_node.get_value(keys=['sources'])#.split(' ')
+ if any( [ len(keys) < 1,
+ keys[0] in ['env', 'command'],
+ keys[0] in comp_list,
+ sources is None ] ):
+ continue
+ sources = sources.split(' ')
+ if source in sources:
+ comp_list.append(keys[0])
+ except Exception as exc:
+ raise ValueError(f'config = {config} may be an empty file... check the config') \
+ from exc
+ return comp_list
+
+
+def make_regrid_var_list(target_file, interp_method = None):
+ """create default list of variables to be regridded within target file."""
+ fin = Dataset(target_file,'r')
+ all_fin_vars = fin.variables
+ regrid_vars = []
+ for var_name in all_fin_vars:
+ if var_name in ['average_T1','average_T2',
+ 'average_DT','time_bnds' ]:
+ continue
+ if len(all_fin_vars[var_name].shape) < 2 :
+ continue
+ regrid_vars.append(var_name)
+ if interp_method is not None:
+ check_interp_method( all_fin_vars[var_name] , interp_method)
+
+ fin.close()
+ return regrid_vars
+
+
+def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None,
+ remap_dir = None, source = None, grid_spec = None, def_xy_interp = None
+ ):
+ """
+ calls fre-nctools' fregrid to regrid net cdf files
+ """
+
+ ## rose config load check
+ config_name = os.getcwd()
+ config_name += '/rose-app-run.conf'
+ #config_name += '/rose-app.conf'
+ print(f'config_name = {config_name}')
+ try:
+ rose_app_config = rose_cfg.load(config_name)
+ except Exception as exc:
+ raise ValueError(f'config_name = {config_name} not found.') \
+ from exc
+
+
+ # mandatory arguments- code exits if any of these are not present
+ #input_dir = os.getenv( 'inputDir' )
+ #output_dir = os.getenv( 'outputDir' )
+ #begin = os.getenv( 'begin' )
+ #tmp_dir = os.getenv( 'TMPDIR' )
+ #remap_dir = os.getenv( 'fregridRemapDir' )
+ #source = os.getenv( 'source' )
+ #grid_spec = os.getenv( 'gridSpec' )
+ #def_xy_interp = os.getenv( 'defaultxyInterp' )
+ if None in [ input_dir , output_dir ,
+ begin , tmp_dir ,
+ remap_dir , source ,
+ grid_spec , def_xy_interp ]:
+ print( f'input_dir = { input_dir }\n' + \
+ f'output_dir = { output_dir }\n' + \
+ f'begin = { begin }\n' + \
+ f'tmp_dir = { tmp_dir }\n' + \
+ f'remap_dir = { remap_dir }\n' + \
+ f'source = { source }\n' + \
+ f'grid_spec = { grid_spec }\n' + \
+ f'def_xy_interp = { def_xy_interp }' )
+ raise ValueError(f'a mandatory input argument to regrid_xy is None... \n {locals()}')
+
+
+ def_xy_interp = def_xy_interp.split(',')
+ def_xy_interp[0] = def_xy_interp[0].replace('"', '')
+ def_xy_interp[1] = def_xy_interp[1].replace('"', '')
+ if any( [ def_xy_interp == [] or len(def_xy_interp) != 2 ] ):
+ raise ValueError(
+ f'default xy interpolation has invalid format: \n def_xy_interp = {def_xy_interp}')
+
+ # input dir must exist
+ if not Path( input_dir ).exists():
+ raise OSError(f'input_dir={input_dir} \n does not exist')
+
+ # tmp_dir check
+ if not Path( tmp_dir ).exists():
+ raise OSError(f'tmp_dir={tmp_dir} \n does not exist.')
+
+ # output dir check
+ Path( output_dir ).mkdir( parents = True, exist_ok = True )
+ if not Path( output_dir ).exists() :
+ raise OSError('the following does not exist and/or could not be created:' +
+ f'output_dir=\n{output_dir}')
+
+ # work/ dir check
+ work_dir = tmp_dir + 'work/'
+ Path( work_dir ).mkdir( exist_ok = True )
+ if not Path( work_dir ).exists():
+ raise OSError('the following does not exist and/or could not be created:' +
+ f'work_dir=\n{work_dir}')
+
+
+ # fregrid remap dir check
+ Path(remap_dir).mkdir( exist_ok = True )
+ if not Path( remap_dir ).exists():
+ raise OSError(f'{remap_dir} could not be created')
+
+
+ # grid_spec file management
+ #starting_dir = os.getcwd()
+ #os.chdir(work_dir) # i hate it
+ if '.tar' in grid_spec:
+ untar_sp = \
+ subprocess.run( ['tar', '-xvf', grid_spec, '-C', input_dir],
+ check = False , capture_output = True )
+ if untar_sp.returncode != 0:
+ raise OSError(
+ f'untarring of {grid_spec} file failed, ' + \
+ f'ret_code={untar_sp.returncode}, stderr={untar_sp.stderr}')
+ if Path( input_dir+'mosaic.nc' ).exists():
+ grid_spec_file=input_dir+'mosaic.nc'
+ elif Path( input_dir+'grid_spec.nc' ).exists():
+ grid_spec_file=input_dir+'grid_spec.nc'
+ else:
+ raise ValueError(f'grid_spec_file cannot be determined from grid_spec={grid_spec}')
+ else:
+ try: #attempt to copy directly from uncompressed grid_spec location
+ grid_spec_file=input_dir+grid_spec.split('/').pop()
+ shutil.copy(grid_spec, grid_spec_file )
+ except Exception as exc:
+ raise OSError(f'grid_spec={grid_spec} could not be copied.') \
+ from exc
+
+ # ------------------------------------------------------------------ component loop
+ component_list = make_component_list(rose_app_config, source)
+ if len(component_list) == 0:
+ raise ValueError('component list empty- source file not found in any source file list!')
+ if len(component_list) > 1: # check settings for uniqueness
+ do_regridding = \
+ check_per_component_settings( \
+ component_list, rose_app_config)
+ print(f'component_list = {component_list}')
+ print(f'do_regridding = {do_regridding}')
+ else:
+ do_regridding=[True]
+
+ for component in component_list:
+ if not do_regridding[
+ component_list.index(component) ]:
+ continue
+ print(f'\n\nregridding \nsource={source} for \ncomponent={component}\n')
+
+ # mandatory per-component inputs, will error if nothing in rose config
+ input_realm, interp_method, input_grid = None, None, None
+ try:
+ input_realm = rose_app_config.get( [component, 'inputRealm'] ).get_value()
+ interp_method = rose_app_config.get( [component, 'interpMethod'] ).get_value()
+ input_grid = rose_app_config.get( [component, 'inputGrid'] ).get_value()
+ except Exception as exc:
+ raise ValueError('at least one of the following are None: \n' + \
+ f'input_grid=\n{input_grid}\n,input_realm=\n' + \
+ f'{input_realm}\n,/interp_method=\n{interp_method}') \
+ from exc
+ print(f'input_grid = {input_grid }\n' + \
+ f'input_realm = {input_realm }\n' + \
+ f'interp_method = {interp_method }' )
+
+ #target input variable resolution
+ is_tiled = 'cubedsphere' in input_grid
+ target_file = input_dir
+ target_file += f"/{truncate_date(begin,'P1D')}.{source}.tile1.nc" \
+ if is_tiled \
+ else f"/{truncate_date(begin,'P1D')}.{source}.nc"
+ if not Path( target_file ).exists():
+ raise OSError(f'regrid_xy target does not exist. \ntarget_file={target_file}')
+ print(f'target_file={target_file}') #DELETE
+
+
+ # optional per-component inputs
+ output_grid_type = safe_rose_config_get( rose_app_config, component, 'outputGridType')
+ remap_file = safe_rose_config_get( rose_app_config, component, 'fregridRemapFile')
+ more_options = safe_rose_config_get( rose_app_config, component, 'fregridMoreOptions')
+ regrid_vars = safe_rose_config_get( rose_app_config, component, 'variables')
+ output_grid_lon = safe_rose_config_get( rose_app_config, component, 'outputGridLon')
+ output_grid_lat = safe_rose_config_get( rose_app_config, component, 'outputGridLat')
+
+ # if no input args specified, grab the defaul values
+ if any( [ output_grid_lon is None,
+ output_grid_lat is None ] ):
+ output_grid_lon = def_xy_interp[0]
+ output_grid_lat = def_xy_interp[1]
+
+ print( f'output_grid_type = {output_grid_type }\n' + \
+ f'remap_file = {remap_file }\n' + \
+ f'more_options = {more_options }\n' + \
+ f'output_grid_lon = {output_grid_lon }\n' + \
+ f'output_grid_lat = {output_grid_lat }\n' + \
+ f'regrid_vars = {regrid_vars }\n' )
+
+
+
+ # prepare to create input_mosaic via ncks call
+ if input_realm in ['atmos', 'aerosol']:
+ mosaic_type = 'atm_mosaic_file'
+ elif input_realm == 'ocean':
+ mosaic_type = 'ocn_mosaic_file'
+ elif input_realm == 'land':
+ mosaic_type = 'lnd_mosaic_file'
+ else:
+ raise ValueError(f'input_realm={input_realm} not recognized.')
+
+ # this is just to get the grid_file name
+ #print(f'mosaic_type = {mosaic_type}')
+ #print(f'grid_spec_file = {grid_spec_file}')
+ #print(f'input_mosaic = get_mosaic_file_name(grid_spec_file, mosaic_type)')
+
+ # assume input_mosaic near input grid_spec, where intially specified.
+ input_mosaic = input_dir + get_mosaic_file_name(grid_spec_file, mosaic_type)
+ #print(f'input_mosaic = {input_mosaic}')
+
+ ## this is to get the tile1 filename?
+ mosaic_grid_file = input_dir + get_mosaic_grid_file_name(input_mosaic)
+ #print(f'mosaic_grid_file = {mosaic_grid_file}')
+
+ # need source file dimenions for lat/lon
+ source_nx = str(int(Dataset(mosaic_grid_file).dimensions['nx'].size / 2 ))
+ source_ny = str(int(Dataset(mosaic_grid_file).dimensions['ny'].size / 2 ))
+ print(f'source_[nx,ny] = ({source_nx},{source_ny})')
+ Dataset(mosaic_grid_file).close()
+
+
+ if remap_file is not None:
+ try:
+ shutil.copy( remap_file,
+ input_dir+remap_file.split('/').pop() )
+ except Exception as exc:
+ raise OSError('remap_file={remap_file} could not be copied to local dir') \
+ from exc
+ else:
+ print('remap_file was not specified nor found. looking for default one')
+ remap_file= f'fregrid_remap_file_{output_grid_lon}_by_{output_grid_lat}.nc' \
+ if \
+ all( [output_grid_lon is not None,
+ output_grid_lat is not None]) \
+ else \
+ f'fregrid_remap_file_{def_xy_interp(0)}_by_{def_xy_interp(1)}.nc'
+ remap_cache_file = \
+ f'{remap_dir}/{input_grid}/{input_realm}/' + \
+ f'{source_nx}-by-{source_ny}/{interp_method}/{remap_file}'
+ central_remap_cache_file = \
+ f'{FREGRID_SHARED_FILES}/{input_grid}/' + \
+ f'{source_nx}_by_{source_ny}/{remap_file}'
+
+ print(f'remap_file = {remap_file }' + \
+ f'remap_cache_file = {remap_cache_file }' + \
+ f'central_remap_cache_file = {central_remap_cache_file}' )
+
+ if Path( remap_cache_file ).exists():
+ print(f'NOTE: using cached remap file {remap_cache_file}')
+ shutil.copy(remap_cache_file,
+ work_dir+remap_cache_file.split('/').pop())
+ elif Path( central_remap_cache_file ).exists():
+ print(f'NOTE: using centrally cached remap file {remap_cache_file}')
+ shutil.copy(central_remap_cache_file,
+ work_dir+central_remap_cache_file.split('/').pop())
+
+
+
+ # if no variables in config, find the interesting ones to regrid
+ if regrid_vars is None:
+ regrid_vars=make_regrid_var_list( target_file , interp_method)
+ print(f'regrid_vars = {regrid_vars}') #DELETE
+
+ #check if there's anything worth regridding
+ if len(regrid_vars) < 1:
+ raise ValueError('make_regrid_var_list found no vars to regrid. and no vars given. exit')
+ print(f'regridding {len(regrid_vars)} variables: {regrid_vars}')
+ regrid_vars_str=','.join(regrid_vars) # fregrid needs comma-demarcated list of vars
+
+
+
+ # massage input file argument to fregrid.
+ input_file = target_file.replace('.tile1.nc','') \
+ if '.tile1' in target_file \
+ else target_file
+ input_file=input_file.split('/').pop()
+
+ # create output file argument...
+ output_file = target_file.replace('.tile1','') \
+ if 'tile1' in target_file \
+ else target_file
+ output_file = work_dir + output_file.split('/').pop()
+
+ fregrid_command = [
+ 'fregrid',
+ '--debug',
+ '--standard_dimension',
+ '--input_mosaic', f'{input_mosaic}',
+ '--input_dir', f'{input_dir}',
+ '--input_file', f'{input_file}',
+ '--associated_file_dir', f'{input_dir}',
+ '--interp_method', f'{interp_method}',
+ '--remap_file', f'{remap_file}',
+ '--nlon', f'{str(output_grid_lon)}',
+ '--nlat', f'{str(output_grid_lat)}',
+ '--scalar_field', f'{regrid_vars_str}',
+ '--output_file', f'{output_file}']
+ if more_options is not None:
+ fregrid_command.append(f'{more_options}')
+
+ print(f"\n\nabout to run the following command: \n{' '.join(fregrid_command)}\n")
+ fregrid_proc = subprocess.run( fregrid_command, check = False )#i hate it
+ fregrid_rc =fregrid_proc.returncode
+ print(f'fregrid_result.returncode()={fregrid_rc}')
+
+
+ # output wrangling
+
+ # copy the remap file to the cache location
+ if not Path( remap_cache_file ).exists():
+ remap_cache_file_dir='/'.join(remap_cache_file.split('/')[0:-1])
+ Path( remap_cache_file_dir ).mkdir( parents = True , exist_ok = True)
+ print(f'copying \nremap_file={remap_file} to')
+ print(f'remap_cache_file_dir={remap_cache_file_dir}')
+ shutil.copy(remap_file, remap_cache_file_dir)
+
+ # more output wrangling
+ final_output_dir = output_dir \
+ if output_grid_type is None \
+ else output_dir + '/' + output_grid_type
+ Path( final_output_dir ).mkdir( exist_ok = True)
+
+ print(f'TRYING TO COPY {output_file} TO {final_output_dir}')
+ shutil.copy(output_file, final_output_dir)
+
+ continue # end of comp loop, exit or next one.
+
+ #os.chdir(starting_dir) # not clear this is necessary.
+ print('done running regrid_xy()')
+ return 0
+
+
+@click.command()
+def _regrid_xy( input_dir, output_dir, begin, tmp_dir,
+ remap_dir, source, grid_spec, def_xy_interp ):
+ """ click entrypoint """
+ click.echo(f'(_regrid_xy) locals={locals()}')
+ click.echo( '(_regrid_xy) click entrypoint hit- calling regrid_xy()')
+ return regrid_xy( input_dir, output_dir, begin, tmp_dir,
+ remap_dir, source, grid_spec, def_xy_interp )
+
+
+def main():
+ """steering, local test/debug"""
+ return regrid_xy()
+
+if __name__=='__main__':
+ main()
diff --git a/lib/fre/app/regrid_xy/tests/test_inputs_outputs/input_directory.tar.gz b/lib/fre/app/regrid_xy/tests/test_inputs_outputs/input_directory.tar.gz
new file mode 100644
index 00000000..003d654a
Binary files /dev/null and b/lib/fre/app/regrid_xy/tests/test_inputs_outputs/input_directory.tar.gz differ
diff --git a/lib/fre/app/regrid_xy/tests/test_regrid_xy.py b/lib/fre/app/regrid_xy/tests/test_regrid_xy.py
new file mode 100644
index 00000000..fa511b66
--- /dev/null
+++ b/lib/fre/app/regrid_xy/tests/test_regrid_xy.py
@@ -0,0 +1,494 @@
+""" test regrid_xy functioning as an importable python module """
+from pathlib import Path
+import subprocess
+import os
+
+import pytest
+
+import fre.app.regrid_xy.regrid_xy as rgxy
+
+# directories for tests
+TEST_DIR = os.getcwd() + "/fre/app/regrid_xy/tests/test_inputs_outputs/"
+
+TAR_IN_DIR = TEST_DIR + 'input_directory.tar.gz' #contains in-dir
+IN_DIR = TEST_DIR + 'in-dir/'
+WORK_DIR = TEST_DIR + 'work/'
+
+TEST_OUT_DIR = TEST_DIR + 'out-dir/'
+REMAP_DIR = TEST_DIR + 'remap-dir/'
+
+ALL_TEST_OUT_DIR = TEST_DIR + "out-test-dir/"
+REMAP_TEST_DIR = TEST_DIR + 'remap-test-dir/'
+
+# official data/mosaic stuff
+GOLD_GRID_SPEC = "/archive/gold/datasets/OM4_05/mosaic_c96.v20180227.tar"
+GOLD_GRID_SPEC_NO_TAR = IN_DIR + "mosaic.nc"
+TEST_CDL_GRID_FILE = IN_DIR + "C96_mosaic.cdl"
+
+
+## for rose configuration
+COMPONENT='atmos'
+NLON=288
+NLAT=180
+INPUT_GRID='cubedsphere'
+INPUT_REALM='atmos'
+INTERP_METHOD='conserve_order1'
+YYYYMMDD='20030101'
+SOURCE='atmos_static_cmip' # generally a list, but for tests, only one
+ # note that components will not always contain a
+ # a source file of the same name
+SOURCES_XY = '96-by-96'
+
+WORK_YYYYMMDD_DIR = WORK_DIR + f'{YYYYMMDD}.nc/'
+TEST_NC_GRID_FILE = WORK_YYYYMMDD_DIR + "C96_mosaic.nc" # output of first ncgen test
+
+def test_setup_clean_up(capfd):
+ """ cleanup i/o directories is present for clean regrid_xy testing """
+ try:
+ Path(IN_DIR).unlink()
+ except OSError:
+ pass
+ try:
+ Path(TEST_NC_GRID_FILE).unlink()
+ except OSError:
+ pass
+ try:
+ Path(WORK_DIR).unlink()
+ except OSError:
+ pass
+ try:
+ Path(TEST_OUT_DIR).unlink()
+ except OSError:
+ pass
+ try:
+ Path(ALL_TEST_OUT_DIR).unlink()
+ except OSError:
+ pass
+ try:
+ Path(REMAP_DIR).unlink()
+ except OSError:
+ pass
+ try:
+ Path(REMAP_TEST_DIR).unlink()
+ except OSError:
+ pass
+ assert True
+ out, err = capfd.readouterr()
+
+
+def test_setup_global_work_dirs(capfd):
+ """ create i/o directories for regrid_xy testing """
+ Path(WORK_YYYYMMDD_DIR).mkdir(parents = True, exist_ok = True)
+ assert Path(WORK_YYYYMMDD_DIR).exists()
+
+ Path(REMAP_TEST_DIR).mkdir(exist_ok = True)
+ assert Path(REMAP_TEST_DIR).exists()
+
+ Path(ALL_TEST_OUT_DIR).mkdir(exist_ok = True)
+ assert Path(ALL_TEST_OUT_DIR).exists()
+
+ Path(TEST_OUT_DIR).mkdir(exist_ok = True)
+ assert Path(TEST_OUT_DIR).exists()
+
+ Path(REMAP_DIR).mkdir(exist_ok = True)
+ assert Path(REMAP_DIR).exists()
+ out, err = capfd.readouterr()
+
+
+def test_untar_inputs(capfd):
+ """ untar input directory tarball to create test inputs """
+ ex = ["tar", "-C", TEST_DIR, "-zxvf", TAR_IN_DIR]
+ sp = subprocess.run( ex , check = True )
+ assert all ( [ sp.returncode == 0,
+ Path(IN_DIR).exists() ] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_make_ncgen3_nc_inputs(capfd):
+ """
+ set-up test: ncgen3 netcdf file inputs for later steps
+ if the output exists, it will not bother remaking it
+ """
+
+ ncgen3_output = TEST_NC_GRID_FILE
+ ncgen3_input = TEST_CDL_GRID_FILE
+ if Path(ncgen3_output).exists():
+ assert True
+ else:
+ assert Path(TEST_DIR).exists()
+ assert Path(ncgen3_input).exists()
+ ex = [ 'ncgen3', '-k', 'netCDF-4', '-o', ncgen3_output , ncgen3_input ]
+ print (' '.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert all( [ sp.returncode == 0,
+ Path(ncgen3_output).exists() ] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_make_ncgen_tile_nc_inputs(capfd):
+ """
+ set-up test: ncgen netcdf tile file inputs for later steps
+ if the output exists, it will not bother remaking it
+ """
+ ncgen_tile_i_nc_output_exists = [ Path( WORK_YYYYMMDD_DIR + \
+ f'{YYYYMMDD}.{SOURCE}.tile{i}.nc' ).exists() \
+ for i in range(1, 6+1) ]
+ if all( ncgen_tile_i_nc_output_exists ):
+ assert True
+ else:
+ for i in range(1, 6+1):
+ ncgen_tile_i_nc_output = WORK_YYYYMMDD_DIR + f'{YYYYMMDD}.{SOURCE}.tile{i}.nc'
+ ncgen_tile_i_cdl_input = IN_DIR + f'{YYYYMMDD}.{SOURCE}.tile{i}.cdl'
+ assert Path(ncgen_tile_i_cdl_input).exists()
+ ex = [ 'ncgen', '-o', ncgen_tile_i_nc_output, ncgen_tile_i_cdl_input ]
+ print (' '.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert all( [sp.returncode == 0,
+ Path(ncgen_tile_i_nc_output).exists()] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_make_ncgen_grid_spec_nc_inputs(capfd):
+ """
+ set-up test: ncgen netcdf grid spec tile file inputs for later steps
+ if the output exists, it will not bother remaking it
+ """
+ ncgen_grid_spec_i_nc_output_exists = [ Path( WORK_YYYYMMDD_DIR + \
+ f'{YYYYMMDD}.grid_spec.tile{i}.nc').exists() \
+ for i in range(1, 6+1) ]
+ if all( ncgen_grid_spec_i_nc_output_exists ):
+ assert True
+ else:
+ for i in range(1, 6+1):
+ ncgen_grid_spec_i_nc_output = WORK_YYYYMMDD_DIR + f'{YYYYMMDD}.grid_spec.tile{i}.nc'
+ ncgen_grid_spec_i_cdl_input = IN_DIR + f'{YYYYMMDD}.grid_spec.tile{i}.cdl'
+ ex = [ 'ncgen', '-o', ncgen_grid_spec_i_nc_output, ncgen_grid_spec_i_cdl_input ]
+ print (' '.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert all( [sp.returncode == 0,
+ Path(ncgen_grid_spec_i_nc_output).exists()] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_make_hgrid_gold_input(capfd):
+ """
+ set-up test: make C96 gold input via make_hgrid for later steps
+ if the output exists in the desired location, it will not bother remaking it
+ """
+ grid_i_file_targ_loc_exists = [ Path( WORK_YYYYMMDD_DIR + f'C96_grid.tile{i}.nc').exists() \
+ for i in range(1,6+1) ]
+ if all( grid_i_file_targ_loc_exists ):
+ assert True
+ else:
+ # remake gold grid file locally, then move
+ ex = [ "make_hgrid",
+ "--grid_type", "gnomonic_ed",
+ "--nlon", "192",
+ "--grid_name", "C96_grid" ]
+ print (' '.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert sp.returncode == 0
+ out, err = capfd.readouterr()
+
+ # now move the files...
+ for i in range(1, 6+1):
+ grid_i_file_curr_loc = f'C96_grid.tile{i}.nc'
+ grid_i_file_targ_loc = WORK_YYYYMMDD_DIR + f'C96_grid.tile{i}.nc'
+ ex = [ 'mv', '-f', grid_i_file_curr_loc, grid_i_file_targ_loc ]
+ print (' '.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert all( [sp.returncode == 0,
+ Path(grid_i_file_targ_loc).exists()] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_make_fregrid_comparison_input(capfd):
+ """
+ set-up test: use fregrid to regrid for later comparison to regrid_xy output
+ if the output exists in the desired location, it will not bother remaking it
+ """
+
+ fregrid_input_mosaic_arg = TEST_NC_GRID_FILE
+ fregrid_input_dir_arg = WORK_YYYYMMDD_DIR
+ fregrid_input_file_arg = f'{YYYYMMDD}.{SOURCE}'
+ fregrid_assoc_file_dir_arg = WORK_YYYYMMDD_DIR
+
+ fregrid_remap_file = f'fregrid_remap_file_{NLON}_by_{NLAT}.nc'
+ fregrid_remap_file_arg = REMAP_TEST_DIR + fregrid_remap_file
+
+ fregrid_nlat_arg = str(NLAT)
+ fregrid_nlon_arg = str(NLON)
+ fregrid_vars_arg = 'grid_xt,grid_yt,orog'
+
+
+ fregrid_output_file_arg = ALL_TEST_OUT_DIR + fregrid_input_file_arg + '.nc'
+
+ ex = [ 'fregrid', '--standard_dimension',
+ '--input_mosaic', fregrid_input_mosaic_arg,
+ '--input_dir', fregrid_input_dir_arg,
+ '--input_file', fregrid_input_file_arg,
+ '--associated_file_dir', fregrid_assoc_file_dir_arg,
+ '--remap_file', fregrid_remap_file_arg,
+ '--nlon', fregrid_nlon_arg,
+ '--nlat', fregrid_nlat_arg,
+ '--scalar_field', fregrid_vars_arg,
+ '--output_file', fregrid_output_file_arg ]
+ print (' \n'.join(ex))
+ sp = subprocess.run( ex , check = True )
+
+ assert all( [ sp.returncode == 0,
+ Path(fregrid_remap_file_arg).exists(),
+ Path(fregrid_output_file_arg).exists() ] )
+ out, err = capfd.readouterr()
+
+
+#@pytest.mark.skip(reason='debug')
+def test_import_regrid_xy(capfd):
+ """
+ check import of regrid_xy as a module
+ """
+ assert all( [ rgxy is not None,
+ rgxy.test_import() == 1 ] )
+ out, err = capfd.readouterr()
+
+#@pytest.mark.skip(reason='debug')
+def test_success_tar_grid_spec_regrid_xy(capfd):
+ """
+ checks for success of regrid_xy with rose app-app run
+ """
+ # this will only work at GFDL for now.
+ if not Path(GOLD_GRID_SPEC).exists():
+ assert True
+ else:
+ # for the time being, still a little dependent on rose for configuration value passing
+ if Path(os.getcwd()+'/rose-app-run.conf').exists():
+ Path(os.getcwd()+'/rose-app-run.conf').unlink()
+
+ with open(os.getcwd()+'/rose-app-run.conf','a',encoding='utf-8') as rose_app_run_config:
+ rose_app_run_config.write( '[command]\n' )
+ rose_app_run_config.write( 'default=regrid-xy\n' )
+ rose_app_run_config.write( '\n' )
+ rose_app_run_config.write( f'[{COMPONENT}]\n' )
+ rose_app_run_config.write( f'sources={SOURCE}\n' )
+ rose_app_run_config.write( f'inputGrid={INPUT_GRID}\n' )
+ rose_app_run_config.write( f'inputRealm={INPUT_REALM}\n' )
+ rose_app_run_config.write( f'interpMethod={INTERP_METHOD}\n' )
+ rose_app_run_config.write( f'outputGridLon={NLON}\n' )
+ rose_app_run_config.write( f'outputGridLat={NLAT}\n' )
+ rose_app_run_config.write( '\n' )
+ assert Path('./rose-app-run.conf').exists()
+
+ rgxy_returncode = rgxy.regrid_xy(
+ input_dir = WORK_YYYYMMDD_DIR,
+ output_dir = TEST_OUT_DIR,
+ begin = f'{YYYYMMDD}T000000',
+ tmp_dir = TEST_DIR,
+ remap_dir = REMAP_DIR,
+ source = SOURCE,
+ grid_spec = GOLD_GRID_SPEC,
+ def_xy_interp = f'"{NLON},{NLAT}"'
+ )
+
+ # uhm....
+ assert rgxy_returncode == 0
+ assert Path( REMAP_DIR + \
+ f'{INPUT_GRID}/{INPUT_REALM}/96-by-96/{INTERP_METHOD}/' + \
+ f'fregrid_remap_file_{NLON}_by_{NLAT}.nc' \
+ ).exists()
+ assert Path( TEST_OUT_DIR ).exists()
+ assert Path( TEST_OUT_DIR + f'{YYYYMMDD}.{SOURCE}.nc' ).exists()
+ assert Path( WORK_DIR ).exists()
+ assert Path( WORK_DIR + f'{YYYYMMDD}.{SOURCE}.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'basin_codes.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile2.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile3.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile4.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile5.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile6.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile1XC96_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile1Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile2XC96_mosaic_tile2.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile2Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile3XC96_mosaic_tile3.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile3Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile4XC96_mosaic_tile4.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile4Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile5XC96_mosaic_tile5.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile5Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile6XC96_mosaic_tile6.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic_tile6Xocean_mosaic_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'hash.md5' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile2.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile3.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile4.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile5.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'land_mask_tile6.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'mosaic.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'ocean_hgrid.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'ocean_mask.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'ocean_mosaic.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'ocean_static.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'ocean_topog.nc' ).exists()
+ out, err = capfd.readouterr()
+ assert True
+
+
+
+
+#@pytest.mark.skip(reason='debug')
+def test_success_no_tar_grid_spec_regrid_xy(capfd):
+ """
+ checks for success of regrid_xy with rose app-app run
+ """
+ # for the time being, still a little dependent on rose for configuration value passing
+ if Path(os.getcwd()+'/rose-app-run.conf').exists():
+ Path(os.getcwd()+'/rose-app-run.conf').unlink()
+
+ with open(os.getcwd()+'/rose-app-run.conf','a',encoding='utf-8') as rose_app_run_config:
+ rose_app_run_config.write( '[command]\n' )
+ rose_app_run_config.write( 'default=regrid-xy\n' )
+ rose_app_run_config.write( '\n' )
+ rose_app_run_config.write( f'[{COMPONENT}]\n' )
+ rose_app_run_config.write( f'sources={SOURCE}\n' )
+ rose_app_run_config.write( f'inputGrid={INPUT_GRID}\n' )
+ rose_app_run_config.write( f'inputRealm={INPUT_REALM}\n' )
+ rose_app_run_config.write( f'interpMethod={INTERP_METHOD}\n' )
+ rose_app_run_config.write( f'outputGridLon={NLON}\n' )
+ rose_app_run_config.write( f'outputGridLat={NLAT}\n' )
+ rose_app_run_config.write( '\n' )
+ assert Path('./rose-app-run.conf').exists()
+
+ rgxy_returncode = rgxy.regrid_xy(
+ input_dir = WORK_YYYYMMDD_DIR,
+ output_dir = TEST_OUT_DIR,
+ begin = f'{YYYYMMDD}T000000',
+ tmp_dir = TEST_DIR,
+ remap_dir = REMAP_DIR,
+ source = SOURCE,
+ grid_spec = GOLD_GRID_SPEC_NO_TAR,
+ def_xy_interp = f'"{NLON},{NLAT}"'
+ )
+
+ # uhm....
+ assert rgxy_returncode == 0
+ assert Path( REMAP_DIR + \
+ f'{INPUT_GRID}/{INPUT_REALM}/96-by-96/{INTERP_METHOD}/' + \
+ f'fregrid_remap_file_{NLON}_by_{NLAT}.nc' \
+ ).exists()
+ assert Path( TEST_OUT_DIR ).exists()
+ assert Path( TEST_OUT_DIR + f'{YYYYMMDD}.{SOURCE}.nc' ).exists()
+ assert Path( WORK_DIR ).exists()
+ assert Path( WORK_DIR + f'{YYYYMMDD}.{SOURCE}.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR ).exists()
+
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile1.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile2.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile3.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile4.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile5.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_grid.tile6.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'C96_mosaic.nc' ).exists()
+ assert Path( WORK_YYYYMMDD_DIR + 'mosaic.nc' ).exists()
+ out, err = capfd.readouterr()
+
+
+
+
+@pytest.mark.skip(reason='debug')
+def test_failure_wrong_datetime_regrid_xy(capfd):
+ """
+ checks for failure of regrid_xy with rose app-run when fed an
+ invalid date for begin
+ """
+ # for the time being, still a little dependent on rose for configuration value passing
+ if Path(os.getcwd()+'/rose-app-run.conf').exists():
+ Path(os.getcwd()+'/rose-app-run.conf').unlink()
+
+ with open(os.getcwd()+'/rose-app-run.conf','a',encoding='utf-8') as rose_app_run_config:
+ rose_app_run_config.write( '[command]\n' )
+ rose_app_run_config.write( 'default=regrid-xy\n' )
+ rose_app_run_config.write( '\n' )
+ rose_app_run_config.write( f'[{COMPONENT}]\n' )
+ rose_app_run_config.write( f'sources={SOURCE}\n' )
+ rose_app_run_config.write( f'inputGrid={INPUT_GRID}\n' )
+ rose_app_run_config.write( f'inputRealm={INPUT_REALM}\n' )
+ rose_app_run_config.write( f'interpMethod={INTERP_METHOD}\n' )
+ rose_app_run_config.write( f'outputGridLon={NLON}\n' )
+ rose_app_run_config.write( f'outputGridLat={NLAT}\n' )
+ rose_app_run_config.write( '\n' )
+ assert Path('./rose-app-run.conf').exists()
+
+ try:
+ rgxy_returncode = rgxy.regrid_xy(
+ input_dir = WORK_YYYYMMDD_DIR,
+ output_dir = TEST_OUT_DIR,
+ begin = '99999999T999999',
+ tmp_dir = TEST_DIR,
+ remap_dir = REMAP_DIR,
+ source = SOURCE,
+ grid_spec = GOLD_GRID_SPEC,
+ def_xy_interp = f'"{NLON},{NLAT}"'
+ )
+ except:
+ # yay good job
+ assert True
+
+ out, err = capfd.readouterr()
+
+
+
+#@pytest.mark.skip(reason='debug')
+
+
+def test_nccmp1_regrid_xy(capfd):
+ """
+ This test compares the output of make_hgrid and fregrid, which are expected to be identical
+ """
+ fregrid_remap_file=f'fregrid_remap_file_{NLON}_by_{NLAT}.nc'
+
+ nccmp_arg1 = REMAP_DIR + INPUT_GRID + '/' + INPUT_REALM + '/' + \
+ SOURCES_XY + '/' + INTERP_METHOD + '/' + fregrid_remap_file
+ nccmp_arg2 = REMAP_TEST_DIR + fregrid_remap_file
+ nccmp= [ 'nccmp', '-m', '--force', nccmp_arg1, nccmp_arg2 ]
+ print (' '.join(nccmp))
+ sp = subprocess.run( nccmp, check = True)
+ assert sp.returncode == 0
+ out, err = capfd.readouterr()
+
+
+def test_nccmp2_regrid_xy(capfd):
+ """
+ This test compares the regridded source file output(s), which are expected to be identical
+ """
+ nccmp_arg1 = TEST_OUT_DIR + f'{YYYYMMDD}.{SOURCE}.nc'
+ nccmp_arg2 = ALL_TEST_OUT_DIR + f'{YYYYMMDD}.{SOURCE}.nc'
+ nccmp= [ 'nccmp', '-m', '--force', nccmp_arg1, nccmp_arg2 ]
+ print (' '.join(nccmp))
+ sp = subprocess.run( nccmp, check = True)
+ assert sp.returncode == 0
+ out, err = capfd.readouterr()
+
+
+@pytest.mark.skip(reason='TODO')
+def test_regrid_one_for_two_comps(capfd):
+ """
+ this test will compare regridding settings for a single source file ref'd in two
+ diff components and regrid that source file twice if the settings are different,
+ and only once if the settings are the same.
+ """
+ assert False
+ out, err = capfd.readouterr()
diff --git a/lib/fre/catalog/__init__.py b/lib/fre/catalog/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/catalog/frecatalog.py b/lib/fre/catalog/frecatalog.py
new file mode 100644
index 00000000..58149e5e
--- /dev/null
+++ b/lib/fre/catalog/frecatalog.py
@@ -0,0 +1,62 @@
+'''
+entry point for fre catalog subcommands
+'''
+
+import click
+#import catalogbuilder
+from catalogbuilder.scripts import gen_intake_gfdl
+from catalogbuilder.scripts import test_catalog
+from catalogbuilder.scripts import combine_cats
+
+
+@click.group(help=click.style(" - access fre catalog subcommands", fg=(64,94,213)))
+def catalog_cli():
+ ''' entry point for click into fre catalog cli calls '''
+
+
+
+@catalog_cli.command()
+#TODO arguments dont have help message. So consider changing arguments to options?
+@click.argument('input_path', required = False, nargs = 1)
+#, help = 'The directory path with the datasets to be cataloged. E.g a GFDL PP path till /pp')
+@click.argument('output_path', required = False, nargs = 1)
+#, help = 'Specify output filename suffix only. e.g. catalog')
+@click.option('--config', required = False, type = click.Path(exists = True), nargs = 1,
+ help = 'Path to your yaml config, Use the config_template in intakebuilder repo')
+@click.option('--filter_realm', nargs = 1)
+@click.option('--filter_freq', nargs = 1)
+@click.option('--filter_chunk', nargs = 1)
+@click.option('--overwrite', is_flag = True, default = False)
+@click.option('--append', is_flag = True, default = False)
+@click.option('--slow', is_flag = True, default = False,
+ help = "Open NetCDF files to retrieve additional vocabulary (standard_name and intrafile static variables")
+@click.pass_context
+def builder(context, input_path = None, output_path = None, config = None, filter_realm = None,
+ filter_freq = None, filter_chunk = None, overwrite = False, append = False, slow = False):
+ # pylint: disable=unused-argument
+ """ - Generate .csv and .json files for catalog """
+ context.forward(gen_intake_gfdl.create_catalog_cli)
+
+@catalog_cli.command()
+@click.argument('json_path', nargs = 1 , required = True)
+@click.argument('json_template_path', nargs = 1 , required = False)
+@click.option('-tf', '--test-failure', is_flag=True, default = False,
+ help="Errors are only printed. Program will not exit.")
+@click.pass_context
+def validate(context, json_path, json_template_path, test_failure):
+ # pylint: disable=unused-argument
+ """ - Validate a catalog against catalog schema """
+ context.forward(test_catalog.main)
+
+@catalog_cli.command()
+@click.option('--input', required = True, multiple = True,
+ help = 'Catalog json files to be merged, space-separated')
+@click.option('--output', required = True, nargs = 1,
+ help = 'Merged catalog')
+@click.pass_context
+def merge(context, input, output):
+ """ - Merge two or more more catalogs into one """
+ context.invoke(combine_cats.combine_cats, inputfiles=input, output_path=output)
+
+if __name__ == "__main__":
+ catalog_cli()
diff --git a/lib/fre/catalog/tests/__init__.py b/lib/fre/catalog/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/catalog/tests/test_fre_catalog.py b/lib/fre/catalog/tests/test_fre_catalog.py
new file mode 100644
index 00000000..7bc36637
--- /dev/null
+++ b/lib/fre/catalog/tests/test_fre_catalog.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+def test_fre_catalog_import():
+ from fre import catalog
+ assert catalog is not None
diff --git a/lib/fre/check/__init__.py b/lib/fre/check/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/check/frecheck.py b/lib/fre/check/frecheck.py
new file mode 100644
index 00000000..9640d138
--- /dev/null
+++ b/lib/fre/check/frecheck.py
@@ -0,0 +1,20 @@
+''' fre check '''
+
+import click
+
+from .frecheckexample import check_test_function
+
+@click.group(help=click.style(" - access fre check subcommands", fg=(162,91,232)))
+def check_cli():
+ ''' entry point to fre check click commands '''
+
+@check_cli.command()
+@click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
+@click.pass_context
+def function(context, uppercase):
+ # pylint: disable=unused-argument
+ """ - Execute fre check test """
+ context.forward(check_test_function)
+
+if __name__ == "__main__":
+ check_cli()
diff --git a/lib/fre/check/frecheckexample.py b/lib/fre/check/frecheckexample.py
new file mode 100644
index 00000000..daac462a
--- /dev/null
+++ b/lib/fre/check/frecheckexample.py
@@ -0,0 +1,18 @@
+"""
+experimentation file for integrating one file's functions into main prototype fre file
+authored by Bennett.Chang@noaa.gov | bcc2761
+NOAA | GFDL
+"""
+
+import click
+
+@click.command()
+def check_test_function(uppercase=None):
+ """Execute fre list testfunction2."""
+ statement = "testingtestingtestingtesting"
+ if uppercase:
+ statement = statement.upper()
+ click.echo(statement)
+
+if __name__ == '__main__':
+ check_test_function()
diff --git a/lib/fre/cmor/README.md b/lib/fre/cmor/README.md
new file mode 100644
index 00000000..865c1ee7
--- /dev/null
+++ b/lib/fre/cmor/README.md
@@ -0,0 +1,102 @@
+UNDER CONSTRUCTION: old usage notes at the top of `cmor_mixer.py`, re-rigged for markdown and CMIP7.
+
+at PP/AN, module load the latest `fre-cli` that's been pushed to the main branch:
+```
+> module load fre/canopy
+> which fre
+ /home/fms/local/opt/fre-commands/canopy/bin/fre
+```
+
+alternatively, with access to conda:
+```
+> conda activate /nbhome/fms/conda/envs/fre-cli
+> which fre
+ /nbhome/fms/conda/envs/fre-cli/bin/fre
+```
+
+this subtool's help, and command-specific `run` help:
+```
+> fre cmor --help
+ Usage: fre cmor [OPTIONS] COMMAND [ARGS]...
+
+ - access fre cmor subcommands
+
+ Options:
+ --help Show this message and exit.
+
+ Commands:
+ run Rewrite climate model output
+
+
+# subtool command-specific help, e.g. for run
+> fre cmor run --help
+Usage: fre cmor run [OPTIONS]
+
+ Rewrite climate model output files with CMIP-compliant metadata for down-
+ stream publishing
+
+Options:
+ -d, --indir TEXT directory containing netCDF files. keys specified
+ in json_var_list are local variable names used for
+ targeting specific files in this directory
+ [required]
+ -l, --varlist TEXT path pointing to a json file containing directory
+ of key/value pairs. the keys are the 'local' names
+ used in the filename, and the values pointed to by
+ those keys are strings representing the name of the
+ variable contained in targeted files. the key and
+ value are often the same, but it is not required.
+ [required]
+ -r, --table_config TEXT json file containing CMIP-compliant per-
+ variable/metadata for specific MIP table. The MIP
+ table can generally be identified by the specific
+ filename (e.g. 'Omon') [required]
+ -p, --exp_config TEXT json file containing metadata dictionary for
+ CMORization. this metadata is effectively appended
+ to the final output file's header [required]
+ -o, --outdir TEXT directory root that will contain the full output
+ and output directory structure generated by the
+ cmor module upon request. [required]
+ -v, --opt_var_name TEXT optional, specify a variable name to specifically
+ process only filenames matching that variable name.
+ I.e., this string help target local_vars, not
+ target_vars.
+ --help Show this message and exit.
+```
+
+
+the tool requires configuration in the form of variable tables and conventions to work appropriately
+clone the following repository and list the following directory contents to get a sense of what
+the code needs from you to work. a few examples shown in the output below. the CV file is of particular interest/necessity
+```
+> git clone https://github.com/PCMDI/cmip6-cmor-tables.git fre/tests/test_files/cmip6-cmor-tables
+> ls fre/tests/test_files/cmip6-cmor-tables/Tables
+...
+ CMIP6_CV.json
+ CMIP6_formula_terms.json
+ CMIP6_grids.json
+ CMIP6_coordinate.json
+ CMIP6_input_example.json
+...
+ CMIP6_3hr.json
+...
+ CMIP6_Efx.json
+...
+ CMIP6_IyrGre.json
+...
+```
+
+
+Simple example call(s) using fre-cli in the root directory of this repository note the line-continuation character at the end for readability,
+you may wish to avoid it when copy/pasting.
+```
+> fre cmor run \
+ -d fre/tests/test_files/ocean_sos_var_file \
+ -l fre/tests/test_files/varlist \
+ -r fre/tests/test_files/cmip6-cmor-tables/Tables/CMIP6_Omon.json \
+ -p fre/tests/test_files/CMOR_input_example.json \
+ -o fre/tests/test_files/outdir
+```
+
+
+
diff --git a/lib/fre/cmor/__init__.py b/lib/fre/cmor/__init__.py
new file mode 100644
index 00000000..465ad02c
--- /dev/null
+++ b/lib/fre/cmor/__init__.py
@@ -0,0 +1,2 @@
+''' for fre.cmor imports '''
+from .cmor_mixer import cmor_run_subtool
diff --git a/lib/fre/cmor/cmor_mixer.py b/lib/fre/cmor/cmor_mixer.py
new file mode 100644
index 00000000..0f4ef243
--- /dev/null
+++ b/lib/fre/cmor/cmor_mixer.py
@@ -0,0 +1,652 @@
+'''
+python module housing the metadata processing routines utilizing the cmor module, in addition to
+click API entry points
+see README.md for additional information on `fre cmor run` (cmor_mixer.py) usage
+'''
+
+import os
+import glob
+import json
+import subprocess
+from pathlib import Path
+
+import netCDF4 as nc
+import click
+import cmor
+
+# ----- \start consts
+DEBUG_MODE_RUN_ONE = True
+
+# ----- \end consts
+
+### ------ helper functions ------ ###
+def copy_nc(in_nc, out_nc):
+ '''
+ copy target input netcdf file in_nc to target out_nc. I have to think this is not a trivial copy
+ operation, as if it were, using shutil's copy would be sufficient. accepts two arguments
+ in_nc: string, path to an input netcdf file we wish to copy
+ out_nc: string, an output path to copy the targeted input netcdf file to
+ '''
+ print(f'(copy_nc) in_nc: {in_nc}\n'
+ f' out_nc: {out_nc}')
+
+ # input file
+ dsin = nc.Dataset(in_nc)
+
+ # output file, same exact data_model as input file.
+ # note- totally infuriating...
+ # the correct value for the format arg is netCDF4.Dataset.data_model
+ # and NOT netCDF4.Dataset.disk_format
+ dsout = nc.Dataset(out_nc, "w",
+ format = dsin.data_model)
+
+ #Copy dimensions
+ for dname, the_dim in dsin.dimensions.items():
+ dsout.createDimension( dname,
+ len(the_dim) if not the_dim.isunlimited() else None )
+
+ # Copy variables and attributes
+ for v_name, varin in dsin.variables.items():
+ out_var = dsout.createVariable(v_name, varin.datatype, varin.dimensions)
+ out_var.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
+ out_var[:] = varin[:]
+ dsout.setncatts({a:dsin.getncattr(a) for a in dsin.ncattrs()})
+
+ # close up
+ dsin.close()
+ dsout.close()
+
+
+def get_var_filenames(indir, var_filenames = None, local_var = None):
+ '''
+ appends files ending in .nc located within indir to list var_filenames accepts three arguments
+ indir: string, representing a path to a directory containing files ending in .nc extension
+ var_filenames: list of strings, empty or non-empty, to append discovered filenames to. the
+ object pointed to by the reference var_filenames is manipulated, and so need
+ not be returned.
+ local_var: string, optional, if not None, will be used for ruling out filename targets
+ '''
+ if var_filenames is None:
+ var_filenames = []
+ filename_pattern='.nc' if local_var is None else f'.{local_var}.nc'
+ print(f'(get_var_filenames) filename_pattern={filename_pattern}')
+ var_filenames_all=glob.glob(f'{indir}/*{filename_pattern}')
+ print(f'(get_var_filenames) var_filenames_all={var_filenames_all}')
+ for var_file in var_filenames_all:
+ var_filenames.append( Path(var_file).name )
+ print(f"(get_var_filenames) var_filenames = {var_filenames}")
+ if len(var_filenames) < 1:
+ raise ValueError(f'target directory had no files with .nc ending. indir =\n {indir}')
+ var_filenames.sort()
+
+
+def get_iso_datetimes(var_filenames, iso_datetime_arr = None):
+ '''
+ appends iso datetime strings found amongst filenames to iso_datetime_arr.
+ var_filenames: non-empty list of strings representing filenames. some of which presumably
+ contain datetime strings
+ iso_datetime_arr: list of strings, empty or non-empty, representing datetimes found in
+ var_filenames entries. the objet pointed to by the reference
+ iso_datetime_arr is manipulated, and so need-not be returned
+ '''
+ if iso_datetime_arr is None:
+ iso_datetime_arr = []
+ for filename in var_filenames:
+ iso_datetime = filename.split(".")[1]
+ if iso_datetime not in iso_datetime_arr:
+ iso_datetime_arr.append(
+ filename.split(".")[1] )
+ iso_datetime_arr.sort()
+ #print(f"(get_iso_datetimes) Available dates: {iso_datetime_arr}")
+ if len(iso_datetime_arr) < 1:
+ raise ValueError('(get_iso_datetimes) ERROR: iso_datetime_arr has length 0!')
+
+def check_dataset_for_ocean_grid(ds):
+ '''
+ checks netCDF4.Dataset ds for ocean grid origin, and throws an error if it finds one. accepts
+ one argument. this function has no return.
+ ds: netCDF4.Dataset object containing variables with associated dimensional information.
+ '''
+ if "xh" in list(ds.variables.keys()):
+ raise NotImplementedError(
+ "(check_dataset_for_ocean_grid) 'xh' found in var_list. ocean grid req'd but not yet unimplemented. stop.")
+
+
+def get_vertical_dimension(ds,target_var):
+ '''
+ determines the vertical dimensionality of target_var within netCDF4 Dataset ds. accepts two
+ arguments and returns an object represnting the vertical dimensions assoc with the target_var.
+ ds: netCDF4.Dataset object containing variables with associated dimensional information.
+ target_var: string, representating a variable contained within the netCDF4.Dataset ds
+
+ '''
+ vert_dim = 0
+ for name, variable in ds.variables.items():
+ # not the var we are looking for? move on.
+ if name != target_var:
+ continue
+ dims = variable.dimensions
+ for dim in dims:
+ # if it is not a vertical axis, move on.
+ print(f'(get_vertical_dimension) dim={dim}')
+ if dim == 'landuse':
+ continue
+ if not (ds[dim].axis and ds[dim].axis == "Z"):
+ continue
+ vert_dim = dim
+ return vert_dim
+
+def create_tmp_dir(outdir):
+ '''
+ creates a tmp_dir based on targeted output directory root. returns the name of the tmp dir.
+ accepts one argument:
+ outdir: string, representing the final output directory root for the cmor modules netcdf
+ file output. tmp_dir will be slightly different depending on the output directory
+ targeted
+ '''
+ print(f"(create_tmp_dir) outdir = {outdir}")
+ tmp_dir = None
+ if any( [ outdir == "/local2",
+ outdir.find("/work") != -1,
+ outdir.find("/net" ) != -1 ] ):
+ print(f'(create_tmp_dir) using /local /work /net ( tmp_dir = {outdir}/ )')
+ tmp_dir = str( Path("{outdir}/").resolve() )
+ else:
+ print(f'(create_tmp_dir) NOT using /local /work /net (tmp_dir = {outdir}/tmp/ )')
+ tmp_dir = str( Path(f"{outdir}/tmp/").resolve() )
+ try:
+ os.makedirs(tmp_dir, exist_ok=True)
+ except Exception as exc:
+ raise OSError('(create_tmp_dir) problem creating temp output directory. stop.') from exc
+ return tmp_dir
+
+
+
+### ------ BULK ROUTINES ------ ###
+def rewrite_netcdf_file_var ( proj_table_vars = None,
+ local_var = None,
+ netcdf_file = None,
+ target_var = None,
+ json_exp_config = None,
+ json_table_config = None):#, tmp_dir = None ):
+ '''
+ rewrite the input netcdf file nc_fl containing target_var in a CMIP-compliant manner.
+ accepts six arguments, all required:
+ proj_table_vars: json dictionary object, variable table read from json_table_config.
+ local_var: string, variable name used for finding files locally containing target_var,
+ this argument is often equal to target_var.
+ netcdf_file: string, representing path to intput netcdf file.
+ target_var: string, representing the variable name attached to the data object in the netcdf file.
+ json_exp_config: string, representing path to json configuration file holding metadata for appending to output
+ this argument is most used for making sure the right grid label is getting attached to the right output
+ json_table_config: string, representing path to json configuration file holding variable names for a given table.
+ proj_table_vars is read from this file, but both are passed anyways.
+ '''
+ print('\n\n-------------------------- START rewrite_netcdf_file_var call -----')
+ print( "(rewrite_netcdf_file_var) input data: " )
+ print(f" local_var = {local_var}" )
+ print(f" target_var = {target_var}")
+
+
+ # open the input file
+ print(f"(rewrite_netcdf_file_var) opening {netcdf_file}" )
+ ds = nc.Dataset(netcdf_file,'a')
+
+
+ # ocean grids are not implemented yet.
+ print( '(rewrite_netcdf_file_var) checking input netcdf file for oceangrid condition')
+ check_dataset_for_ocean_grid(ds)
+
+
+ # figure out the dimension names programmatically TODO
+ # Define lat and lon dimensions
+ # Assume input file is lat/lon grid
+ lat = ds["lat"][:]
+ lon = ds["lon"][:]
+ lat_bnds = ds["lat_bnds"][:]
+ lon_bnds = ds["lon_bnds"][:]
+
+ ## Define time
+ #time = ds["time"][:]
+
+ # read in time_coords + units
+ time_coords = ds["time"][:]
+ time_coord_units = ds["time"].units
+ print(f"(rewrite_netcdf_file_var) time_coord_units = {time_coord_units}")
+
+ # read in time_bnds , if present
+ time_bnds = []
+ try:
+ time_bnds = ds["time_bnds"][:]
+ #print(f"(rewrite_netcdf_file_var) time_bnds = {time_bnds}")
+ except ValueError:
+ print( "(rewrite_netcdf_file_var) WARNING grabbing time_bnds didnt work... moving on")
+
+
+ # read the input variable data, i believe
+ var = ds[target_var][:]
+
+ # determine the vertical dimension by looping over netcdf variables
+ vert_dim = get_vertical_dimension(ds, target_var)
+ print(f"(rewrite_netcdf_file_var) Vertical dimension of {target_var}: {vert_dim}")
+
+ # grab var_dim
+ var_dim = len(var.shape)
+ print(f"(rewrite_netcdf_file_var) var_dim = {var_dim}, local_var = {local_var}")
+
+ # Check var_dim
+ if var_dim not in [3, 4]:
+ raise ValueError(f"var_dim == {var_dim} != 3 nor 4. stop.")
+
+ # Check var_dim and vert_dim and assign lev if relevant.
+ # error if vert_dim wrong given var_dim
+ lev = None
+ if var_dim == 4:
+ if vert_dim not in [ "plev30", "plev19", "plev8",
+ "height2m", "level", "lev", "levhalf"] :
+ raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported')
+ lev = ds[vert_dim]
+
+
+ # now we set up the cmor module object
+ # initialize CMOR
+ cmor.setup(
+ netcdf_file_action = cmor.CMOR_PRESERVE,
+ set_verbosity = cmor.CMOR_QUIET, #default is CMOR_NORMAL
+ exit_control = cmor.CMOR_NORMAL,
+ logfile = None,
+ create_subdirectories = 1
+ )
+
+ # read experiment configuration file
+ print(f"(rewrite_netcdf_file_var) cmor is opening: json_exp_config = {json_exp_config}")
+ cmor.dataset_json(json_exp_config)
+
+ # load CMOR table
+ print(f"(rewrite_netcdf_file_var) cmor is opening json_table_config = {json_table_config}")
+ cmor.load_table(json_table_config)
+
+ units = proj_table_vars["variable_entry"] [target_var] ["units"]
+ print(f"(rewrite_netcdf_file_var) units={units}")
+
+ cmor_lat = cmor.axis("latitude", coord_vals = lat, cell_bounds = lat_bnds, units = "degrees_N")
+ cmor_lon = cmor.axis("longitude", coord_vals = lon, cell_bounds = lon_bnds, units = "degrees_E")
+ try:
+ print( f"(rewrite_netcdf_file_var) Executing cmor.axis('time', \n"
+ f" coord_vals = \n{time_coords}, \n"
+ f" cell_bounds = time_bnds, units = {time_coord_units}) ")
+ cmor_time = cmor.axis("time", coord_vals = time_coords,
+ cell_bounds = time_bnds, units = time_coord_units)
+ except ValueError as exc:
+ print(f"(rewrite_netcdf_file_var) WARNING exception raised... exc={exc}\n"
+ " cmor_time = cmor.axis('time', \n"
+ " coord_vals = time_coords, units = time_coord_units)")
+ cmor_time = cmor.axis("time", coord_vals = time_coords, units = time_coord_units)
+
+ # initializations
+ save_ps = False
+ ps = None
+ ierr_ap, ierr_b = None, None
+ ips = None
+
+ # set axes for 3-dim case
+ if var_dim == 3:
+ axes = [cmor_time, cmor_lat, cmor_lon]
+ print(f"(rewrite_netcdf_file_var) axes = {axes}")
+ # set axes for 4-dim case
+ elif var_dim == 4:
+
+ if vert_dim in ["plev30", "plev19", "plev8", "height2m"]:
+ cmor_lev = cmor.axis( vert_dim,
+ coord_vals = lev[:], units = lev.units )
+
+ elif vert_dim in ["level", "lev", "levhalf"]:
+ # find the ps file nearby
+ ps_file = netcdf_file.replace(f'.{target_var}.nc', '.ps.nc')
+ ds_ps = nc.Dataset(ps_file)
+ ps = ds_ps['ps'][:].copy()
+ ds_ps.close()
+
+ # assign lev_half specifics
+ if vert_dim == "levhalf":
+ cmor_lev = cmor.axis( "alternate_hybrid_sigma_half",
+ coord_vals = lev[:],
+ units = lev.units )
+ ierr_ap = cmor.zfactor( zaxis_id = cmor_lev,
+ zfactor_name = "ap_half",
+ axis_ids = [cmor_lev, ],
+ zfactor_values = ds["ap_bnds"][:],
+ units = ds["ap_bnds"].units )
+ ierr_b = cmor.zfactor( zaxis_id = cmor_lev,
+ zfactor_name = "b_half",
+ axis_ids = [cmor_lev, ],
+ zfactor_values = ds["b_bnds"][:],
+ units = ds["b_bnds"].units )
+ else:
+ cmor_lev = cmor.axis( "alternate_hybrid_sigma",
+ coord_vals = lev[:],
+ units = lev.units,
+ cell_bounds = ds[vert_dim+"_bnds"] )
+ ierr_ap = cmor.zfactor( zaxis_id = cmor_lev,
+ zfactor_name = "ap",
+ axis_ids = [cmor_lev, ],
+ zfactor_values = ds["ap"][:],
+ zfactor_bounds = ds["ap_bnds"][:],
+ units = ds["ap"].units )
+ ierr_b = cmor.zfactor( zaxis_id = cmor_lev,
+ zfactor_name = "b",
+ axis_ids = [cmor_lev, ],
+ zfactor_values = ds["b"][:],
+ zfactor_bounds = ds["b_bnds"][:],
+ units = ds["b"].units )
+
+ print(f'(rewrite_netcdf_file_var) ierr_ap after calling cmor_zfactor: {ierr_ap}\n'
+ f'(rewrite_netcdf_file_var) ierr_b after calling cmor_zfactor: {ierr_b}' )
+ ips = cmor.zfactor( zaxis_id = cmor_lev,
+ zfactor_name = "ps",
+ axis_ids = [cmor_time, cmor_lat, cmor_lon],
+ units = "Pa" )
+ save_ps = True
+ # assign axes at end of 4-dim case
+ axes = [cmor_time, cmor_lev, cmor_lat, cmor_lon]
+
+
+
+
+ # read positive attribute and create cmor_var? can this return none? TODO
+ positive = proj_table_vars["variable_entry"] [target_var] ["positive"]
+ print(f"(rewrite_netcdf_file_var) positive = {positive}")
+ cmor_var = cmor.variable(target_var, units, axes, positive = positive)
+
+ # Write the output to disk
+ #var = ds[target_var][:] #was this ever needed? why?
+ cmor.write(cmor_var, var)
+ if save_ps:
+ if any( [ ips is None, ps is None ] ):
+ print( '(rewrite_netcdf_file_var) WARNING: ps or ips is None!, but save_ps is True!\n'
+ f' ps = {ps}, ips = {ips}\n'
+ ' skipping ps writing!' )
+ else:
+ cmor.write(ips, ps, store_with = cmor_var)
+ cmor.close(ips, file_name = True, preserve = False)
+ filename = cmor.close(cmor_var, file_name = True, preserve = False)
+ print(f"(rewrite_netcdf_file_var) returned by cmor.close: filename = {filename}")
+ #cmor.close()
+ ds.close()
+
+ print('-------------------------- END rewrite_netcdf_file_var call -----\n\n')
+ return filename
+
+
+def cmorize_target_var_files( indir = None, target_var = None, local_var = None,
+ iso_datetime_arr = None, name_of_set = None,
+ json_exp_config = None, outdir = None,
+ proj_table_vars = None, json_table_config = None ):
+ ''' processes a target directory/file
+ this routine is almost entirely exposed data movement before/after calling
+ rewrite_netcdf_file_var it is also the most hopelessly opaque routine in this entire dang macro.
+ this badboy right here accepts... lord help us... !!!NINE!!! arguments, NINE.
+ indir: string, path to target directories containing netcdf files to cmorize
+ target_var: string, name of variable inside the netcdf file to cmorize
+ local_var: string, value of the variable name in the filename, right before the .nc
+ extension. often identical to target_var but not always.
+ iso_datetime_arr: list of strings, each one a unique ISO datetime string found in targeted
+ netcdf filenames
+ name_of_set: string, representing the post-processing component (GFDL convention) of the
+ targeted files.
+ json_exp_config: see cmor_run_subtool arg desc
+ outdir: string, path to output directory root to move the cmor module output to, including
+ the whole directory structure
+ proj_table_vars: an opened json file object, read from json_table_config
+ json_table_config: see cmor_run_subtool arg desc
+
+ '''
+ print('\n\n-------------------------- START cmorize_target_var_files call -----')
+ print(f"(cmorize_target_var_files) local_var = {local_var} to be used for file-targeting.\n"
+ f" target_var = {target_var} to be used for reading the data \n"
+ " from the file\n"
+ f" outdir = {outdir}")
+
+
+ #determine a tmp dir for working on files.
+ tmp_dir = create_tmp_dir( outdir ) + '/'
+ print(f'(cmorize_target_var_files) will use tmp_dir={tmp_dir}')
+
+
+ # loop over sets of dates, each one pointing to a file
+ nc_fls = {}
+ for i, iso_datetime in enumerate(iso_datetime_arr):
+
+ # why is nc_fls a filled list/array/object thingy here? see above line
+ nc_fls[i] = f"{indir}/{name_of_set}.{iso_datetime}.{local_var}.nc"
+ print(f"(cmorize_target_var_files) input file = {nc_fls[i]}")
+ if not Path(nc_fls[i]).exists():
+ print ("(cmorize_target_var_files) input file(s) not found. Moving on.")
+ continue
+
+
+ # create a copy of the input file with local var name into the work directory
+ nc_file_work = f"{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc"
+
+ print(f"(cmorize_target_var_files) nc_file_work = {nc_file_work}")
+ copy_nc( nc_fls[i], nc_file_work)
+
+ # if the ps file exists, we'll copy it to the work directory too
+ nc_ps_file = nc_fls[i].replace(f'.{local_var}.nc', '.ps.nc')
+ nc_ps_file_work = nc_file_work.replace(f'.{local_var}.nc', '.ps.nc')
+ if Path(nc_ps_file).exists():
+ print(f"(cmorize_target_var_files) nc_ps_file_work = {nc_ps_file_work}")
+ copy_nc(nc_ps_file, nc_ps_file_work)
+
+
+ # TODO think of better way to write this kind of conditional data movement...
+ # now we have a file in our targets, point CMOR to the configs and the input file(s)
+ make_cmor_write_here = None
+ print( Path( tmp_dir ) )
+ print( Path( os.getcwd() ) )
+ if Path( tmp_dir ).is_absolute():
+ print(f'tmp_dir is absolute')
+ make_cmor_write_here = tmp_dir
+ elif Path( tmp_dir ).exists(): # relative to where we are
+ print(f'tmp_dir is relative to CWD!')
+ make_cmor_write_here = os.getcwd() + '/'+tmp_dir # unavoidable, cmor module FORCES write to CWD
+ assert make_cmor_write_here is not None
+
+ gotta_go_back_here=os.getcwd()
+ try:
+ print(f"cd'ing to \n {make_cmor_write_here}" )
+ os.chdir( make_cmor_write_here )
+ except:
+ raise OSError(f'could not chdir to {make_cmor_write_here}')
+
+ print ("(cmorize_target_var_files) calling rewrite_netcdf_file_var")
+ local_file_name = rewrite_netcdf_file_var( proj_table_vars ,
+ local_var ,
+ nc_file_work ,
+ target_var ,
+ json_exp_config ,
+ json_table_config )
+ os.chdir( gotta_go_back_here )
+
+
+ # now that CMOR has rewritten things... we can take our post-rewriting actions
+ # the final output filename will be...
+ print(f'(cmorize_target_var_files) local_file_name={local_file_name}')
+ filename =f"{outdir}/{local_file_name}"
+ print(f"(cmorize_target_var_files) filename = {filename}")
+
+ # the final output file directory will be...
+ filedir = Path(filename).parent
+ print(f"(cmorize_target_var_files) filedir = {filedir}")
+ try:
+ print(f'(cmorize_target_var_files) attempting to create filedir={filedir}')
+ os.makedirs(filedir)
+ except FileExistsError:
+ print(f'(cmorize_target_var_files) WARNING: directory {filedir} already exists!')
+
+ # hmm.... this is making issues for pytest
+ mv_cmd = f"mv {tmp_dir}/{local_file_name} {filedir}"
+ print(f"(cmorize_target_var_files) moving files...\n {mv_cmd}")
+ subprocess.run(mv_cmd, shell = True, check = True)
+
+ # ------ refactor this into function? #TODO
+ # ------ what is the use case for this logic really??
+ filename_no_nc = filename[:filename.rfind(".nc")]
+ chunk_str = filename_no_nc[-6:]
+ if not chunk_str.isdigit():
+ print(f'(cmorize_target_var_files) WARNING: chunk_str is not a digit: '
+ f'chunk_str = {chunk_str}')
+ filename_corr = "{filename[:filename.rfind('.nc')]}_{iso_datetime}.nc"
+ mv_cmd = f"mv {filename} {filename_corr}"
+ print(f"(cmorize_target_var_files) moving files, strange chunkstr logic...\n {mv_cmd}")
+ subprocess.run(mv_cmd, shell = True, check = True)
+ # ------ end refactor this into function?
+
+ # delete files in work dirs
+ if Path(nc_file_work).exists():
+ Path(nc_file_work).unlink()
+
+ if Path(nc_ps_file_work).exists():
+ Path(nc_ps_file_work).unlink()
+
+ if DEBUG_MODE_RUN_ONE:
+ print(f'WARNING: DEBUG_MODE_RUN_ONE is True!!!!')
+ print(f'WARNING: done processing one file!!!')
+ break
+
+
+
+
+def cmor_run_subtool( indir = None,
+ json_var_list = None,
+ json_table_config = None,
+ json_exp_config = None ,
+ outdir = None, opt_var_name = None
+ ):
+ '''
+ primary steering function for the cmor_mixer tool, i.e essentially main. Accepts six args:
+ indir: string, directory containing netCDF files. keys specified in json_var_list are local
+ variable names used for targeting specific files
+ json_var_list: string, path pointing to a json file containing directory of key/value
+ pairs. the keys are the "local" names used in the filename, and the
+ values pointed to by those keys are strings representing the name of the
+ variable contained in targeted files. the key and value are often the same,
+ but it is not required.
+ json_table_config: json file containing CMIP-compliant per-variable/metadata for specific
+ MIP table. The MIP table can generally be identified by the specific
+ filename (e.g. "Omon")
+ json_exp_config: json file containing metadata dictionary for CMORization. this metadata is effectively
+ appended to the final output file's header
+ outdir: string, directory root that will contain the full output and output directory
+ structure generated by the cmor module upon request.
+ opt_var_name: string, optional, specify a variable name to specifically process only filenames matching
+ that variable name. I.e., this string help target local_vars, not target_vars.
+ '''
+ # check req'd inputs
+ if None in [indir, json_var_list, json_table_config, json_exp_config, outdir]:
+ raise ValueError(f'(cmor_run_subtool) all input arguments except opt_var_name are required!\n'
+ ' [indir, json_var_list, json_table_config, json_exp_config, outdir] = \n'
+ f' [{indir}, {json_var_list}, {json_table_config}, '
+ ' {json_exp_config}, {outdir}]' )
+
+ # open CMOR table config file
+ print( '(cmor_run_subtool) getting table variables from json_table_config = \n'
+ f' {json_table_config}' )
+ try:
+ with open( json_table_config, "r", encoding = "utf-8") as table_config_file:
+ proj_table_vars=json.load(table_config_file)
+
+ except Exception as exc:
+ raise FileNotFoundError(
+ f'ERROR: json_table_config file cannot be opened.\n'
+ f' json_table_config = {json_table_config}' ) from exc
+
+ # now resolve the json_table_config path after confirming it can be open
+ json_table_config= str( Path(json_table_config).resolve() )
+
+ # open input variable list
+ print('(cmor_run_subtool) opening variable list json_var_list')
+ try:
+ with open( json_var_list, "r", encoding = "utf-8" ) as var_list_file:
+ var_list = json.load( var_list_file )
+
+ except Exception as exc:
+ raise FileNotFoundError(
+ f'ERROR: json_var_list file cannot be opened.\n'
+ f' json_var_list = {json_var_list}' ) from exc
+
+ # make sure the exp config exists too while we're at it...
+ if Path(json_exp_config).exists(): # if so, resolve to absolute path
+ json_exp_config = str( Path( json_exp_config).resolve() )
+ else:
+ raise FileNotFoundError(
+ f'ERROR: json_exp_config file cannot be opened.\n'
+ f' json_exp_config = {json_exp_config}' )
+
+ # loop over entries in the json_var_list, read into var_list
+ for local_var in var_list:
+
+ # if its not in the table configurations variable_entry list, skip
+ if var_list[local_var] not in proj_table_vars["variable_entry"]:
+ print(f"(cmor_run_subtool) WARNING: skipping local_var = {local_var} /\n"
+ f" target_var = {var_list[local_var]}\n"
+ " ... target_var not found in CMOR variable group")
+ continue
+
+ if all( [ opt_var_name is not None,
+ local_var != opt_var_name ] ):
+ print(f'(cmor_run_subtool) WARNING: skipping local_var={local_var} as it is not equal\n'
+ ' to the opt_var_name argument.')
+ continue
+
+ # it is in there, get the name of the data inside the netcdf file.
+ target_var=var_list[local_var] # often equiv to local_var but not necessarily.
+ if local_var != target_var:
+ print(f'(cmor_run_subtool) WARNING: local_var == {local_var} \n'
+ f' != {target_var} == target_var\n'
+ f' i am expecting {local_var} to be in the filename, and i expect the variable\n'
+ f' in that file to be {target_var}')
+
+
+ # examine input directory to obtain a list of input file targets
+ var_filenames = []
+ get_var_filenames(indir, var_filenames, local_var)
+ print(f"(cmor_run_subtool) found filenames = \n {var_filenames}")
+
+ # examine input files to obtain target date ranges
+ iso_datetime_arr = []
+ get_iso_datetimes(var_filenames, iso_datetime_arr)
+ print(f"(cmor_run_subtool) found iso datetimes = \n {iso_datetime_arr}")
+
+ # name_of_set == component label...
+ # which is not relevant for CMOR/CMIP... or is it?
+ name_of_set = var_filenames[0].split(".")[0]
+ print(f"(cmor_run_subtool) setting name_of_set = {name_of_set}")
+
+
+
+ print(f'(cmor_run_subtool) ..............beginning CMORization for {local_var}/\n'
+ f' {target_var}..........')
+ cmorize_target_var_files(
+ indir, target_var, local_var, iso_datetime_arr, # OK
+ name_of_set, json_exp_config,
+ outdir,
+ proj_table_vars, json_table_config # a little redundant
+ )
+
+ if DEBUG_MODE_RUN_ONE:
+ print(f'WARNING: DEBUG_MODE_RUN_ONE is True. breaking var_list loop')
+ break
+ return 0
+
+
+@click.command()
+def _cmor_run_subtool(indir = None,
+ json_var_list = None, json_table_config = None, json_exp_config = None,
+ outdir = None, opt_var_name = None):
+ ''' entry point to fre cmor run for click. see cmor_run_subtool for argument descriptions.'''
+ return cmor_run_subtool(indir, json_var_list, json_table_config, json_exp_config, outdir, opt_var_name)
+
+
+if __name__ == '__main__':
+ cmor_run_subtool()
diff --git a/lib/fre/cmor/frecmor.py b/lib/fre/cmor/frecmor.py
new file mode 100644
index 00000000..e882186a
--- /dev/null
+++ b/lib/fre/cmor/frecmor.py
@@ -0,0 +1,66 @@
+''' fre cmor '''
+
+import click
+
+from .cmor_mixer import _cmor_run_subtool
+
+@click.group(help=click.style(" - access fre cmor subcommands", fg=(232,91,204)))
+def cmor_cli():
+ ''' entry point to fre cmor click commands '''
+
+@cmor_cli.command()
+@click.option("-d", "--indir",
+ type=str,
+ help="directory containing netCDF files. keys specified in json_var_list are local " + \
+ "variable names used for targeting specific files in this directory",
+ required=True)
+@click.option("-l", "--varlist",
+ type=str,
+ help="path pointing to a json file containing directory of key/value pairs. " + \
+ "the keys are the \'local\' names used in the filename, and the values " + \
+ "pointed to by those keys are strings representing the name of the variable " + \
+ "contained in targeted files. the key and value are often the same, " + \
+ "but it is not required.",
+ required=True)
+@click.option("-r", "--table_config",
+ type=str,
+ help="json file containing CMIP-compliant per-variable/metadata for specific " + \
+ "MIP table. The MIP table can generally be identified by the specific " + \
+ "filename (e.g. \'Omon\')",
+ required=True)
+@click.option("-p", "--exp_config",
+ type=str,
+ help="json file containing metadata dictionary for CMORization. this metadata is " + \
+ "effectively appended to the final output file's header",
+ required=True)
+@click.option("-o", "--outdir",
+ type=str,
+ help="directory root that will contain the full output and output directory " + \
+ "structure generated by the cmor module upon request.",
+ required=True)
+@click.option('-v', "--opt_var_name",
+ type = str,
+ help="optional, specify a variable name to specifically process only filenames " + \
+ "matching that variable name. I.e., this string help target local_vars, not " + \
+ "target_vars.",
+ required=False)
+@click.pass_context
+def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name):
+ # pylint: disable=unused-argument
+ """
+ Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing
+ """
+ context.invoke(
+ _cmor_run_subtool,
+ indir = indir,
+ json_var_list = varlist,
+ json_table_config = table_config,
+ json_exp_config = exp_config,
+ outdir = outdir,
+ opt_var_name = opt_var_name
+ )
+ # context.forward(
+ # _cmor_run_subtool() )
+
+if __name__ == "__main__":
+ cmor_cli()
diff --git a/lib/fre/cmor/tests/__init__.py b/lib/fre/cmor/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/cmor/tests/test_cmor_run_subtool.py b/lib/fre/cmor/tests/test_cmor_run_subtool.py
new file mode 100644
index 00000000..043b5705
--- /dev/null
+++ b/lib/fre/cmor/tests/test_cmor_run_subtool.py
@@ -0,0 +1,249 @@
+''' tests for fre.cmor.cmor_run_subtool '''
+import subprocess
+import shutil
+from pathlib import Path
+from datetime import date
+
+import git
+
+import fre
+
+# where are we? we're running pytest from the base directory of this repo
+ROOTDIR = 'fre/tests/test_files'
+
+# setup- cmip/cmor variable table(s)
+CLONE_CMIP_TABLE_URL = \
+ 'https://github.com/PCMDI/cmip6-cmor-tables.git'
+CLONE_REPO_PATH = \
+ f'{ROOTDIR}/cmip6-cmor-tables'
+TABLE_CONFIG = \
+ f'{CLONE_REPO_PATH}/Tables/CMIP6_Omon.json'
+
+def test_setup_cmor_cmip_table_repo():
+ ''' setup routine, if it doesnt exist, clone the repo holding CMOR/CMIP6 tables '''
+ if Path(TABLE_CONFIG).exists():
+ pass
+ else:
+ git.Repo.clone_from(
+ CLONE_CMIP_TABLE_URL,
+ CLONE_REPO_PATH )
+ assert Path(TABLE_CONFIG).exists()
+
+# explicit inputs to tool
+INDIR = f'{ROOTDIR}/ocean_sos_var_file'
+VARLIST = f'{ROOTDIR}/varlist'
+EXP_CONFIG = f'{ROOTDIR}/CMOR_input_example.json'
+OUTDIR = f'{ROOTDIR}/outdir'
+TMPDIR = f'{OUTDIR}/tmp'
+
+# determined by cmor_run_subtool
+YYYYMMDD = date.today().strftime('%Y%m%d')
+CMOR_CREATES_DIR = \
+ 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Omon/sos/gn'
+FULL_OUTPUTDIR = \
+ f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}"
+FULL_OUTPUTFILE = \
+f"{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199807.nc"
+
+# FYI but helpful for tests
+FILENAME = 'ocean_monthly_1x1deg.199301-199712.sos.nc' # unneeded, this is mostly for reference
+FULL_INPUTFILE=f"{INDIR}/{FILENAME}"
+
+def test_setup_fre_cmor_run_subtool(capfd):
+ ''' checks for outputfile from prev pytest runs, removes it if it's present.
+ this routine also checks to make sure the desired input file is present'''
+ if Path(FULL_OUTPUTFILE).exists():
+ Path(FULL_OUTPUTFILE).unlink()
+ if Path(OUTDIR).exists():
+ shutil.rmtree(OUTDIR)
+ assert not any ( [ Path(FULL_OUTPUTFILE).exists(),
+ Path(OUTDIR).exists() ] )
+ assert Path(FULL_INPUTFILE).exists()
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case1(capfd):
+ ''' fre cmor run, test-use case '''
+
+ #debug
+ #print(
+ # f"fre.cmor.cmor_run_subtool("
+ # f"\'{INDIR}\',"
+ # f"\'{VARLIST}\',"
+ # f"\'{TABLE_CONFIG}\',"
+ # f"\'{EXP_CONFIG}\',"
+ # f"\'{OUTDIR}\'"
+ # ")"
+ #)
+
+ # test call, where meat of the workload gets done
+ fre.cmor.cmor_run_subtool(
+ indir = INDIR,
+ json_var_list = VARLIST,
+ json_table_config = TABLE_CONFIG,
+ json_exp_config = EXP_CONFIG,
+ outdir = OUTDIR
+ )
+
+ assert all( [ Path(FULL_OUTPUTFILE).exists(),
+ Path(FULL_INPUTFILE).exists() ] )
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case1_output_compare_data(capfd):
+ ''' I/O data-only comparison of test case1 '''
+ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}')
+ print(f'FULL_INPUTFILE={FULL_INPUTFILE}')
+
+ nccmp_cmd= [ "nccmp", "-f", "-d",
+ f"{FULL_INPUTFILE}",
+ f"{FULL_OUTPUTFILE}" ]
+ print(f"via subprocess, running {' '.join(nccmp_cmd)}")
+ result = subprocess.run( ' '.join(nccmp_cmd),
+ shell=True,
+ check=False,
+ capture_output=True
+ )
+ # err_list has length two if end in newline
+ err_list = result.stderr.decode().split('\n')
+ expected_err = \
+ "DIFFER : FILE FORMATS : NC_FORMAT_64BIT <> NC_FORMAT_NETCDF4_CLASSIC"
+ assert all( [result.returncode == 1,
+ len(err_list)==2,
+ '' in err_list,
+ expected_err in err_list ] )
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd):
+ ''' I/O metadata-only comparison of test case1 '''
+ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}')
+ print(f'FULL_INPUTFILE={FULL_INPUTFILE}')
+
+ nccmp_cmd= [ "nccmp", "-f", "-m", "-g",
+ f"{FULL_INPUTFILE}",
+ f"{FULL_OUTPUTFILE}" ]
+ print(f"via subprocess, running {' '.join(nccmp_cmd)}")
+ result = subprocess.run( ' '.join(nccmp_cmd),
+ shell=True,
+ check=False
+ )
+
+ assert result.returncode == 1
+ _out, _err = capfd.readouterr()
+
+
+# FYI, but again, helpful for tests
+FILENAME_DIFF = \
+ 'ocean_monthly_1x1deg.199301-199712.sosV2.nc'
+FULL_INPUTFILE_DIFF = \
+ f"{INDIR}/{FILENAME_DIFF}"
+VARLIST_DIFF = \
+ f'{ROOTDIR}/varlist_local_target_vars_differ'
+def test_setup_fre_cmor_run_subtool_case2(capfd):
+ ''' make a copy of the input file to the slightly different name.
+ checks for outputfile from prev pytest runs, removes it if it's present.
+ this routine also checks to make sure the desired input file is present'''
+ if Path(FULL_OUTPUTFILE).exists():
+ Path(FULL_OUTPUTFILE).unlink()
+ assert not Path(FULL_OUTPUTFILE).exists()
+
+ if Path(OUTDIR+'/CMIP6').exists():
+ shutil.rmtree(OUTDIR+'/CMIP6')
+ assert not Path(OUTDIR+'/CMIP6').exists()
+
+
+ # VERY ANNOYING !!! FYI WARNING TODO
+ if Path(TMPDIR).exists():
+ try:
+ shutil.rmtree(TMPDIR)
+ except OSError as exc:
+ print(f'WARNING: TMPDIR={TMPDIR} could not be removed.')
+ print( ' this does not matter that much, but is unfortunate.')
+ print( ' supicion: something the cmor module is using is not being closed')
+
+ #assert not Path(TMPDIR).exists() # VERY ANNOYING !!! FYI WARNING TODO
+
+ # VERY ANNOYING !!! FYI WARNING TODO
+ if Path(OUTDIR).exists():
+ try:
+ shutil.rmtree(OUTDIR)
+ except OSError as exc:
+ print(f'WARNING: OUTDIR={OUTDIR} could not be removed.')
+ print( ' this does not matter that much, but is unfortunate.')
+ print( ' supicion: something the cmor module is using is not being closed')
+
+ #assert not Path(OUTDIR).exists() # VERY ANNOYING !!! FYI WARNING TODO
+
+ # make a copy of the usual test file.
+ if not Path(FULL_INPUTFILE_DIFF).exists():
+ shutil.copy(
+ Path(FULL_INPUTFILE),
+ Path(FULL_INPUTFILE_DIFF) )
+ assert Path(FULL_INPUTFILE_DIFF).exists()
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case2(capfd):
+ ''' fre cmor run, test-use case2 '''
+
+ #debug
+ #print(
+ # f"fre.cmor.cmor_run_subtool("
+ # f"\'{INDIR}\',"
+ # f"\'{VARLIST_DIFF}\',"
+ # f"\'{TABLE_CONFIG}\',"
+ # f"\'{EXP_CONFIG}\',"
+ # f"\'{OUTDIR}\'"
+ # ")"
+ #)
+
+ # test call, where meat of the workload gets done
+ fre.cmor.cmor_run_subtool(
+ indir = INDIR,
+ json_var_list = VARLIST_DIFF,
+ json_table_config = TABLE_CONFIG,
+ json_exp_config = EXP_CONFIG,
+ outdir = OUTDIR
+ )
+
+ # check we ran on the right input file.
+ assert all( [ Path(FULL_OUTPUTFILE).exists(),
+ Path(FULL_INPUTFILE_DIFF).exists() ] )
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case2_output_compare_data(capfd):
+ ''' I/O data-only comparison of test case2 '''
+ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}')
+ print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}')
+
+ nccmp_cmd= [ "nccmp", "-f", "-d",
+ f"{FULL_INPUTFILE_DIFF}",
+ f"{FULL_OUTPUTFILE}" ]
+ print(f"via subprocess, running {' '.join(nccmp_cmd)}")
+ result = subprocess.run( ' '.join(nccmp_cmd),
+ shell=True,
+ check=False,
+ capture_output=True
+ )
+
+ err_list = result.stderr.decode().split('\n')#length two if end in newline
+ expected_err="DIFFER : FILE FORMATS : NC_FORMAT_64BIT <> NC_FORMAT_NETCDF4_CLASSIC"
+ assert all( [result.returncode == 1,
+ len(err_list)==2,
+ '' in err_list,
+ expected_err in err_list ] )
+ _out, _err = capfd.readouterr()
+
+def test_fre_cmor_run_subtool_case2_output_compare_metadata(capfd):
+ ''' I/O metadata-only comparison of test case2 '''
+ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}')
+ print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}')
+
+ nccmp_cmd= [ "nccmp", "-f", "-m", "-g",
+ f"{FULL_INPUTFILE_DIFF}",
+ f"{FULL_OUTPUTFILE}" ]
+ print(f"via subprocess, running {' '.join(nccmp_cmd)}")
+ result = subprocess.run( ' '.join(nccmp_cmd),
+ shell=True,
+ check=False
+ )
+
+ assert result.returncode == 1
+ _out, _err = capfd.readouterr()
diff --git a/lib/fre/coveragerc b/lib/fre/coveragerc
new file mode 100644
index 00000000..f95c3c41
--- /dev/null
+++ b/lib/fre/coveragerc
@@ -0,0 +1,3 @@
+[run]
+omit =
+ */test_*py
diff --git a/lib/fre/fre.py b/lib/fre/fre.py
new file mode 100644
index 00000000..9dd65eb0
--- /dev/null
+++ b/lib/fre/fre.py
@@ -0,0 +1,41 @@
+"""
+Main host file for FRE-CLI program scripts
+authored by Bennett.Chang@noaa.gov | bcc2761
+NOAA | GFDL
+2023-2024
+principal click group for main/fre allows for subgroup functions to
+be called via this script. I.e. 'fre' is the entry point
+"""
+
+import click
+from .lazy_group import LazyGroup
+
+@click.group(
+ cls = LazyGroup,
+ lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+ "catalog": ".catalog.frecatalog.catalog_cli",
+ "list": ".list.frelist.list_cli",
+ "check": ".check.frecheck.check_cli",
+ "run": ".run.frerun.run_cli",
+ "test": ".test.fretest.test_cli",
+ "yamltools": ".yamltools.freyamltools.yamltools_cli",
+ "make": ".make.fremake.make_cli",
+ "app": ".app.freapp.app_cli",
+ "cmor": ".cmor.frecmor.cmor_cli" },
+ help = click.style(
+ "'fre' is the main CLI click group that houses the other tool groups as lazy subcommands.",
+ fg='cyan')
+)
+
+@click.version_option(
+ package_name = "fre-cli",
+ message = click.style("%(package)s | %(version)s",
+ fg = (155,255,172) )
+)
+
+def fre():
+ ''' entry point function to subgroup functions '''
+
+
+if __name__ == '__main__':
+ fre()
diff --git a/lib/fre/gfdl_msd_schemas/.github/CODEOWNERS b/lib/fre/gfdl_msd_schemas/.github/CODEOWNERS
new file mode 100644
index 00000000..22c95973
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/.github/CODEOWNERS
@@ -0,0 +1,4 @@
+* @ceblanton @J-Lentz
+FMS/diag_table.yaml @uramirez8707
+FMS/data_table.yaml @J-Lentz
+FMS/field_table.yaml @J-Lentz
diff --git a/lib/fre/gfdl_msd_schemas/FMS/data_table.json b/lib/fre/gfdl_msd_schemas/FMS/data_table.json
new file mode 100644
index 00000000..f0683374
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/FMS/data_table.json
@@ -0,0 +1,114 @@
+{
+ "title": "Data table schema for data_override",
+ "type": "object",
+ "properties": {
+ "data_table": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "grid_name": {
+ "type": "string",
+ "enum": ["OCN", "LND", "ATM", "ICE"]
+ },
+ "fieldname_in_model": {
+ "type": "string",
+ "minLength": 1
+ },
+ "override_file": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "fieldname_in_file": {
+ "type": "string",
+ "minLength": 1
+ },
+ "interp_method": {
+ "type": "string",
+ "enum": ["bilinear", "bicubic", "none"]
+ },
+ "multi_file": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "next_file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "prev_file_name": {
+ "type": "string",
+ "minLength": 1
+ }
+ },
+ "required": ["next_file_name", "prev_file_name"],
+ "additionalProperties": false
+ }
+ },
+ "external_weights": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "source": {
+ "type": "string",
+ "minLength": 1,
+ "enum": ["fregrid"]
+ }
+ },
+ "required": ["file_name", "source"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required": ["file_name", "fieldname_in_file", "interp_method"],
+ "additionalProperties": false
+ }
+ },
+ "factor": {
+ "type": "number"
+ },
+ "subregion": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": ["inside_region", "outside_region"]
+ },
+ "lon_start": {
+ "type": "number"
+ },
+ "lon_end": {
+ "type": "number"
+ },
+ "lat_start": {
+ "type": "number"
+ },
+ "lat_end": {
+ "type": "number"
+ }
+ },
+ "required": ["type", "lon_start", "lon_end", "lat_start", "lat_end"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required": ["grid_name", "fieldname_in_model", "factor"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required": ["data_table"],
+ "additionalProperties": false
+}
diff --git a/lib/fre/gfdl_msd_schemas/FMS/diag_table.json b/lib/fre/gfdl_msd_schemas/FMS/diag_table.json
new file mode 100644
index 00000000..a968ca6b
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/FMS/diag_table.json
@@ -0,0 +1,250 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "required": ["title", "base_date"],
+ "additionalProperties": false,
+ "properties": {
+ "title": {
+ "type": "string",
+ "minLength": 1
+ },
+ "base_date": {
+ "type": "string",
+ "anyOf": [
+ {"enum": ["$baseDate"]},
+ {"pattern": "^([0-9]*[1-9][0-9]* ){3}([0-9]+ ){2}[0-9]+$"}
+ ]
+ },
+ "diag_files": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["file_name", "freq", "time_units", "unlimdim"],
+ "additionalProperties": false,
+ "properties": {
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "freq": {
+ "anyOf": [
+ {"type": "string"},
+ {"type": "number"}
+ ],
+ "pattern": "^(-1|0|((-1|[0-9]+) +(seconds|minutes|hours|days|months|years)( *, *)?)+)$"
+ },
+ "time_units": {
+ "type": "string",
+ "enum": ["seconds", "minutes", "hours", "days", "months", "years"]
+ },
+ "unlimdim": {
+ "type": "string",
+ "minLength": 1
+ },
+ "write_file": {
+ "type": "boolean"
+ },
+ "global_meta": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": {
+ "oneOf": [
+ {"type": "string"},
+ {"type": "number"},
+ {"type": "boolean"}
+ ]
+ }
+ }
+ },
+ "sub_region": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 1,
+ "required": ["grid_type", "corner1", "corner2", "corner3", "corner4"],
+ "properties": {
+ "grid_type": {
+ "type": "string",
+ "enum": ["indices", "latlon"]
+ },
+ "corner1": {
+ "type": "array",
+ "minItems": 2,
+ "maxItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "corner2": {
+ "type": "array",
+ "minItems": 2,
+ "maxItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "corner3": {
+ "type": "array",
+ "minItems": 2,
+ "maxItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "corner4": {
+ "type": "array",
+ "minItems": 2,
+ "maxItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "tile": {
+ "type": "number"
+ }
+ }
+ },
+ "new_file_freq": {
+ "type": "string",
+ "pattern": "^(0*[1-9][0-9]* (seconds|minutes|hours|days|months|years), )*0*[1-9][0-9]* (seconds|minutes|hours|days|months|years)$"
+ },
+ "start_time": {
+ "type": "array",
+ "minItems": 6,
+ "maxItems": 6,
+ "items": {
+ "type": "integer",
+ "minimum": 0
+ }
+ },
+ "file_duration": {
+ "type": "string",
+ "pattern": "^(0*[1-9][0-9]* (seconds|minutes|hours|days|months|years), )*0*[1-9][0-9]* (seconds|minutes|hours|days|months|years)$"
+ },
+ "is_ocean": {
+ "type": "boolean"
+ },
+ "kind": {
+ "type": "string",
+ "enum": ["r4", "r8", "i4", "i8"]
+ },
+ "module": {
+ "type": "string",
+ "minLength": 1
+ },
+ "reduction": {
+ "type": "string",
+ "pattern": "^average$|^min$|^max$|^none$|^rms$|^sum$|^diurnal[1-9]+|^pow[1-9]+"
+ },
+ "varlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["var_name"],
+ "additionalProperties": false,
+ "properties": {
+ "kind": {
+ "type": "string",
+ "enum": ["r4", "r8", "i4", "i8"]
+ },
+ "module": {
+ "type": "string",
+ "minLength": 1
+ },
+ "reduction": {
+ "type": "string",
+ "pattern": "^average$|^min$|^max$|^none$|^rms$|^sum$|^diurnal[1-9]+|^pow[1-9]+"
+ },
+ "var_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "write_var": {
+ "type": "boolean"
+ },
+ "output_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "long_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "attributes": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": {
+ "oneOf": [
+ {"type": "string"},
+ {"type": "number"},
+ {"type": "boolean"}
+ ]
+ }
+ }
+ },
+ "zbounds": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "allOf": [
+ {
+ "if": {
+ "required": ["kind"]
+ },
+ "else": {
+ "properties": {
+ "varlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["kind"]
+ }
+ }
+ }
+ }
+ },
+ {
+ "if": {
+ "required": ["module"]
+ },
+ "else": {
+ "properties": {
+ "varlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["module"]
+ }
+ }
+ }
+ }
+ },
+ {
+ "if": {
+ "required": ["reduction"]
+ },
+ "else": {
+ "properties": {
+ "varlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["reduction"]
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/lib/fre/gfdl_msd_schemas/FMS/field_table.json b/lib/fre/gfdl_msd_schemas/FMS/field_table.json
new file mode 100644
index 00000000..64e069cb
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/FMS/field_table.json
@@ -0,0 +1,48 @@
+{
+ "title": "Field table for field_manager",
+ "type": "object",
+ "properties": {
+ "field_table": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "field_type": {
+ "type": "string"
+ },
+ "modlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "model_type": {
+ "type": "string",
+ "enum": ["coupler_mod", "atmos_mod", "ocean_mod", "land_mod", "ice_mod"]
+ },
+ "varlist": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "variable": {
+ "type": "string"
+ }
+ },
+ "required": ["variable"],
+ "additionalProperties": true
+ }
+ }
+ },
+ "required": ["model_type"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required": ["field_type"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required": ["field_table"],
+ "additionalProperties": false
+}
diff --git a/lib/fre/gfdl_msd_schemas/FRE/fre_make.json b/lib/fre/gfdl_msd_schemas/FRE/fre_make.json
new file mode 100644
index 00000000..44a153f4
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/FRE/fre_make.json
@@ -0,0 +1,212 @@
+{
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "name": {
+ "description": "The name of the experiment",
+ "type": "string"
+ },
+ "platform": {
+ "description": "The platforms listed in the command",
+ "type": "string"
+ },
+ "target": {
+ "description": "The targets listed in the command",
+ "type": "string"
+ },
+ "build": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "compileYaml": {
+ "description": "Path to the compile yaml.",
+ "type": "string"
+ },
+ "platformYaml": {
+ "description": "Path to the platform yaml.",
+ "type": "string"
+ }
+ }
+ },
+ "compile": {
+ "description": "The source code descriptions",
+ "$ref": "#/$defs/Compile"
+ },
+ "platforms": {
+ "description": "FRE platforms",
+ "type": "array",
+ "items": {"$ref": "#/$defs/Platform"}
+ }
+ },
+ "$defs": {
+ "Compile": {
+ "type": "object",
+ "properties": {
+ "experiment": {
+ "description": "The name of the model",
+ "type": "string"
+ },
+ "container_addlibs": {
+ "description": "Libraries and packages needed for linking in the container",
+ "oneOf": [
+ {"type": "null"},
+ {
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ ]
+ },
+ "baremetal_linkerflags": {
+ "description": "Linker flags of libraries and packages needed for linking in the bare-metal build",
+ "oneOf": [
+ {"type": "null"},
+ {
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ ]
+ },
+ "src": {
+ "type": "array",
+ "items": {"$ref": "#/$defs/Src"}
+ }
+ }
+ },
+ "Src": {
+ "type": "object",
+ "properties": {
+ "component": {
+ "description": "The name of the model component",
+ "type": "string"
+ },
+ "repo": {
+ "anyOf": [
+ {
+ "description": "The URL of the code repository",
+ "type": "array",
+ "items": {
+ "type": "string",
+ "format": "uri",
+ "qt-uri-protocols": [
+ "https"
+ ],
+ "qt-uri-extensions": [
+ ".git"
+ ]
+ }
+ },
+ {
+ "description": "The URL of the code repository",
+ "type": "string",
+ "format": "uri",
+ "qt-uri-protocols": [
+ "https"
+ ],
+ "qt-uri-extensions": [
+ ".git"
+ ]
+ }
+ ]
+ },
+ "cppdefs": {
+ "description": "String of CPPDEFs to include in compiling the component",
+ "type": "string"
+ },
+ "branch": {
+ "anyOf": [
+ {
+ "description": "The version of code to clone",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ {
+ "description": "The version of code to clone",
+ "type": "string"
+ }
+ ]
+ },
+ "otherFlags": {
+ "description": "String of Include flags necessary to retrieve other code needed",
+ "type": "string"
+ },
+ "requires": {
+ "description": "list of componets that this component depends on",
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "paths": {
+ "description": "A list of the paths in the component to compile",
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "doF90Cpp": {
+ "description": "True if the preprocessor needs to be run",
+ "type": "boolean"
+ },
+ "makeOverrides": {
+ "description": "Overrides openmp target for MOM6",
+ "type": "string"
+ }
+ }
+ },
+ "Platform": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "description": "The name of the platform",
+ "type": "string"
+ },
+ "compiler": {
+ "description": "The compiler used to build the model",
+ "type": "string"
+ },
+ "modulesInit": {
+ "description": "Array of commands to run before loading modules",
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "modules": {
+ "description": "List (array) of modules to load",
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "fc": {
+ "description": "The Fortran compiler",
+ "type": "string"
+ },
+ "cc": {
+ "description": "The C compiler",
+ "type": "string"
+ },
+ "mkTemplate": {
+ "description": "Path to the mk template file",
+ "type": "string"
+ },
+ "modelRoot": {
+ "description": "Path to the root for all model install files",
+ "type": "string"
+ },
+ "RUNenv": {
+ "description": "Commands needed at the beginning of a RUN in dockerfile",
+ "type": ["array"],
+ "items": {"type": "string"}
+ },
+ "container": {
+ "description": "True/False if using container to compile",
+ "type": "boolean"
+ },
+ "containerBuild": {
+ "description": "Program used to build the container",
+ "type": "string"
+ },
+ "containerRun": {
+ "description": "Program used to run the container",
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/lib/fre/gfdl_msd_schemas/README.md b/lib/fre/gfdl_msd_schemas/README.md
new file mode 100644
index 00000000..7fd70f3a
--- /dev/null
+++ b/lib/fre/gfdl_msd_schemas/README.md
@@ -0,0 +1,26 @@
+# About
+This repository contains schemas for various types of YAML files that are used
+by FMS and FRE at GFDL. These schemas are written in the
+[JSON Schema](https://json-schema.org/) format, and are intended for use with
+the `validate_schema.py` tool in
+[fms_yaml_tools](https://github.com/NOAA-GFDL/fms_yaml_tools).
+
+# Inventory
+* `FMS/data_table.json`
+* `FMS/diag_table.json`
+* `FMS/field_table.json`
+
+# Disclaimer
+
+The United States Department of Commerce (DOC) GitHub project code is provided
+on an 'as is' basis and the user assumes responsibility for its use. DOC has
+relinquished control of the information and no longer has responsibility to
+protect the integrity, confidentiality, or availability of the information. Any
+claims against the Department of Commerce stemming from the use of its GitHub
+project will be governed by all applicable Federal law. Any reference to
+specific commercial products, processes, or services by service mark,
+trademark, manufacturer, or otherwise, does not constitute or imply their
+endorsement, recommendation or favoring by the Department of Commerce. The
+Department of Commerce seal and logo, or the seal and logo of a DOC bureau,
+shall not be used in any manner to imply endorsement of any commercial product
+or activity by DOC or the United States Government.
diff --git a/lib/fre/lazy_group.py b/lib/fre/lazy_group.py
new file mode 100644
index 00000000..1ceefca7
--- /dev/null
+++ b/lib/fre/lazy_group.py
@@ -0,0 +1,43 @@
+'''
+for lazy-style loading of commands and subcommands while using click
+see https://click.palletsprojects.com/en/8.1.x/complex/
+'''
+
+import importlib
+import click
+
+class LazyGroup(click.Group):
+ ''' class defining lazygroup command/subcommand loading '''
+ def __init__(self, *args, lazy_subcommands=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ # lazy_subcommands is a map of the form:
+ #
+ # {command-name} -> {module-name}.{command-object-name}
+ #
+ self.lazy_subcommands = lazy_subcommands or {}
+
+ def list_commands(self, ctx):
+ base = super().list_commands(ctx)
+ lazy = sorted(self.lazy_subcommands.keys())
+ return base + lazy
+
+ def get_command(self, ctx, cmd_name):
+ if cmd_name in self.lazy_subcommands:
+ return self._lazy_load(cmd_name)
+ return super().get_command(ctx, cmd_name)
+
+ def _lazy_load(self, cmd_name):
+ # lazily loading a command, first get the module name and attribute name
+ import_path = self.lazy_subcommands[cmd_name]
+ modname, cmd_object_name = import_path.rsplit(".", 1)
+ # do the import
+ mod = importlib.import_module(modname, package="fre")
+ # get the Command object from that module
+ cmd_object = getattr(mod, cmd_object_name)
+ # check the result to make debugging easier
+ if not isinstance(cmd_object, click.BaseCommand):
+ raise ValueError(
+ f"Lazy loading of {import_path} failed by returning "
+ "a non-command object"
+ )
+ return cmd_object
diff --git a/lib/fre/list/__init__.py b/lib/fre/list/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/list/frelist.py b/lib/fre/list/frelist.py
new file mode 100644
index 00000000..fea3b6ea
--- /dev/null
+++ b/lib/fre/list/frelist.py
@@ -0,0 +1,20 @@
+''' fre list '''
+
+import click
+
+from .frelistexample import list_test_function
+
+@click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91)))
+def list_cli():
+ ''' entry point to fre list click commands '''
+
+@list_cli.command()
+@click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
+@click.pass_context
+def function(context, uppercase):
+ # pylint: disable=unused-argument
+ """ - Execute fre list test """
+ context.forward(list_test_function)
+
+if __name__ == "__main__":
+ list_cli()
diff --git a/lib/fre/list/frelistexample.py b/lib/fre/list/frelistexample.py
new file mode 100644
index 00000000..7e40e195
--- /dev/null
+++ b/lib/fre/list/frelistexample.py
@@ -0,0 +1,18 @@
+"""
+experimentation file for integrating one file's functions into main prototype fre file
+authored by Bennett.Chang@noaa.gov | bcc2761
+NOAA | GFDL
+"""
+
+import click
+
+@click.command()
+def list_test_function(uppercase=None):
+ """Execute fre list testfunction2."""
+ statement = "testingtestingtestingtesting"
+ if uppercase:
+ statement = statement.upper()
+ click.echo(statement)
+
+if __name__ == '__main__':
+ list_test_function()
diff --git a/lib/fre/make/README.md b/lib/fre/make/README.md
new file mode 100644
index 00000000..961e564c
--- /dev/null
+++ b/lib/fre/make/README.md
@@ -0,0 +1,180 @@
+# **Fremake Canopy**
+Through the fre-cli, `fre make` can be used to create and run a checkout script, makefile, and compile a model.
+
+* Fremake Canopy Supports:
+ - multiple targets; use `-t` flag to define each target
+ - bare-metal build
+ - container creation
+ - parallel checkouts for bare-metal build**
+
+** **Note: Users will not be able to create containers without access to podman**
+
+The fremake canopy fre-cli subcommands are described below ([Subtools](#subtools)), as well as a Guide on the order in which to use them ([Guide](#guide)).
+
+Additionally, as mentioned, multiple targets can be used more multiple target-platform combinations. Below is an example of this usage for both the bare-metal build and container build, using the AM5 model
+
+- [Bare-metal Example](#bare-metal-build)
+- [Bare-metal Multi-target Example](#bare-metal-build-multi-target)
+- [Container Example](#container-build)
+
+## **Usage (Users)**
+* Refer to fre-cli [README.md](https://github.com/NOAA-GFDL/fre-cli/blob/main/README.md) for foundational fre-cli usage guide and tips.
+
+## **Quickstart**
+### **Bare-metal Build:**
+```bash
+# Create checkout script
+fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod
+
+# Create and run checkout script
+fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod --execute
+
+# Create Makefile
+fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod
+
+# Create the compile script
+fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod
+
+# Create and run the compile script
+fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod --execute
+```
+### **Bare-metal Build Multi-target:**
+```bash
+# Create checkout script
+fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+
+# Create and run checkout script
+fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+
+# Create Makefile
+fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+
+# Create the compile script
+fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+
+# Create and run the compile script
+fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+```
+
+### **Container Build:**
+In order for the container to build successfully, a `-npc`, or `--no-parallel-checkout` is needed.
+```bash
+# Create checkout script
+fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc
+
+# Create and run checkout script
+fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc --execute
+
+# Create Makefile
+fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod
+
+# Create Dockerfile
+fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod
+
+# Create and run the Dockerfile
+fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod --execute
+```
+
+### **Run all of fremake:**
+```bash
+# Bare-metal
+fre make run-fremake -y am5.yaml -p ncrc5.intel23 -t prod
+
+# Container
+fre make run-fremake -y am5.yaml -p hpcme.2023 -t prod -npc
+```
+
+## Subtools
+- `fre make create-checkout [options]`
+ - Purpose: Creates the checkout script and can check out source code (with execute option)
+ - Options:
+ - `-y, --yamlfile [experiment yaml] (required)`
+ - `-p, --platform [platform] (required)`
+ - `-t, --target [target] (required)`
+ - `-j, --jobs [number of jobs to run simultneously]`
+ - `-npc, --no-parallel-checkout (for container build)`
+ - `-e, --execute`
+
+- `fre make create-makefile [options]`
+ - Purpose: Creates the makefile
+ - Options:
+ - `-y, --yamlfile [experiment yaml] (required)`
+ - `-p, --platform [platform] (required)`
+ - `-t, --target [target] (required)`
+
+- `fre make create-compile [options]`
+ - Purpose: Creates the compile script and compiles the model (with execute option)
+ - Options:
+ - `-y, --yamlfile [experiment yaml] (required)`
+ - `-p, --platform [platform] (required)`
+ - `-t, --target [target] (required)`
+ - `-j, --jobs [number of jobs to run simultneously]`
+ - `-n, --parallel [number of concurrent modile compiles]`
+ - `-e, --execute`
+
+- `fre make create-dockerfile [options]`
+ - Purpose: Creates the dockerfile and creates the container (with execute option)
+ - With the creation of the dockerfile, the Makefile, checkout script, and any other necessary script is copied into the container from a temporary location
+ - Options:
+ - `-y, --yamlfile [experiment yaml] (required)`
+ - `-p, --platform [platform] (required)`
+ - `-t, --target [target] (required)`
+ - `-e, --execute`
+
+- `fre make run-fremake [options]`
+ - Purpose: Create the checkout script, Makefile, compile script, and dockerfile (platform dependent) for the compilation of the model
+ - Options:
+ - `-y, --yamlfile [experiment yaml] (required)`
+ - `-p, --platform [platform] (required)`
+ - `-t, --target [target] (required)`
+ - `-npc, --no-parallel-checkout (for container build)`
+ - `-j, --jobs [number of jobs to run simultneously]`
+ - `-n, --parallel [number of concurrent modile compiles]`
+
+## Guide
+In order to use the `fre make` tools, remember to create a combined yaml first. This can be done with the `fre yamltools combine-yamls` tool. This combines the model, compile, platform, experiment, and any analysis yamls into ONE yaml file for parsing and validation.
+
+To combine:
+`fre yamltools combine-yamls -y [model yaml file] -e [experiment name] -p [platform] -t [target]`
+
+### **Bare-metal Build:**
+```bash
+# Create checkout script
+fre make create-checkout -y [model yaml file] -p [platform] -t [target]
+
+# Create and run checkout script
+fre make create-checkout -y [model yaml file] -p [platform] -t [target] --execute
+
+# Create Makefile
+fre make create-makefile -y [model yaml file] -p [platform] -t [target]
+
+# Creat the compile script
+fre make create-compile -y [model yaml file] -p [platform] -t [target]
+
+# Create and run the compile script
+fre make create-compile -y [model yaml file] -p [platform] -t [target] --execute
+
+# Run all of fremake
+fre make run-fremake -y [model yaml file] -p [platform] -t [target] [other options...]
+```
+
+### **Container Build:**
+For the container build, parallel checkouts are not supported, so the `-npc` options must be used for the checkout script. In addition the platform must be a container platform.
+
+***To reiterate, users will not be able to create containers unless they have podman access on gaea.***
+```bash
+# Create checkout script
+fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] -npc
+
+# Create and run checkout script
+fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute
+
+# Create Makefile
+fre make create-makefile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target]
+
+#Create a Dockerfile
+fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target]
+
+# Create and run the Dockerfile
+fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute
+```
diff --git a/lib/fre/make/__init__.py b/lib/fre/make/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/make/createCheckout.py b/lib/fre/make/createCheckout.py
new file mode 100644
index 00000000..fc4d2df4
--- /dev/null
+++ b/lib/fre/make/createCheckout.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+import logging
+import sys
+import click
+import fre.yamltools.combine_yamls as cy
+from .gfdlfremake import varsfre, yamlfre, checkout, targetfre
+
+def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+ # Define variables
+ yml = yamlfile
+ name = yamlfile.split(".")[0]
+ run = execute
+ jobs = str(jobs)
+ pcheck = no_parallel_checkout
+
+ if pcheck:
+ pc = ""
+ else:
+ pc = " &"
+
+ if verbose:
+ logging.basicConfig(level=logging.INFO)
+ else:
+ logging.basicConfig(level=logging.ERROR)
+
+ srcDir="src"
+ checkoutScriptName = "checkout.sh"
+ baremetalRun = False # This is needed if there are no bare metal runs
+
+ ## Split and store the platforms and targets in a list
+ plist = platform
+ tlist = target
+
+ # Combine model, compile, and platform yamls
+ # Default behavior - combine yamls / rewrite combined yaml
+ comb = cy.init_compile_yaml(yml,platform,target)
+ full_combined = cy.get_combined_compileyaml(comb)
+
+ ## Get the variables in the model yaml
+ freVars = varsfre.frevars(full_combined)
+
+ ## Open the yaml file, validate the yaml, and parse as fremakeYaml
+ modelYaml = yamlfre.freyaml(full_combined,freVars)
+ fremakeYaml = modelYaml.getCompileYaml()
+
+ ## Error checking the targets
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+
+ ## Loop through the platforms specified on the command line
+ ## If the platform is a baremetal platform, write the checkout script and run it once
+ ## This should be done separately and serially because bare metal platforms should all be using
+ ## the same source code.
+ for platformName in plist:
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError (platformName + " does not exist in platforms.yaml")
+ ( compiler, modules, modulesInit, fc, cc, modelRoot,
+ iscontainer, mkTemplate, containerBuild, ContainerRun,
+ RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Create the source directory for the platform
+ if iscontainer is False:
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ # if the source directory does not exist, it is created
+ if not os.path.exists(srcDir):
+ os.system("mkdir -p " + srcDir)
+ # if the checkout script does not exist, it is created
+ if not os.path.exists(srcDir+"/checkout.sh"):
+ freCheckout = checkout.checkout("checkout.sh",srcDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+ # Make checkout script executable
+ os.chmod(srcDir+"/checkout.sh", 0o744)
+ print("\nCheckout script created in "+ srcDir + "/checkout.sh \n")
+
+ # Run the checkout script
+ if run is True:
+ freCheckout.run()
+ else:
+ sys.exit()
+ else:
+ print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n")
+ if run == True:
+ try:
+ subprocess.run(args=[srcDir+"/checkout.sh"], check=True)
+ except:
+ print("\nThere was an error with the checkout script "+srcDir+"/checkout.sh.",
+ "\nTry removing test folder: " + modelRoot +"\n")
+ raise
+ else:
+ sys.exit()
+
+ else:
+ image="ecpe4s/noaa-intel-prototype:2023.09.25"
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
+ tmpDir = "tmp/"+platformName
+ freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+ print("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n")
+
+@click.command()
+def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+ '''
+ Decorator for calling checkout_create - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose)
+
+if __name__ == "__main__":
+ checkout_create()
diff --git a/lib/fre/make/createCompile.py b/lib/fre/make/createCompile.py
new file mode 100644
index 00000000..36068329
--- /dev/null
+++ b/lib/fre/make/createCompile.py
@@ -0,0 +1,99 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import logging
+from pathlib import Path
+from multiprocessing.dummy import Pool
+import click
+from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal
+import fre.yamltools.combine_yamls as cy
+
+def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
+ # Define variables
+ yml = yamlfile
+ name = yamlfile.split(".")[0]
+ nparallel = parallel
+ jobs = str(jobs)
+ run = execute
+
+ if verbose:
+ logging.basicCOnfig(level=logging.INFO)
+ else:
+ logging.basicConfig(level=logging.ERROR)
+
+ srcDir="src"
+ checkoutScriptName = "checkout.sh"
+ baremetalRun = False # This is needed if there are no bare metal runs
+
+ ## Split and store the platforms and targets in a list
+ plist = platform
+ tlist = target
+
+ # Combined compile yaml file
+ combined = Path(f"combined-{name}.yaml")
+
+ ## If combined yaml exists, note message of its existence
+ ## If combined yaml does not exist, combine model, compile, and platform yamls
+ full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
+ ## Get the variables in the model yaml
+ freVars = varsfre.frevars(full_combined)
+
+ ## Open the yaml file and parse as fremakeYaml
+ modelYaml = yamlfre.freyaml(full_combined,freVars)
+ fremakeYaml = modelYaml.getCompileYaml()
+
+ ## Error checking the targets
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+
+ fremakeBuildList = []
+ ## Loop through platforms and targets
+ for platformName in plist:
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+
+ (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
+ ## Make the bldDir based on the modelRoot, the platform, and the target
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ ## Check for type of build
+ if iscontainer is False:
+ baremetalRun = True
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec"
+ os.system("mkdir -p " + bldDir)
+ ## Create a list of compile scripts to run in parallel
+ fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"],
+ mkTemplatePath = mkTemplate,
+ srcDir = srcDir,
+ bldDir = bldDir,
+ target = target,
+ modules = modules,
+ modulesInit = modulesInit,
+ jobs = jobs)
+ for c in fremakeYaml['src']:
+ fremakeBuild.writeBuildComponents(c)
+ fremakeBuild.writeScript()
+ fremakeBuildList.append(fremakeBuild)
+ click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
+ if run:
+ if baremetalRun:
+ pool = Pool(processes=nparallel) # Create a multiprocessing Pool
+ pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool
+ else:
+ sys.exit()
+
+@click.command()
+def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
+ '''
+ Decorator for calling compile_create - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose)
+
+if __name__ == "__main__":
+ compile_create()
diff --git a/lib/fre/make/createDocker.py b/lib/fre/make/createDocker.py
new file mode 100644
index 00000000..59b73ee9
--- /dev/null
+++ b/lib/fre/make/createDocker.py
@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+
+import os
+import sys
+from pathlib import Path
+import click
+#from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker
+from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker
+import fre.yamltools.combine_yamls as cy
+
+def dockerfile_create(yamlfile,platform,target,execute):
+ srcDir="src"
+ checkoutScriptName = "checkout.sh"
+ baremetalRun = False # This is needed if there are no bare metal runs
+ ## Split and store the platforms and targets in a list
+ plist = platform
+ tlist = target
+ yml = yamlfile
+ name = yamlfile.split(".")[0]
+ run = execute
+
+ # Combined compile yaml file
+ combined = Path(f"combined-{name}.yaml")
+
+ ## If combined yaml exists, note message of its existence
+ ## If combined yaml does not exist, combine model, compile, and platform yamls
+ full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
+ ## Get the variables in the model yaml
+ freVars = varsfre.frevars(full_combined)
+
+ ## Open the yaml file and parse as fremakeYaml
+ modelYaml = yamlfre.freyaml(full_combined,freVars)
+ fremakeYaml = modelYaml.getCompileYaml()
+
+ fremakeBuildList = []
+ ## Loop through platforms and targets
+ for platformName in plist:
+ for targetName in tlist:
+ targetObject = targetfre.fretarget(targetName)
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError (platformName + " does not exist in " + \
+ modelYaml.combined.get("compile").get("platformYaml"))
+
+ ( compiler, modules, modulesInit, fc, cc, modelRoot,
+ iscontainer, mkTemplate, containerBuild, containerRun,
+ RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Make the bldDir based on the modelRoot, the platform, and the target
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ ## Check for type of build
+ if iscontainer is True:
+ image="ecpe4s/noaa-intel-prototype:2023.09.25"
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
+ tmpDir = "tmp/"+platformName
+
+ dockerBuild = buildDocker.container(base = image,
+ exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ RUNenv = RUNenv,
+ target = targetObject)
+ dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
+ dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh")
+
+ for c in fremakeYaml['src']:
+ dockerBuild.writeDockerfileMkmf(c)
+
+ dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
+ currDir = os.getcwd()
+ click.echo("\ntmpDir created in " + currDir + "/tmp")
+ click.echo("Dockerfile created in " + currDir +"\n")
+
+ if run:
+ dockerBuild.build(containerBuild, containerRun)
+ else:
+ sys.exit()
+
+@click.command()
+def _dockerfile_create(yamlfile,platform,target,execute):
+ '''
+ Decorator for calling dockerfile_create - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return dockerfile_create(yamlfile,platform,target,execute)
+
+if __name__ == "__main__":
+ dockerfile_create()
diff --git a/lib/fre/make/createMakefile.py b/lib/fre/make/createMakefile.py
new file mode 100644
index 00000000..eaf340dd
--- /dev/null
+++ b/lib/fre/make/createMakefile.py
@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+
+import os
+from pathlib import Path
+
+import click
+
+from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre
+import fre.yamltools.combine_yamls as cy
+
+def makefile_create(yamlfile,platform,target):
+ srcDir="src"
+ checkoutScriptName = "checkout.sh"
+ baremetalRun = False # This is needed if there are no bare metal runs
+ ## Split and store the platforms and targets in a list
+ plist = platform
+ tlist = target
+ yml = yamlfile
+ name = yamlfile.split(".")[0]
+
+ combined = Path(f"combined-{name}.yaml")
+
+ ## If combined yaml exists, note message of its existence
+ ## If combined yaml does not exist, combine model, compile, and platform yamls
+ full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
+ ## Get the variables in the model yaml
+ freVars = varsfre.frevars(full_combined)
+
+ ## Open the yaml file and parse as fremakeYaml
+ modelYaml = yamlfre.freyaml(full_combined,freVars)
+ fremakeYaml = modelYaml.getCompileYaml()
+
+ fremakeBuildList = []
+ ## Loop through platforms and targets
+ for platformName in plist:
+ for targetName in tlist:
+ targetObject = targetfre.fretarget(targetName)
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
+
+ (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
+ ## Make the bldDir based on the modelRoot, the platform, and the target
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ ## Check for type of build
+ if iscontainer is False:
+ baremetalRun = True
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + targetObject.gettargetName() + "/exec"
+ os.system("mkdir -p " + bldDir)
+ ## Create the Makefile
+ freMakefile = makefilefre.makefile(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["baremetal_linkerflags"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate)
+ # Loop through components and send the component name, requires, and overrides for the Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+ click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n")
+ else:
+ image="ecpe4s/noaa-intel-prototype:2023.09.25"
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
+ tmpDir = "tmp/"+platformName
+ freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate,
+ tmpDir = tmpDir)
+
+ # Loop through compenents and send the component name and requires for the Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+ click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n")
+
+@click.command()
+def _makefile_create(yamlfile,platform,target):
+ '''
+ Decorator for calling makefile_create - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return makefile_create(yamlfile,platform,target)
+
+if __name__ == "__main__":
+ makefile_create()
diff --git a/lib/fre/make/fremake.py b/lib/fre/make/fremake.py
new file mode 100644
index 00000000..f39a6be0
--- /dev/null
+++ b/lib/fre/make/fremake.py
@@ -0,0 +1,208 @@
+import click
+from fre.make import createCheckout
+from fre.make import createMakefile
+from fre.make import createCompile
+from fre.make import createDocker
+from fre.make import runFremake
+
+yamlfile_opt_help = """Experiment yaml compile FILE
+"""
+experiment_opt_help = """Name of experiment"""
+platform_opt_help = """Hardware and software FRE platform space separated list of STRING(s).
+This sets platform-specific data and instructions
+"""
+target_opt_help = """a space separated list of STRING(s) that defines compilation settings and
+linkage directives for experiments. Predefined targets refer to groups of directives that exist in
+the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod',
+'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive
+(cannot not use more than one of these in the target list). Any number of targets can be used.
+"""
+parallel_opt_help = """Number of concurrent model compiles (default 1)
+"""
+jobs_opt_help = """Number of jobs to run simultaneously. Used for make -jJOBS and git clone
+recursive --jobs=JOBS
+"""
+no_parallel_checkout_opt_help = """Use this option if you do not want a parallel checkout.
+The default is to have parallel checkouts.
+"""
+verbose_opt_help = """Get verbose messages (repeat the option to increase verbosity level)
+"""
+
+
+
+@click.group(help=click.style(" - access fre make subcommands", fg=(210,73,57)))
+def make_cli():
+ pass
+
+@make_cli.command()
+@click.option("-y",
+ "--yamlfile",
+ type = str,
+ help = yamlfile_opt_help,
+ required = True) # use click.option() over click.argument(), we want help statements
+@click.option("-p",
+ "--platform",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = platform_opt_help, required = True)
+@click.option("-t", "--target",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = target_opt_help,
+ required = True)
+@click.option("-n",
+ "--parallel",
+ type = int,
+ metavar = '',
+ default = 1,
+ help = parallel_opt_help)
+@click.option("-j",
+ "--jobs",
+ type = int,
+ metavar = '',
+ default = 4,
+ help = jobs_opt_help)
+@click.option("-npc",
+ "--no-parallel-checkout",
+ is_flag = True,
+ help = no_parallel_checkout_opt_help)
+@click.option("-v",
+ "--verbose",
+ is_flag = True,
+ help = verbose_opt_help)
+@click.pass_context
+def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose):
+ """ - Perform all fremake functions to run checkout and compile model"""
+ context.forward(runFremake._fremake_run)
+
+####
+@make_cli.command()
+@click.option("-y",
+ "--yamlfile",
+ type = str,
+ help = yamlfile_opt_help,
+ required = True) # use click.option() over click.argument(), we want help statements
+@click.option("-p",
+ "--platform",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = platform_opt_help,
+ required = True)
+@click.option("-t", "--target",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = target_opt_help,
+ required = True)
+@click.option("-j",
+ "--jobs",
+ type = int,
+ metavar = '',
+ default = 4,
+ help = jobs_opt_help)
+@click.option("-npc",
+ "--no-parallel-checkout",
+ is_flag = True,
+ help = no_parallel_checkout_opt_help)
+@click.option("--execute",
+ is_flag = True,
+ default = False,
+ help = "Use this to run the created checkout script.")
+@click.option("-v",
+ "--verbose",
+ is_flag = True,
+ help = verbose_opt_help)
+@click.pass_context
+def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+ """ - Write the checkout script """
+ context.forward(createCheckout._checkout_create)
+
+#####
+@make_cli.command
+@click.option("-y",
+ "--yamlfile",
+ type = str,
+ help = yamlfile_opt_help,
+ required = True) # use click.option() over click.argument(), we want help statements
+@click.option("-p",
+ "--platform",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = platform_opt_help, required = True)
+@click.option("-t", "--target",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = target_opt_help,
+ required = True)
+@click.pass_context
+def create_makefile(context,yamlfile,platform,target):
+ """ - Write the makefile """
+ context.forward(createMakefile._makefile_create)
+
+#####
+
+@make_cli.command
+@click.option("-y",
+ "--yamlfile",
+ type = str,
+ help = yamlfile_opt_help,
+ required = True) # use click.option() over click.argument(), we want help statements
+@click.option("-p",
+ "--platform",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = platform_opt_help, required = True)
+@click.option("-t", "--target",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = target_opt_help,
+ required = True)
+@click.option("-j",
+ "--jobs",
+ type = int,
+ metavar = '',
+ default = 4,
+ help = jobs_opt_help)
+@click.option("-n",
+ "--parallel",
+ type = int,
+ metavar = '', default = 1,
+ help = parallel_opt_help)
+@click.option("--execute",
+ is_flag = True,
+ default = False,
+ help = "Use this to run the created checkout script.")
+@click.option("-v",
+ "--verbose",
+ is_flag = True,
+ help = verbose_opt_help)
+@click.pass_context
+def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose):
+ """ - Write the compile script """
+ context.forward(createCompile._compile_create)
+
+@make_cli.command
+@click.option("-y",
+ "--yamlfile",
+ type = str,
+ help = yamlfile_opt_help,
+ required = True) # use click.option() over click.argument(), we want help statements
+@click.option("-p",
+ "--platform",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = platform_opt_help, required = True)
+@click.option("-t", "--target",
+ multiple = True, # replaces nargs = -1, since click.option()
+ type = str,
+ help = target_opt_help,
+ required = True)
+@click.option("--execute",
+ is_flag = True,
+ help = "Build Dockerfile that has been generated by create-docker.")
+@click.pass_context
+def create_dockerfile(context,yamlfile,platform,target,execute):
+ """ - Write the dockerfile """
+ context.forward(createDocker._dockerfile_create)
+
+if __name__ == "__main__":
+ make_cli()
diff --git a/lib/fre/make/gfdlfremake/.gitignore b/lib/fre/make/gfdlfremake/.gitignore
new file mode 100644
index 00000000..a623caf0
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/.gitignore
@@ -0,0 +1,5 @@
+__pycache__/
+Dockerfile
+checkout.sh
+compile.sh
+Makefile
diff --git a/lib/fre/make/gfdlfremake/.gitlab-ci.yml b/lib/fre/make/gfdlfremake/.gitlab-ci.yml
new file mode 100644
index 00000000..1e1b666c
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/.gitlab-ci.yml
@@ -0,0 +1,12 @@
+stages:
+ - test
+
+test_build_am5:
+ stage: test
+ script:
+# conda env
+# - /ncrc/sw/gaea-c5/python/3.9/anaconda-base/envs/noaa_py3.9
+ - cd yamls/
+ - ../fremake -y am5.yaml -p ncrc5.intel -t prod
+ tags:
+ - ncrc5
diff --git a/lib/fre/make/gfdlfremake/README.md b/lib/fre/make/gfdlfremake/README.md
new file mode 100644
index 00000000..18aa8d44
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/README.md
@@ -0,0 +1,147 @@
+# FREMAKE CANOPY
+The fremake canopy prototype embodies the checking out of source code and compilation of the model. This rewrite was developed with the idea of portability and flexibilty in mind.
+
+Fremake canopy is written in python and can support:
+- Bare-metal and container builds (with container platform)
+ - bare-metal build:
+ - supports parallel checkouts (default behavior)
+ - supports parallel model builds
+ - container:
+ - e4s container with spack stack package list
+- Multiple platforms and targets can be given (for multiple platform-target combinations)
+- One yaml format
+- Additional library support if needed by model
+
+Fremake has also been integrated as a fre-cli tool, creating a more modular and interactive experience.
+
+## Get Started
+Steps for how to build an example with the AM5 model, for both the bare-metal compilation and container, are outlined below.
+
+- ### [Set-up environment](#how-to-get-started)
+ - Fremake tools are provided through the fre-cli
+ - For more information on the fre-cli, see its associated github page: https://github.com/NOAA-GFDL/fre-cli
+- ### [Bare-metal build](#how-to-build-the-bare-metal-example)
+ - Note: make sure to have the correct experiment yaml, compile yaml, and platform yaml available for the model compilation
+- ### [Container build](#how-to-build-the-container-example)
+ - Note: make sure to have the correct experiment yaml, compile yaml and platform yaml available for the model compilation
+- ### [Full fremake: bare-metal and container](#how-to-run-complete-fremake)
+ - this can be done for either bare-metal or container builds
+ - this tool is available to give the option to run fremake fully, not in steps
+
+
+## YAMLS:
+experiment yaml: This is the main yaml and gives the path to the other yamls
+```yaml
+platformYaml: path to yaml containing the platforms
+layoutYaml: path to the yaml containing the layouts (not used currently)
+compileYaml: path to the yaml with the model source code and build information
+experiments: path to the yaml that has information about expeiment configurations (not currently used)
+
+```
+
+compile yaml: Containers information about source code and how to build
+```yaml
+experiment: name of the model
+container_addlibs: ["comma separated list of additional packages/libraries"]
+baremetal_linkerflags: ["comma separated list of linker flags needed for each additional package/library"]
+src:
+ - component: The name of the component model
+ requires: [Array of required components]
+ repo: scalar URL of the repo --OR-- [array of URLs to the code repos if more than one is required]
+ branch: scalar (or array of same size of repo) of the version of the code to clone
+ doF90Cpp: True if the F90cpp needs to be done (land)
+ cppdefs: a single string containing all CPPDEFs to add during compilations
+ paths: [array of paths to build]
+ otherFlags: Additional flags defined as environment variables in the experiment.yaml
+```
+
+platform yaml: User defined platform specifications. This will require more user input that bronx
+```yaml
+platforms:
+ - name: the platform name
+ compiler: the compiler you are using
+ modulesInit: ["array of commands that are needed to load modules." , "each command must end with a newline character"]
+ modules: [array of modules to load including compiler]
+ fc: the name of the fortran compiler
+ cc: the name of the C compiler
+ mkTemplate: The location of the mkmf make template
+ modelRoot: The root directory of the model (where src, exec, experiments will go)
+ container: True if this is a container platform
+ containerBuild: "podman" - the container build program
+ containerRun: "apptainer" - the container run program
+```
+
+## How to Get started
+Set up the environment:
+```bash
+# Load the fre-cli (if on gaea or somewhere where the fre-cli is available)
+module load fre/canopy
+
+# If the fre-cli is not available, follow steps in the fre-cli repository to either create your own conda environment with the fre-cli or activate the fre-cli environment available
+# If no fre-cli env available:
+conda create -n fremake-env # create environment
+conda config --append channels noaa-gfdl # append necessary channels
+conda config --append channels conda-forge # append necessary channels
+conda activate fremake-env # activate environment
+conda install noaa-gfdl::fre-cli # install the fre-cli
+```
+
+### How to build the Bare-metal example
+```bash
+# Have access to yamls
+# older versino: git clone -b 2023.00 https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+
+git clone https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+cd fremake_canopy/yamls
+
+# Create and run checkout
+# `-e` will run the checkout script after creating it
+# without `-e`, the checkout script will just be created
+fre make create-checkout -y am5.yaml -p ncrc5.intel -t prod -e
+
+# Create the Makefile
+fre make create-makefile -y am5.yaml -p ncrc5.intel -t prod
+
+# Create and run the compile script
+# `-e` will run the compile script after creating it
+# without `-e`, the compile script will just be created
+fre make create-compile -y am5.yaml -p ncrc5.intel -t prod -e
+```
+*Corresponding files, such as the checkout script, makefile, compile script, and experiment executable will be in the `/exec` folder created in `$HOME/$USER/fremake-canopy/am5/ncrc5.intel-prod/test`*
+
+### How to build the Container example
+```bash
+# Have access to yamls
+# older version: git clone -b 2023.00 https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+
+git clone https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+cd fremake_canopy/yamls
+
+# Create and run checkout
+# `-e` will run the checkout script after creating it
+# without `-e`, the checkout script will just be created
+# Be sure to specify `-npc` for non-parallel checkout
+fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc -e
+
+# Create the Makefile
+fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod
+
+# Create and run the dockerfile
+# `-e` will run the dockerfile after creating it, to build a container
+# without `-e`, the dockerfile will just be created
+fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod -e
+```
+*Corresponding files, such as the checkout script and makefile, will be in a tmp location and copied into the container via the dockerfile. The dockerfile,, as well as the container sif file, will be created in the users current location (yaml folder).*
+
+### How to run complete fremake
+```bash
+# Have access to yamls
+# older version: git clone -b 2023.00 https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+
+git clone https://gitlab.gfdl.noaa.gov/portable_climate/fremake_canopy.git
+cd fremake_canopy/yamls
+
+# Run fremake
+# Can run fremake for bare-metal or container; be sure to give the correct platform for your build
+fre make run-fremake -y am5.yaml -p [ncrc5.intel OR hpcme.2023] -t prod
+```
diff --git a/lib/fre/make/gfdlfremake/buildBaremetal.py b/lib/fre/make/gfdlfremake/buildBaremetal.py
new file mode 100644
index 00000000..fdb4e2d8
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/buildBaremetal.py
@@ -0,0 +1,129 @@
+#!/usr/bin/python3
+## \date 2023
+## \author Tom Robinson
+## \email thomas.robinson@noaa.gov
+## \description
+
+import subprocess
+import os
+
+def fremake_parallel(fremakeBuildList):
+ """
+ Brief: Called for parallel execution purposes. Runs the builds.
+ Param:
+ - fremakeBuildList : fremakeBuild object list passes by pool.map
+ """
+ fremakeBuildList.run()
+
+class buildBaremetal():
+ """
+ Brief: Creates the build script to compile the model
+ Param:
+ - self : The buildScript object
+ - exp : The experiment name
+ - mkTemplatePath : The template used by mkmf to compile the model
+ - srcDir : The source directory
+ - bldDir : The build directory
+ - modules : The list of modules to load before compilation
+ - modulesInit : A list of commands with new line characters to initialize modules
+ """
+ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jobs):
+ """
+ Initialize variables and set-up the compile script.
+ """
+ self.e = exp
+ self.t = target.gettargetName()
+ self.src = srcDir
+ self.bld = bldDir
+ self.make = "make --jobs="+str(jobs)+" "+target.getmakeline_add() #make line
+ self.mkmf = True
+ self.template = mkTemplatePath
+ self.modules = ""
+ for m in modules:
+ self.modules = self.modules +" "+ m
+
+ ## Set up the top portion of the compile script
+ self.setup=[ "#!/bin/sh -fx \n",
+ "bld_dir="+self.bld+"/ \n",
+ "src_dir="+self.src+"/ \n",
+ "mkmf_template="+self.template+" \n"]
+ if self.modules != "":
+ self.setup.extend(modulesInit) #extend - this is a list
+ self.setup.append("module load "+self.modules+" \n") # Append -this is a single string
+
+ ## Create the build directory
+ os.system("mkdir -p "+self.bld)
+
+ ## Create the compile script
+ self.f=open(self.bld+"/compile.sh","w")
+ self.f.writelines(self.setup)
+
+ def writeBuildComponents(self, c):
+ """
+ Brief: Adds components to the build script
+ Param:
+ - self : The build script object
+ - c : Component from the compile yaml
+ """
+ # Shorthand for component
+ comp = c["component"]
+
+ # Make the component directory
+ self.f.write("\n mkdir -p $bld_dir/"+comp+"\n")
+
+ # Get the paths needed for compiling
+ pstring = ""
+ for paths in c["paths"]:
+ pstring = pstring+"$src_dir/"+paths+" "
+
+ # Run list_paths
+ self.f.write(" list_paths -l -o $bld_dir/"+comp+"/pathnames_"+comp+" "+pstring+"\n")
+ self.f.write(" cd $bld_dir/"+comp+"\n")
+
+ # Create the mkmf line
+ # If this lib doesnt have any code dependencies and
+ # it requires the preprocessor (no -o and yes --use-cpp)
+ if c["requires"] == [] and c["doF90Cpp"]:
+ self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ elif c["requires"] == []: # If this lib doesnt have any code dependencies (no -o)
+ self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ else: #Has requirements
+ #Set up the requirements as a string to inclue after the -o
+ reqstring = ""
+ for r in c["requires"]:
+ reqstring = reqstring+"-I$bld_dir/"+r+" "
+
+ #Figure out if we need the preprocessor
+ if c["doF90Cpp"]:
+ self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ else:
+ self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+
+##TODO: add targets input
+ def writeScript(self):
+ """
+ Brief: Finishes and writes the build script
+ Param:
+ - self : The buildScript object
+ """
+ self.f.write("cd "+self.bld+"\n")
+ self.f.write(self.make+"\n")
+ self.f.close()
+
+ # Make compile script executable
+ os.chmod(self.bld+"/compile.sh", 0o744)
+
+## TODO run as a batch job on the login cluster
+ def run(self):
+ """
+ Brief: Run the build script
+ Param:
+ - self : The dockerfile object
+ """
+###### TODO make the Makefile
+ command = [self.bld+"/compile.sh","|","tee",self.bld+"/log.compile"]
+ try:
+ subprocess.run(args=command, check=True)
+ except:
+ print("There was an error running "+self.bld+"/compile.sh")
+ raise
diff --git a/lib/fre/make/gfdlfremake/buildDocker.py b/lib/fre/make/gfdlfremake/buildDocker.py
new file mode 100644
index 00000000..6d33d0d2
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/buildDocker.py
@@ -0,0 +1,200 @@
+#!/usr/bin/python3
+## \date 2023
+## \author Tom Robinson
+## \email thomas.robinson@noaa.gov
+## \description
+
+import os
+
+class container():
+ """
+ Brief: Opens the Dockerfile for writing
+ Param:
+ - self : The dockerfile object
+ - base : The docker base image to start from
+ - libs : Additional libraries defined by user
+ - exp : The experiment name
+ - RUNenv : The commands that have to be run at
+ the beginning of a RUN in the dockerfile
+ to set up the environment
+ """
+ def __init__(self,base,exp,libs,RUNenv,target):
+ """
+ Initialize variables and write to the dockerfile
+ """
+ self.base = base
+ self.e = exp
+ self.l = libs
+ self.src = "/apps/"+self.e+"/src"
+ self.bld = "/apps/"+self.e+"/exec"
+ self.mkmf = True
+ self.target = target
+ self.template = "/apps/mkmf/templates/hpcme-intel21.mk"
+
+ # Set up spack loads in RUN commands in dockerfile
+ if RUNenv == "":
+ self.setup = ["RUN \\ \n"]
+ else:
+ self.setup = ["RUN "+RUNenv[0]+" \\ \n"]
+ self.setup
+ for env in RUNenv[1:]:
+ self.setup.append(" && "+env+" \\ \n")
+ if self.l:
+ for l in self.l:
+ self.setup.append(" && spack load "+l+" \\ \n")
+
+ # Clone and copy mkmf through Dockerfile
+ self.mkmfclone=["RUN cd /apps \\ \n",
+ " && git clone --recursive https://github.com/NOAA-GFDL/mkmf \\ \n",
+ " && cp mkmf/bin/* /usr/local/bin \n"]
+
+ # Set bld_dir, src_dir, mkmf_template
+ self.bldsetup=["RUN bld_dir="+self.bld+" \\ \n",
+ " && src_dir="+self.src+" \\ \n",
+ " && mkmf_template="+self.template+ " \\ \n"]
+ self.d=open("Dockerfile","w")
+ self.d.writelines("FROM "+self.base+" \n")
+
+ def writeDockerfileCheckout(self, cScriptName, cOnDisk):
+ """
+ Brief: writes to the checkout part of the Dockerfile and sets up the compile
+ Param:
+ - self : The dockerfile object
+ - cScriptName : The name of the checkout script in the container
+ - cOnDisk : The relative path to the checkout script on disk
+ """
+ self.checkoutPath = self.src+"/"+ cScriptName
+ self.d.write("COPY " + cOnDisk +" "+ self.checkoutPath +" \n")
+ self.d.write("RUN chmod 744 "+self.src+"/checkout.sh \n")
+ self.d.writelines(self.setup)
+ self.d.write(" && "+self.src+"/checkout.sh \n")
+ # Clone mkmf
+ self.d.writelines(self.mkmfclone)
+
+ def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath):
+ """
+ Brief: Copies the Makefile into the bldDir in the dockerfile
+ Param:
+ - self : The dockerfile object
+ - makefileOnDiskPath : The path to Makefile on the local disk
+ - linklineonDiskPath : The path to the link line script on the local disk
+ """
+ # Set up the bldDir
+ # If no additional libraries defined
+ if self.l == None:
+ self.bldCreate=["RUN mkdir -p "+self.bld+" \n",
+ "COPY "+ makefileOnDiskPath +" "+self.bld+"/Makefile \n"]
+ self.d.writelines(self.bldCreate)
+ # If additional libraries defined
+ if self.l != None:
+ self.bldCreate=["RUN mkdir -p "+self.bld+" \n",
+ "COPY "+ makefileOnDiskPath +" "+self.bld+"/Makefile \n",
+ "RUN chmod +rw "+self.bld+"/Makefile \n",
+ "COPY "+ linklineonDiskPath +" "+self.bld+"/linkline.sh \n",
+ "RUN chmod 744 "+self.bld+"/linkline.sh \n"]
+ self.d.writelines(self.bldCreate)
+ self.d.writelines(self.setup)
+ self.d.write(" && "+self.bld+"/linkline.sh \n")
+
+ def writeDockerfileMkmf(self, c):
+ """
+ Brief: Adds components to the build part of the Dockerfile
+ Param:
+ - self : The dockerfile object
+ - c : Component from the compile yaml
+ """
+ # Set up the compile variables
+ self.d.writelines(self.bldsetup)
+
+ # Shorthand for component
+ comp = c["component"]
+
+ # Make the component directory
+ self.d.write(" && mkdir -p $bld_dir/"+comp+" \\ \n")
+
+ # Get the paths needed for compiling
+ pstring = ""
+ for paths in c["paths"]:
+ pstring = pstring+"$src_dir/"+paths+" "
+
+ # Run list_paths
+ self.d.write(" && list_paths -l -o $bld_dir/"+comp+"/pathnames_"+comp+" "+pstring+" \\ \n")
+ self.d.write(" && cd $bld_dir/"+comp+" \\ \n")
+
+ # Create the mkmf line
+ if c["requires"] == [] and c["doF90Cpp"]: # If this lib doesnt have any code dependencies and it requires the preprocessor (no -o and yes --use-cpp)
+ self.d.write(" && mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ elif c["requires"] == []: # If this lib doesnt have any code dependencies (no -o)
+ self.d.write(" && mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ else: #Has requirements
+ #Set up the requirements as a string to inclue after the -o
+ reqstring = ""
+ for r in c["requires"]:
+ reqstring = reqstring+"-I$bld_dir/"+r+" "
+
+ #Figure out if we need the preprocessor
+ if c["doF90Cpp"]:
+ self.d.write(" && mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+ else:
+ self.d.write(" && mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n")
+
+ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
+ """
+ Brief: Writes a runscript to set up spack loads/environment
+ in order to run the executable in the container;
+ runscript copied into container
+ Param:
+ - self : The dockerfile object
+ - RUNEnv : The commands that have to be run at
+ the beginning of a RUN in the dockerfile
+ - containerRun : The container platform used with `exec`
+ to run the container; apptainer
+ or singularity used
+ - runOnDisk : The path to the run script on the local disk
+ """
+ #create runscript in tmp - create spack environment, install necessary packages,
+ self.createscript = ["#!/bin/bash \n",
+ "export BACKUP_LD_LIBRARY_PATH=$LD_LIBRARY_PATH\n",
+ "# Set up spack loads\n",
+ RUNenv[0]+"\n"]
+ with open(runOnDisk,"w") as f:
+ f.writelines(self.createscript)
+ f.write("# Load spack packages\n")
+ for env in RUNenv[1:]:
+ f.write(env+"\n")
+
+ if self.l:
+ for l in self.l:
+ self.spackloads = "spack load "+l+"\n"
+ f.write(self.spackloads)
+
+ f.write("export LD_LIBRARY_PATH=$BACKUP_LD_LIBRARY_PATH:$LD_LIBRARY_PATH\n")
+ f.write("# Run executable\n")
+ f.write(self.bld+"/"+self.e+".x\n")
+ #copy runscript into container in dockerfile
+ self.d.write("COPY "+runOnDisk+" "+self.bld+"/execrunscript.sh\n")
+ #make runscript executable
+ self.d.write("RUN chmod 744 "+self.bld+"/execrunscript.sh\n")
+ #link runscript to more general location (for frerun container usage)
+ self.d.write("RUN mkdir -p /apps/bin \ \n")
+ self.d.write(" && ln -sf "+self.bld+"/execrunscript.sh "+"/apps/bin/execrunscript.sh \n")
+ #finish the dockerfile
+ self.d.writelines(self.setup)
+ self.d.write(" && cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
+ self.d.write('ENTRYPOINT ["/bin/bash"]')
+ self.d.close()
+
+ def build(self,containerBuild,containerRun):
+ """
+ Brief: Builds the container image for the model
+ Param:
+ - self : The dockerfile object
+ - containerBuild : The tool used to build the container;
+ docker or podman used
+ - containerRun : The container platform used with `exec` to
+ run the container; apptainer or singularity used
+ """
+ os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName())
+ os.system("rm -f "+self.e+".tar "+self.e+".sif")
+ os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName())
+ os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar")
diff --git a/lib/fre/make/gfdlfremake/checkout.py b/lib/fre/make/gfdlfremake/checkout.py
new file mode 100644
index 00000000..d3fa5d4e
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/checkout.py
@@ -0,0 +1,154 @@
+import os
+import subprocess
+
+## TODO: Add parallelizations using () and simplify
+def writeRepo(file,repo,component,srcDir,branch,add,multi,jobs,pc):
+ """
+ Brief: Creates the clone lines for the checkout script
+ Param:
+ - file Checkout script file
+ - repo the repo(s) to clone
+ - component Model component name
+ - srcDir The source directory
+ - branch The version to clone/checkout
+ - add Additional instrcutions after the clone
+ - multi True if a component has more than one repo to clone
+ """
+ ## Write message about cloning repo and branch in component
+ file.write("echo cloning "+repo+" -b "+branch+" into "+srcDir+"/"+component+"\n")
+
+ ## If this component has multiple repos, clone everything in the component folder
+ ## If it's not multi, then use the component name (comp) as the folder name to clone into
+ if multi:
+ file.write("mkdir -p "+component+"\n")
+ file.write("cd "+component+"\n")
+ comp=""
+ else:
+ comp=component
+
+ ## Check if there is a branch/version and then write the clone line;
+ ## record the pid of that clone in dictionary `pids` if parallel
+ ## checkout option is defined
+ if pc:
+ if branch=="":
+ file.write("(git clone --recursive --jobs="+jobs+" "+repo+" "+comp+")"+pc+"\n")
+ if multi:
+ r=repo.split("/")[4].strip(".git")
+ file.write("pids+=("+r+"pid:$!)\n")
+ else:
+ file.write("pids+=("+comp+"pid:$!)\n")
+ else:
+ file.write("(git clone --recursive --jobs="+jobs+" "+repo+" -b "+branch+" "+comp+")"+pc+"\n")
+ if multi:
+ r=repo.split("/")[4].strip(".git")
+ file.write("pids+=("+r+"pid:$!)\n")
+ else:
+ file.write("pids+=("+comp+"pid:$!)\n")
+ else:
+ if branch=="":
+ file.write("git clone --recursive --jobs="+jobs+" "+repo+" "+comp+"\n")
+ else:
+ file.write("git clone --recursive --jobs="+jobs+" "+repo+" -b "+branch+" "+comp+"\n")
+
+ ## Make sure to go back up in the folder structure
+ if multi:
+ file.write("cd .. \n")
+ if add!="":
+ file.write(add)
+
+class checkout():
+ """
+ Brief: Class to create the checkout script
+ """
+ def __init__(self,fname,srcDir):
+ """
+ Brief: Opens the checkout script with the specified name
+ Param:
+ - self The checkout script object
+ - fname The file name of the checkout script
+ - srcDir The source directory where fname will be run and source will exist
+ """
+ self.fname = fname
+ self.src = srcDir
+ os.system("mkdir -p "+self.src)
+ ##TODO: Force checkout
+ os.system("rm -rf "+self.src+"/*")
+ self.checkoutScript = open(self.src+"/"+fname, 'w')
+ self.checkoutScript.write("#!/bin/sh -f \n")
+ self.checkoutScript.write("export GIT_TERMINAL_PROMPT=0 \n")
+ def writeCheckout(self,y,jobs,pc):
+ """
+ Brief: Writes the contents of the checkout script by looping through the input yaml
+ Param:
+ - self The checkout script object
+ - y The fremake compile yaml
+ """
+ self.checkoutScript.write("cd "+self.src +"\n")
+ for c in y['src']:
+ if type(c['repo']) is list and type(c['branch']) is list:
+ for (repo,branch) in zip(c['repo'],c['branch']):
+ writeRepo(self.checkoutScript,repo,c['component'],self.src,branch,c['additionalInstructions'],True,jobs,pc)
+ else:
+ writeRepo(self.checkoutScript,c['repo'],c['component'],self.src,c['branch'],c['additionalInstructions'],False,jobs,pc)
+
+ def finish (self,pc):
+ """
+ Brief: If pc is defined: Loops through dictionary of pids,
+ waits for each pid individually, writes exit code in
+ `check` list;
+ allows checkoutscript to exit if exit code is not 0;
+ closes the checkout script when writing is done
+ Param:
+ - self The checkout script object
+ """
+ if pc:
+ self.checkoutScript.write('for id in ${pids[@]}; do\n wait ${id##*:}\n check+=("clone of ${id%%:*} exited with status $?")\ndone\n')
+ self.checkoutScript.write('for stat in "${check[@]}"; do\n echo $stat \n if [ ${stat##* } -ne 0 ]; then\n exit ${stat##* }\n fi\ndone')
+ self.checkoutScript.close()
+ else:
+ self.checkoutScript.close()
+
+## TODO: batch script building
+ def run (self):
+ """
+ Brief: Runs the checkout script
+ Param:
+ - self The checkout script object
+ """
+ try:
+ subprocess.run(args=[self.src+"/"+self.fname], check=True)
+ except:
+ print("There was an error with the checkout script "+self.src+"/"+self.fname)
+ raise
+###################################################################################################
+## Subclass for container checkout
+class checkoutForContainer(checkout):
+ """
+ Brief: Subclass for container checkout
+ """
+ def __init__(self,fname,srcDir,tmpdir):
+ """
+ Brief: Opens the checkout script with the specified name
+ Param:
+ - self : The checkout script object
+ - fname : The file name of the checkout script
+ - srcDir : The source directory where fname will be run and source will exist
+ - tmpdir : The relative path on disk that fname will be created (and copied from into the
+ container)
+ """
+ self.fname = fname
+ self.src = srcDir
+ self.tmpdir = tmpdir
+ os.system("mkdir -p "+self.tmpdir)
+ os.system("rm -rf "+self.tmpdir+"/*")
+ self.checkoutScript = open(self.tmpdir+"/"+fname, 'w')
+ self.checkoutScript.write("#!/bin/sh -fx \n")
+ self.checkoutScript.write("export GIT_TERMINAL_PROMPT=0 \n")
+
+ def cleanup (self):
+ """
+ Brief: Removes the self.tmpdir and contents
+ Param:
+ - self The checkout script object
+ """
+ os.system("rm -rf "+self.tmpdir)
diff --git a/lib/fre/make/gfdlfremake/fremake b/lib/fre/make/gfdlfremake/fremake
new file mode 100755
index 00000000..6c245424
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/fremake
@@ -0,0 +1,249 @@
+#!/usr/bin/python3
+## \date 2023
+## \author Tom Robinson
+## \author Dana Singh
+## \description fremake is used to create and run a code checkout script and compile a model.
+
+import subprocess
+import os
+import yaml
+import argparse
+import logging
+from . import targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal
+from multiprocessing.dummy import Pool
+
+## Add in cli options
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='Fremake is used to create a code checkout script to compile models for FRE experiments.')
+ parser.add_argument("-y",
+ "--yamlfile",
+ type=str, help="Experiment yaml compile FILE",required=True)
+ parser.add_argument("-p",
+ "--platform",
+ nargs='*',
+ type=str, help="Hardware and software FRE platform space separated list of STRING(s). This sets platform-specific data and instructions",required=True)
+ parser.add_argument("-t",
+ "--target",
+ nargs='*',
+ type=str, help="FRE target space separated list of STRING(s) that defines compilation settings and linkage directives for experiments.\n\nPredefined targets refer to groups of directives that exist in the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod', 'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive (cannot not use more than one of these in the target list). Any number of targets can be used.",required=True)
+ parser.add_argument("-f",
+ "--force-checkout",
+ action="store_true",
+ help="Force checkout to get a fresh checkout to source directory in case the source directory exists")
+ parser.add_argument("-F",
+ "--force-compile",
+ action="store_true",
+ help="Force compile to compile a fresh executable in case the executable directory exists")
+ parser.add_argument("-K",
+ "--keep-compiled",
+ action="store_true",
+ help="Keep compiled files in the executable directory for future use")
+ parser.add_argument("--no-link",
+ action="store_true",
+ help="Do not link the executable")
+ parser.add_argument("-E",
+ "--execute",
+ action="store_true",
+ help="Execute all the created scripts in the current session")
+ parser.add_argument("-n",
+ "--parallel",
+ type=int,
+ metavar='', default=1,
+ help="Number of concurrent model compiles (default 1)")
+ parser.add_argument("-j",
+ "--jobs",
+ type=int,
+ metavar='', default=4,
+ help="Number of jobs to run simultaneously. Used for make -jJOBS and git clone recursive --jobs=JOBS")
+ parser.add_argument("-npc",
+ "--no-parallel-checkout",
+ action="store_true",
+ help="Use this option if you do not want a parallel checkout. The default is to have parallel checkouts.")
+ parser.add_argument("-s",
+ "--submit",
+ action="store_true",
+ help="Submit all the created scripts as batch jobs")
+ parser.add_argument("-v",
+ "--verbose",
+ action="store_true",
+ help="Get verbose messages (repeat the option to increase verbosity level)")
+ parser.add_argument("-w NUM",
+ "--walltime=NUM",
+ type=int, metavar='',
+ help="Maximum wall time NUM (in minutes) to use")
+ parser.add_argument("--mail-list=STRING",
+ action="store_true",
+ help="Email the comma=separated STRING list of emails rather than \$USER\@noaa.gov")
+
+ ## Parse the arguments
+ args = parser.parse_args()
+
+ ## Define arguments as variables
+ yml = args.yamlfile
+ ps = args.platform
+ ts = args.target
+ nparallel = args.parallel
+ jobs = str(args.jobs)
+ pcheck = args.no_parallel_checkout
+
+ ## Define parallelism addition for checkouts
+ # If pcheck is defined, no parallel checkouts
+ # If pcheck is not defined, default is to have parallel checkouts
+ if pcheck:
+ pc = ""
+ else:
+ pc = " &"
+
+ ## Define operation of option(s) above
+ if args.verbose:
+ logging.basicCOnfig(level=logging.INFO)
+ else:
+ logging.basicConfig(level=logging.ERROR)
+
+#### Main
+srcDir="src"
+checkoutScriptName = "checkout.sh"
+baremetalRun = False # This is needed if there are no bare metal runs
+
+## Split and store the platforms and targets in a list
+plist = args.platform
+tlist = args.target
+
+## Get the variables in the model yaml
+freVars = varsfre.frevars(yml)
+
+## Open the yaml file and parse as fremakeYaml
+modelYaml = yamlfre.freyaml(yml,freVars)
+fremakeYaml = modelYaml.getCompileYaml()
+
+## Error checking the targets
+for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+
+## Loop through the platforms specified on the command line
+## If the platform is a baremetal platform, write the checkout script and run it once
+## This should be done separately and serially because bare metal platforms should all be using
+## the same source code.
+for platformName in plist:
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+ (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Create the checkout script
+ if iscontainer == False:
+ ## Create the source directory for the platform
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ if not os.path.exists(srcDir):
+ os.system("mkdir -p " + srcDir)
+ if not os.path.exists(srcDir+"/checkout.sh"):
+ freCheckout = checkout.checkout("checkout.sh",srcDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+
+## TODO: Options for running on login cluster?
+ freCheckout.run()
+
+fremakeBuildList = []
+## Loop through platforms and targets
+for platformName in plist:
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise SystemExit (platformName + " does not exist in " + modelYaml.platformsfile)
+ (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,containerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Make the source directory based on the modelRoot and platform
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+
+ ## Check for type of build
+ if iscontainer == False:
+ baremetalRun = True
+ ## Make the build directory based on the modelRoot, the platform, and the target
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec"
+ os.system("mkdir -p " + bldDir)
+
+ ## Create the Makefile
+ freMakefile = makefilefre.makefile(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["baremetal_linkerflags"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate)
+
+
+ # Loop through components and send the component name, requires, and overrides for the Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+
+## Create a list of compile scripts to run in parallel
+ fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"],
+ mkTemplatePath = mkTemplate,
+ srcDir = srcDir,
+ bldDir = bldDir,
+ target = target,
+ modules = modules,
+ modulesInit = modulesInit,
+ jobs = jobs)
+
+ for c in fremakeYaml['src']:
+ fremakeBuild.writeBuildComponents(c)
+ fremakeBuild.writeScript()
+ fremakeBuildList.append(fremakeBuild)
+ ## Run the build
+ fremakeBuild.run()
+ else:
+#################################### container stuff below ###########################################################
+ ## Run the checkout script
+# image="hpc-me-intel:2021.1.1"
+ image="ecpe4s/noaa-intel-prototype:2023.09.25"
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
+ tmpDir = "tmp/"+platformName
+
+ ## Create the checkout script
+ freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+
+ ## Create the makefile
+### Should this even be a separate class from "makefile" in makefilefre? ~ ejs
+ freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate,
+ tmpDir = tmpDir)
+
+ # Loop through components and send the component name and requires for the Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+
+ ## Build the dockerfile
+ dockerBuild = buildDocker.container(base = image,
+ exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ RUNenv = RUNenv,
+ target = target)
+
+ dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
+ dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile", freMakefile.getTmpDir()+"/linkline.sh")
+
+ for c in fremakeYaml['src']:
+ dockerBuild.writeDockerfileMkmf(c)
+
+ dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
+
+ ## Run the dockerfile; build the container
+ dockerBuild.build(containerBuild,containerRun)
+
+ #freCheckout.cleanup()
+ #buildDockerfile(fremakeYaml,image)
+
+if baremetalRun:
+ if __name__ == '__main__':
+ pool = Pool(processes=nparallel) # Create a multiprocessing Pool
+ pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool
diff --git a/lib/fre/make/gfdlfremake/makefilefre.py b/lib/fre/make/gfdlfremake/makefilefre.py
new file mode 100644
index 00000000..dfe15fff
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/makefilefre.py
@@ -0,0 +1,204 @@
+import os
+import textwrap
+
+def linklineBuild(self):
+ """
+ Brief: Writes the link line for bare metal and container builds
+ Param:
+ - self The Makefile object
+ """
+ linkline=""
+
+#if additional libraries are defined, populate the link line with the correct information for libraries
+## CONTAINER; write a script that will execute in the container, to fill in link line with additional libraries in Makefile
+ if "tmp" in self.filePath:
+ with open(self.filePath+"/linkline.sh","a") as fh:
+ fh.write("set -- ")
+ for l in self.l:
+ fh.write(l+" ")
+ fh.write("\n")
+
+ self.linklinecreate = '''
+ line=''
+ for l in $@; do
+ loc=$(spack location -i $l)
+ libraries=$(ls $loc/lib)
+ if echo "$libraries" | grep -q "_d"; then
+ for i in $libraries; do
+ if [ "$i" != "cmake" ] && echo "$i" | grep -q "_d"; then
+ ln1=${i%.*}
+ ln2=${ln1#???}
+ line=$line" -L$loc/lib -l$ln2"
+ fi
+ done
+ else
+ for i in $libraries; do
+ if [ "$i" != "cmake" ]; then
+ ln1=${i%.*}
+ ln2=${ln1#???}
+ line=$line" -L$loc/lib -l$ln2"
+ fi
+ done
+ fi
+ done
+ '''
+
+ with open(self.filePath+"/linkline.sh","a") as fh:
+ fh.writelines(textwrap.dedent(self.linklinecreate))
+ fh.write("MF_PATH='/apps/"+self.e+"/exec/Makefile'\n")
+ fh.write('sed -i "/MK_TEMPLATE = /a LL = $line" $MF_PATH\n')
+ fh.write("sed -i 's|\($^\) \($(LDFLAGS)\)|\\1 $(LL) \\2|' $MF_PATH\n")
+
+## BARE METAL; if addlibs defined on bare metal, include those additional libraries in link line
+ elif "tmp" not in self.filePath:
+ for l in self.l: # baremetal_linkerflags
+ linkline = linkline + " " + l
+ os.system(f"sed -i '/MK_TEMPLATE = /a LL = {linkline}' {self.filePath}/Makefile")
+ os.system(f"sed -i 's|\($(LDFLAGS)\)|$(LL) \\1|' {self.filePath}/Makefile")
+
+class makefile():
+ def __init__(self,exp,libs,srcDir,bldDir,mkTemplatePath):
+ """
+ Brief: Opens Makefile and sets the experiment and other common variables
+ Param:
+ - self The Makefile object
+ - exp Experiment name
+ - libs Additional libraries/linker flags defined by user
+ - srcDir The path to the source directory
+ - bldDir The path to the build directory
+ - mkTemplatePath The path of the template .mk file for compiling
+ """
+ self.e = exp
+ self.l = libs
+ self.src = srcDir
+ self.bld = bldDir
+ self.template = mkTemplatePath
+ self.c =[] #components
+ self.r=[] #requires
+ self.o=[] #overrides
+ os.system("mkdir -p "+self.bld)
+ self.filePath = self.bld # Needed so that the container and bare metal builds can
+ # use the same function to create the Makefile
+
+ def addComponent (self,c,r,o):
+ """
+ Brief: Adds a component and corresponding requires to the list
+ Param:
+ - self The Makefile object
+ - c The component
+ - r The requires for that componenet
+ - o The overrides for that component
+ """
+ self.c.append(c)
+ self.r.append(r)
+ self.o.append(o)
+
+ def createLibstring (self,c,r,o):
+ """
+ Brief: Sorts the component by how many requires there are for that component
+ Param:
+ - self The Makefile object
+ - c The component
+ - r The requires for that component
+ - o The overrides for that component
+ """
+ # org_comp : returns a zip object
+ org_comp = zip(self.c,self.r,self.o)
+ # Sort zip object so that the component with the most requires (self.r) is listed first, and so on
+ sort = sorted(org_comp,key=lambda values:len(values[1]),reverse=True)
+
+ return sort
+
+ def writeMakefile (self):
+ """
+ Brief: Writes the Makefile. Should be called after all components are added
+ Param:
+ - self The Makefile object
+ """
+ # Get the list of all of the libraries
+ sd=self.createLibstring(self.c,self.r,self.o)
+ libstring=" "
+ for i in sd:
+ lib=i[0]
+ libstring = libstring+lib+"/lib"+lib+".a "
+
+ # Open the Makefile for Writing
+ with open(self.filePath+"/Makefile","w") as fh:
+ # Write the header information for the Makefile
+ fh.write("# Makefile for "+self.e+"\n")
+ fh.write("SRCROOT = "+self.src+"/\n")
+ fh.write("BUILDROOT = "+self.bld+"/\n")
+ fh.write("MK_TEMPLATE = "+self.template+"\n")
+ fh.write("include $(MK_TEMPLATE)"+"\n")
+
+ # Write the main experiment compile
+ fh.write(self.e+".x: "+libstring+"\n")
+ fh.write("\t$(LD) $^ $(LDFLAGS) -o $@ $(STATIC_LIBS)"+"\n")
+
+ # Write the link line script with user-provided libraries
+ if self.l:
+ linklineBuild(self)
+
+ # Write the individual component library compiles
+ with open(self.filePath+"/Makefile","a") as fh:
+ for (c,r,o) in sd:
+ libstring = " "
+ for lib in r:
+ libstring = libstring+lib+"/lib"+lib+".a "
+ cstring = c+"/lib"+c+".a: "
+ fh.write(cstring+libstring+" FORCE"+"\n")
+ if o == "":
+ fh.write("\t$(MAKE) SRCROOT=$(SRCROOT) BUILDROOT=$(BUILDROOT) MK_TEMPLATE=$(MK_TEMPLATE) --directory="+c+" $(@F)\n")
+ else:
+ fh.write("\t$(MAKE) SRCROOT=$(SRCROOT) BUILDROOT=$(BUILDROOT) MK_TEMPLATE=$(MK_TEMPLATE) "+o+" --directory="+c+" $(@F)\n")
+ fh.write("FORCE:\n")
+ fh.write("\n")
+
+ # Set up the clean
+ fh.write("clean:\n")
+ for c in self.c:
+ fh.write("\t$(MAKE) --directory="+c+" clean\n")
+
+ # Set up localize
+ fh.write("localize:\n")
+ for c in self.c:
+ fh.write("\t$(MAKE) -f $(BUILDROOT)"+c+" localize\n")
+
+ # Set up distclean
+ fh.write("distclean:\n")
+ for c in self.c:
+ fh.write("\t$(RM) -r "+c+"\n")
+ fh.write("\t$(RM) -r "+self.e+"\n")
+ fh.write("\t$(RM) -r Makefile \n")
+
+### This seems incomplete? ~ ejs
+## The makefile class for a container. It gets built into a temporary directory so it can be copied
+## into the container.
+## \param exp Experiment name
+## \param libs Additional libraries/linker flags defined by user
+## \param srcDir The path to the source directory
+## \param bldDir The path to the build directory
+## \param mkTemplatePath The path of the template .mk file for compiling
+## \param tmpDir A local path to temporarily store files build to be copied to the container
+class makefileContainer(makefile):
+ def __init__(self,exp,libs,srcDir,bldDir,mkTemplatePath,tmpDir):
+ self.e = exp
+ self.l = libs
+ self.src = srcDir
+ self.bld = bldDir
+ self.template = mkTemplatePath
+ self.tmpDir = tmpDir
+ self.c =[] #components
+ self.r=[] #requires
+ self.o=[] #overrides
+ os.system("mkdir -p "+self.tmpDir)
+ self.filePath = self.tmpDir # Needed so that the container and bare metal builds can
+ # use the same function to create the Makefile
+
+ def getTmpDir(self):
+ """
+ Brief: Return the tmpDir
+ Param:
+ - self The makefile object
+ """
+ return self.tmpDir
diff --git a/lib/fre/make/gfdlfremake/platformfre.py b/lib/fre/make/gfdlfremake/platformfre.py
new file mode 100644
index 00000000..4f8d0eed
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/platformfre.py
@@ -0,0 +1,108 @@
+import yaml
+
+class platforms ():
+ def __init__(self,platforminfo):
+ """
+ Param:
+ - self The platform yaml object
+ - platforminfo dictionary with platform information
+ from the combined yaml
+ """
+ self.yaml = platforminfo
+
+ ## Check the yaml for errors/omissions
+ ## Loop through the platforms
+ for p in self.yaml:
+ ## Check the platform name
+ try:
+ p["name"]
+ except:
+ raise Exception("At least one of the platforms is missing a name in "+fname+"\n")
+ ## Check the compiler
+ try:
+ p["compiler"]
+ except:
+ raise Exception("You must specify a compiler in your "+p["name"]+" platform in the file "+fname+"\n")
+ ## Check for the Fortran (fc) and C (cc) compilers
+ try:
+ p["fc"]
+ except:
+ raise Exception("You must specify the name of the Fortran compiler as fc on the "+p["name"]+" platform in the file "+fname+"\n")
+ try:
+ p["cc"]
+ except:
+ raise Exception("You must specify the name of the Fortran compiler as cc on the "+p["name"]+" platform in the file "+fname+"\n")
+ ## Check for modules to load
+ try:
+ p["modules"]
+ except:
+ p["modules"]=[""]
+ ## Check for modulesInit to set up the modules environment
+ try:
+ p["modulesInit"]
+ except:
+ p["modulesInit"]=[""]
+ ## Get the root for the build
+ try:
+ p["modelRoot"]
+ except:
+ p["modelRoot"] = "/apps"
+ ## Check if we are working with a container and get the info for that
+ try:
+ p["container"]
+ except:
+ p["container"] = False
+ p["RUNenv"] = [""]
+ p["containerBuild"] = ""
+ p["containerRun"] = ""
+ if p["container"]:
+ ## Check the container builder
+ try:
+ p["containerBuild"]
+ except:
+ raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
+ if p["containerBuild"] != "podman" and p["containerBuild"] != "docker":
+ raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
+ ## Check for container environment set up for RUN commands
+ try:
+ p["RUNenv"]
+ except:
+ p["RUNenv"] = ""
+ ## Check the container runner
+ try:
+ p["containerRun"]
+ except:
+ raise Exception("You must specify the program used to run the container (containerRun) on the "+p["name"]+" platform in the file "+fname+"\n")
+ if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity":
+ raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n")
+ ## set the location of the mkTemplate.
+ ## In a container, it uses the hpc-me template cloned from mkmf
+ p["mkTemplate"] = "/apps/mkmf/templates/hpcme-intel21.mk"
+ else:
+ try:
+ p["mkTemplate"]
+ except:
+ raise ValueError("The non-container platform "+p["name"]+" must specify a mkTemplate \n")
+
+ def hasPlatform(self,name):
+ """
+ Brief: Checks if the platform yaml has the named platform
+ """
+ for p in self.yaml:
+ if p["name"] == name:
+ return True
+ return False
+
+ def getPlatformsYaml(self):
+ """
+ Brief: Get the platform yaml
+ """
+ return self.yaml
+
+ def getPlatformFromName(self,name):
+ """
+ Brief: Get the platform information from the name of the platform
+ """
+ for p in self.yaml:
+ if p["name"] == name:
+ return (p["compiler"], p["modules"], p["modulesInit"], p["fc"], p["cc"], p["modelRoot"],p["container"], p["mkTemplate"],p["containerBuild"], p["containerRun"], p["RUNenv"])
diff --git a/lib/fre/make/gfdlfremake/targetfre.py b/lib/fre/make/gfdlfremake/targetfre.py
new file mode 100644
index 00000000..0b2b5964
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/targetfre.py
@@ -0,0 +1,81 @@
+class fretarget:
+ """
+ Class: Stores information about the target
+ """
+ def __init__(self,t):
+ """
+ Brief: Sets up information about the target and handles errors
+ Note: The default target is prod
+ Param:
+ - self the fretarget object
+ - t The target string
+ """
+ self.target = t # The target string
+
+ ## Split the target string
+ targ = self.target.split('-')
+ self.makeline_add = ""
+ self.debug = False
+ self.repro = False
+ self.prod = False
+
+ ## Parse the target string for prod, repro, and debug. Set up what to add to the
+ ## make line during compile when using mkmf builds
+ for target in targ:
+ if target == "debug":
+ targ = target.upper()
+ self.makeline_add = self.makeline_add + targ + "=on "
+ self.debug = True
+ elif target == "prod":
+ targ = target.upper()
+ self.makeline_add = self.makeline_add + targ + "=on "
+ self.prod = True
+ elif target == "repro":
+ targ = target.upper()
+ self.makeline_add = self.makeline_add + targ + "=on "
+ self.repro = True
+
+ ## Check to see if openmp is included in the target and add that to the makeline add string
+ if target == "openmp":
+ targ = target.upper()
+ self.makeline_add = self.makeline_add + targ + "=on "
+ self.openmp = True
+ else:
+ self.openmp = False
+
+ ## Check to make sure only one of the prod, debug, repro are used
+ errormsg = "You can only list one mutually exclusive target, but your target '"+self.target+"' lists more than one of the following targets: \n debug \n prod \n repro"
+ if self.debug:
+ try:
+ if self.repro or self.prod == True:
+ raise ValueError(errormsg)
+ except ValueError:
+ raise
+ elif self.repro:
+ try:
+ if self.prod == True:
+ raise ValueError(errormsg)
+ except ValueError:
+ raise
+ else:
+ try:
+ if self.prod == False:
+ raise ValueError("Your target '"+self.target+"' needs to include one of the following: prod, repro, debug")
+ except ValueError:
+ raise
+
+ def gettargetName(self):
+ """
+ Brief: Returns the name of the target
+ Param:
+ - self The fretarget object
+ """
+ return self.target
+
+ def getmakeline_add(self):
+ """
+ Brief: Returns the makeline_add
+ Param:
+ - self The fretarget object
+ """
+ return self.makeline_add
diff --git a/lib/fre/make/gfdlfremake/varsfre.py b/lib/fre/make/gfdlfremake/varsfre.py
new file mode 100644
index 00000000..c153e845
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/varsfre.py
@@ -0,0 +1,56 @@
+import yaml
+import os
+import re
+
+## Removes enclosing braces or parentheses and optionally leading dollar sign from a string
+## \param inString A string from which characters will be removed
+## \param bound A set of characters to be removed from the beginning and end of the string
+## \param leadingDollar (optional) If True, removed leading dollar sign
+def removeEnclosing(inString, bound='()', leadingDollar=True):
+ if leadingDollar:
+ inString = inString.lstrip('$')
+ return inString.lstrip(bound[0]).rstrip(bound[1])
+
+## Retrieves an environmental variable based on a string of the form $(VAR)
+## \param inString A string that specifies the variable to be retrieved
+def getEnvSub(inString):
+ return os.getenv(removeEnclosing(inString.group(), bound='{}'))
+
+## Replaces all instances of Linux environment variables (${VAR}) in a string using getEnvSub
+## \param string A string to have the variables replaced
+def envReplace(string):
+ return re.sub("\$\{\w+\}", getEnvSub, string)
+
+## Reads stores and replaces the fre variables set in the model YAML
+class frevars():
+## Grabs the FRE variables from the model yaml
+## \param self The frevars object
+## \param y The model yaml file name
+ def __init__(self,y):
+ with open(y, 'r') as file:
+ self.modelyaml = yaml.safe_load(file)
+## Substitutes fre variables with the format $(variable) into the input string
+## \string The string that contains the variables
+## \returns String with the fre variables filled in
+ def freVarReplace(self,string):
+ ## Retrieves a value from the modelyaml based on a key of the form $(VAR)
+ ## \param inString A string that specifies the value to be retrieved
+ def getVarYamlSub(inString):
+ return self.modelyaml[removeEnclosing(inString.group())]
+ return re.sub("\$\(\w+\)", getVarYamlSub, string)
+
+## Wrapper that relaces environment ${} and FRE $() variables
+## \param self the FRE yaml varaibles (FRE properties)
+## \param string The YAML string that is having its variables replaced
+## \returns string with the environment and FRE variables replaced
+ def freVarSub(self, string):
+ tmpString = envReplace(string)
+ returnString = self.freVarReplace(tmpString)
+ return returnString
+
+## Wrapper that takes in a string (yaml) and fills in the FRE and Environment variables
+## \param y Path to yaml file whose variables need to be filled in
+ def fillInYamlWithVars(self,y):
+ with open(y, 'r') as file:
+ yamlString=read(file)
+ return self.freVarSub(yamlString)
diff --git a/lib/fre/make/gfdlfremake/yamlfre.py b/lib/fre/make/gfdlfremake/yamlfre.py
new file mode 100644
index 00000000..72458a8c
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamlfre.py
@@ -0,0 +1,199 @@
+import os
+import json
+from pathlib import Path
+import yaml
+from jsonschema import validate, ValidationError, SchemaError
+from . import platformfre
+
+def parseCompile(fname,v):
+ """
+ Brief: Open the yaml file and parse as fremakeYaml
+ Param:
+ - fname the name of the yaml file to parse
+ - v the FRE yaml variables
+ """
+ # Open the yaml file and parse as fremakeYaml
+ with open(fname, 'r') as yamlfile:
+ y = yaml.safe_load(v.freVarSub(yamlfile.read()))
+
+ return y
+
+##### THIS SEEMS UNUSED
+## \brief Checks the yaml for variables. Required variables will dump and error. Non-required variables will
+## set a default value for the variable
+#def yamlVarCheck(var,val="",req=False,err="error"):
+# """
+# Brief: Checks the yaml for variables. Required variables will dump and error.
+# Non-required variables will set a default value for the variable
+# Param:
+# - var A variable in the yaml
+# - val a default value for var
+# - req if true, the variable is required in the yaml and an exception will be raised
+# - err An error message to print if the variable is required and doesn't exist
+# """
+# try:
+# var
+# except:
+# if req:
+# print (err)
+# raise
+# else:
+# var = val
+
+class compileYaml():
+ """
+ Brief: This will read the compile yaml for FRE and then fill in any of the missing non-required variables
+ """
+ def __init__(self,compileinfo):
+ """
+ Brief: Read get the compile yaml and fill in the missing pieces
+ Param:
+ - self the compile Yaml object
+ - compileinfo dictionary with compile information from the combined yaml
+ """
+ # compile information from the combined yaml
+ self.yaml = compileinfo
+
+ ## Check the yaml for required things
+ ## Check for required experiment name
+ try:
+ self.yaml["experiment"]
+ except:
+ print("You must set an experiment name to compile \n")
+ raise
+ ## Check for optional libraries and packages for linking in container
+ try:
+ self.yaml["container_addlibs"]
+ except:
+ self.yaml["container_addlibs"]=""
+ ## Check for optional libraries and packages for linking on bare-metal system
+ try:
+ self.yaml["baremetal_linkerflags"]
+ except:
+ self.yaml["baremetal_linkerflags"]=""
+ ## Check for required src
+ try:
+ self.yaml["src"]
+ except:
+ print("You must set a src to specify the sources in modelRoot/"+self.yaml["experiment"]+"\n")
+ raise
+ ## Loop through the src array
+ for c in self.yaml['src']:
+ ## Check for required componenet name
+ try:
+ c['component']
+ except:
+ print("You must set the 'componet' name for each src component")
+ raise
+ ## Check for required repo url
+ try:
+ c['repo']
+ except:
+ print("'repo' is missing from the component "+c['component']+" in "+self.yaml["experiment"]+"\n")
+ raise
+ # Check for optional branch. Otherwise set it to blank
+ try:
+ c['branch']
+ except:
+ c['branch']=""
+ # Check for optional cppdefs. Otherwise set it to blank
+ try:
+ c['cppdefs']
+ except:
+ c['cppdefs']=""
+ # Check for optional doF90Cpp. Otherwise set it to False
+ try:
+ c['doF90Cpp']
+ except:
+ c['doF90Cpp']=False
+ # Check for optional additional instructions. Otherwise set it to blank
+ try:
+ c['additionalInstructions']
+ except:
+ c['additionalInstructions']=""
+ # Check for optional paths. Otherwise set it to blank
+ try:
+ c['paths']
+ except:
+ c['paths']=[c['component']]
+ # Check for optional requires. Otherwise set it to blank
+ try:
+ c['requires']
+ except:
+ c['requires']=[]
+ # Check for optional overrides. Otherwise set it to blank
+ try:
+ c['makeOverrides']
+ except:
+ c['makeOverrides']=""
+ # Check for optional flags. Otherwise set it to blank.
+ try:
+ c["otherFlags"]
+ except:
+ c["otherFlags"]=""
+
+ def getCompileYaml(self):
+ """
+ Brief: Returns the compile yaml
+ """
+ try:
+ self.yaml
+ except:
+ print ("You must initialize the compile YAML object before you try to get the yaml \n")
+ raise
+ return self.yaml
+
+class freyaml():
+ """
+ Brief: This will take the combined yaml file, parse information, and fill in missing variables
+ to make the full freyaml that can be used and checked
+ Note:
+ - platformYaml: platforms.yaml
+ - compileYaml: compile.yaml
+ """
+ def __init__(self,combinedyaml,v):
+ """
+ Param:
+ - self The freyaml object
+ - combinedyaml The name of the combined yaml file
+ - v FRE yaml variables
+ """
+ self.combinedfile = combinedyaml
+
+ self.freyaml = parseCompile(self.combinedfile, v)
+
+ #get compile info
+ self.compiledict = self.freyaml.get("compile")
+ self.compile = compileYaml(self.compiledict)
+ self.compileyaml = self.compile.getCompileYaml()
+
+ #self.freyaml.update(self.compileyaml)
+
+ #get platform info
+ self.platformsdict = self.freyaml.get("platforms")
+ self.platforms = platformfre.platforms(self.platformsdict)
+ self.platformsyaml = self.platforms.getPlatformsYaml()
+
+ #self.freyaml.update(self.platformsyaml)
+
+ ## VALIDATION OF COMBINED YAML FOR COMPILATION
+ fremake_package_dir = Path(__file__).resolve().parents[2]
+ schema_path = os.path.join(fremake_package_dir, 'gfdl_msd_schemas', 'FRE', 'fre_make.json')
+ with open(schema_path, 'r') as f:
+ s = f.read()
+ schema = json.loads(s)
+
+ validate(instance=self.freyaml,schema=schema)
+ print("\nCOMBINED YAML VALID")
+
+ def getCompileYaml(self):
+ """
+ Brief: Returns the compile yaml
+ """
+ return self.compileyaml
+
+ def getPlatformsYaml(self):
+ """
+ Brief: Returns the compile yaml
+ """
+ return self.platformsyaml
diff --git a/lib/fre/make/gfdlfremake/yamls/SHiELD_example/SHiELD.yaml b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/SHiELD.yaml
new file mode 100644
index 00000000..9f063d0a
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/SHiELD.yaml
@@ -0,0 +1,10 @@
+platformYaml: platforms.yaml
+compileYaml: compile.yaml
+fv3_release: main
+phy_release: main
+fms_release: "2023.02"
+drivers_release: main
+coupler_release: "2023.02"
+FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+INTEL: intel-classic
diff --git a/lib/fre/make/gfdlfremake/yamls/SHiELD_example/compile.yaml b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/compile.yaml
new file mode 100644
index 00000000..a83bb1ce
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/compile.yaml
@@ -0,0 +1,38 @@
+experiment: shield_nh
+compileInclude: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+container_addlibs: ["bacio","sp","w3emc","w3nco"]
+baremetal_addlibs: ["-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_4", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_8", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/sp-2.5.0-7bumbmx/lib64 -lsp_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3emc-2.10.0-zmuykep/lib64 -lw3emc_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3nco-2.4.1-76qm6h2/lib -lw3nco_d"]
+src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DHAVE_SCHED_GETAFFINITY -DINTERNAL_FILE_NML -DGFS_PHYS -DGFS_CONSTANTS -DHAVE_GETTID"
+ branch: "$(fms_release)"
+ - component: "SHiELD_physics"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/SHiELD_physics.git"
+ branch: "$(phy_release)"
+ paths: [SHiELD_physics/gsmphys,
+ SHiELD_physics/GFS_layer,
+ SHiELD_physics/IPD_layer]
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ otherFlags: "$(FMSincludes)"
+ - component: "fv3"
+ requires: ["FMS", "SHiELD_physics"]
+ repo: ["https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git",
+ "https://github.com/NOAA-GFDL/atmos_drivers.git"]
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ branch: ["$(fv3_release)","$(drivers_release)"]
+ paths: [SHiELD_physics/FV3GFS/,
+ fv3/atmos_drivers/SHiELD/atmos_model.F90,
+ fv3/GFDL_atmos_cubed_sphere/driver/SHiELD/atmosphere.F90,
+ fv3/GFDL_atmos_cubed_sphere/tools/,
+ fv3/GFDL_atmos_cubed_sphere/model/,
+ fv3/GFDL_atmos_cubed_sphere/GFDL_tools/fv_diag_column.F90]
+ otherFlags: "$(FMSincludes)"
+ - component: "FMScoupler"
+ requires: ["FMS", "SHiELD_physics", "fv3"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ branch: "$(coupler_release)"
+ paths: ["FMScoupler/SHiELD/coupler_main.F90"]
+ otherFlags: "$(FMSincludes)"
diff --git a/lib/fre/make/gfdlfremake/yamls/SHiELD_example/platforms.yaml b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/platforms.yaml
new file mode 100644
index 00000000..9f72043b
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/SHiELD_example/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2022.2.1","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/SHiELDtest
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2023.1.0","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/SHiELDtest
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.12.1"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/gfdlfremake/yamls/am5.yaml b/lib/fre/make/gfdlfremake/yamls/am5.yaml
new file mode 100644
index 00000000..4b7bf8d4
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/am5.yaml
@@ -0,0 +1,6 @@
+platformYaml: platforms.yaml
+compileYaml: compile.yaml
+release: f1a1r1
+INTEL: "intel-classic"
+FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
diff --git a/lib/fre/make/gfdlfremake/yamls/compile.yaml b/lib/fre/make/gfdlfremake/yamls/compile.yaml
new file mode 100644
index 00000000..5200599c
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/compile.yaml
@@ -0,0 +1,66 @@
+experiment: "am5"
+container_addlibs:
+baremetal_linkerflags:
+src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF"
+ branch: "2022.01"
+ cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF"
+ otherFlags: "$(FMSincludes)"
+ - component: "am5_phys"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git"
+ branch: "2022.01"
+ otherFlags: "$(FMSincludes)"
+ - component: "GFDL_atmos_cubed_sphere"
+ requires: ["FMS", "am5_phys"]
+ repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML"
+ branch: "2022.01"
+ paths: ["GFDL_atmos_cubed_sphere/driver/GFDL",
+ "GFDL_atmos_cubed_sphere/model",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90",
+ "GFDL_atmos_cubed_sphere/tools",
+ "GFDL_atmos_cubed_sphere/GFDL_tools"]
+ otherFlags: "$(FMSincludes)"
+ - component: "atmos_drivers"
+ requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"]
+ repo: "https://github.com/NOAA-GFDL/atmos_drivers.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE"
+ branch: "2022.01"
+ paths: ["atmos_drivers/coupled"]
+ otherFlags: "$(FMSincludes)"
+ - component: "ice_sis"
+ requires: ["FMS", "ice_param", "mom6"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git"
+ branch: "2021.02"
+ otherFlags: "$(FMSincludes) $(momIncludes)"
+ - component: "ice_param"
+ repo: "https://github.com/NOAA-GFDL/ice_param.git"
+ cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF"
+ branch: "2021.02"
+ requires: ["FMS", "mom6"]
+ otherFlags: "$(FMSincludes) $(momIncludes)"
+ - component: "land_lad2"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git"
+ branch: "2022.01"
+ branch: "land_lad2_2021.02"
+ doF90Cpp: True
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: "$(FMSincludes)"
+ - component: "mom6"
+ requires: ["FMS"]
+ paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"]
+ branch: ["2021.02","dev/gfdl/2018.04.06"]
+ repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"]
+ makeOverrides: 'OPENMP=""'
+ otherFlags: "$(FMSincludes) $(momIncludes)"
+ - component: "FMScoupler"
+ paths: ["FMScoupler/full", "FMScoupler/shared"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ branch: "2022.01"
+ requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"]
+ otherFlags: "$(FMSincludes) $(momIncludes)"
diff --git a/lib/fre/make/gfdlfremake/yamls/platforms.yaml b/lib/fre/make/gfdlfremake/yamls/platforms.yaml
new file mode 100644
index 00000000..02b7d222
--- /dev/null
+++ b/lib/fre/make/gfdlfremake/yamls/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2022.2.1","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2023.1.0","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/runFremake.py b/lib/fre/make/runFremake.py
new file mode 100644
index 00000000..ffe2ec96
--- /dev/null
+++ b/lib/fre/make/runFremake.py
@@ -0,0 +1,212 @@
+#!/usr/bin/python3
+'''
+date 2023
+author(s): Tom Robinson, Dana Singh, Bennett Chang
+fre make is used to create, run and checkout code, and compile a model.
+'''
+
+import os
+import logging
+from multiprocessing.dummy import Pool
+from pathlib import Path
+import click
+import fre.yamltools.combine_yamls as cy
+from .gfdlfremake import (
+ targetfre, varsfre, yamlfre, checkout,
+ makefilefre, buildDocker, buildBaremetal )
+
+def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+ ''' run fremake via click'''
+ yml = yamlfile
+ name = yamlfile.split(".")[0]
+ nparallel = parallel
+ jobs = str(jobs)
+ pcheck = no_parallel_checkout
+
+ if pcheck:
+ pc = ""
+ else:
+ pc = " &"
+
+ if verbose:
+ logging.basicConfig(level=logging.INFO)
+ else:
+ logging.basicConfig(level=logging.ERROR)
+
+ #### Main
+ srcDir="src"
+ checkoutScriptName = "checkout.sh"
+ baremetalRun = False # This is needed if there are no bare metal runs
+
+ ## Split and store the platforms and targets in a list
+ plist = platform
+ tlist = target
+
+ # Combined compile yaml file
+ combined = Path(f"combined-{name}.yaml")
+
+ ## If combined yaml exists, note message of its existence
+ ## If combined yaml does not exist, combine model, compile, and platform yamls
+ full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
+
+ ## Get the variables in the model yaml
+ freVars = varsfre.frevars(full_combined)
+
+ ## Open the yaml file and parse as fremakeYaml
+ modelYaml = yamlfre.freyaml(full_combined,freVars)
+ fremakeYaml = modelYaml.getCompileYaml()
+
+ ## Error checking the targets
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+
+ ## Loop through the platforms specified on the command line
+ ## If the platform is a baremetal platform, write the checkout script and run it once
+ ## This should be done separately and serially because bare metal platforms should all be using
+ ## the same source code.
+ for platformName in plist:
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError(f'{platformName} does not exist in '
+ f'{modelYaml.combined.get("compile").get("platformYaml")}')
+
+ ( compiler, modules, modulesInit,
+ fc, cc, modelRoot, iscontainer,
+ mkTemplate, containerBuild, ContainerRun,
+ RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Create the checkout script
+ if not iscontainer:
+ ## Create the source directory for the platform
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+ if not os.path.exists(srcDir):
+ os.system("mkdir -p " + srcDir)
+ if not os.path.exists(srcDir+"/checkout.sh"):
+ freCheckout = checkout.checkout("checkout.sh",srcDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+ os.chmod(srcDir+"/checkout.sh", 0o744)
+ ## TODO: Options for running on login cluster?
+ freCheckout.run()
+
+ fremakeBuildList = []
+ ## Loop through platforms and targets
+ for platformName in plist:
+ for targetName in tlist:
+ target = targetfre.fretarget(targetName)
+ if modelYaml.platforms.hasPlatform(platformName):
+ pass
+ else:
+ raise ValueError (platformName + " does not exist in " + modelYaml.platformsfile)
+ ( compiler, modules, modulesInit,
+ fc, cc, modelRoot, iscontainer,
+ mkTemplate, containerBuild, containerRun,
+ RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName)
+
+ ## Make the source directory based on the modelRoot and platform
+ srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
+
+ ## Check for type of build
+ if not iscontainer:
+ baremetalRun = True
+ ## Make the build directory based on the modelRoot, the platform, and the target
+ bldDir = f'{modelRoot}/{fremakeYaml["experiment"]}/' + \
+ f'{platformName}-{target.gettargetName()}/exec'
+ os.system("mkdir -p " + bldDir)
+
+ ## Create the Makefile
+ freMakefile = makefilefre.makefile(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["baremetal_linkerflags"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate)
+
+
+ # Loop through components, send component name/requires/overrides for Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+
+ ## Create a list of compile scripts to run in parallel
+ fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"],
+ mkTemplatePath = mkTemplate,
+ srcDir = srcDir,
+ bldDir = bldDir,
+ target = target,
+ modules = modules,
+ modulesInit = modulesInit,
+ jobs = jobs)
+
+ for c in fremakeYaml['src']:
+ fremakeBuild.writeBuildComponents(c)
+ fremakeBuild.writeScript()
+ fremakeBuildList.append(fremakeBuild)
+ ## Run the build
+ fremakeBuild.run()
+ else:
+ ###################### container stuff below #######################################
+ ## Run the checkout script
+ # image="hpc-me-intel:2021.1.1"
+ image="ecpe4s/noaa-intel-prototype:2023.09.25"
+ bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
+ tmpDir = "tmp/"+platformName
+
+ ## Create the checkout script
+ freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir)
+ freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
+ freCheckout.finish(pc)
+
+ ## Create the makefile
+ ### Should this even be a separate class from "makefile" in makefilefre? ~ ejs
+ freMakefile = makefilefre.makefileContainer(exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ srcDir = srcDir,
+ bldDir = bldDir,
+ mkTemplatePath = mkTemplate,
+ tmpDir = tmpDir)
+
+ # Loop through components and send the component name and requires for the Makefile
+ for c in fremakeYaml['src']:
+ freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides'])
+ freMakefile.writeMakefile()
+
+ ## Build the dockerfile
+ dockerBuild = buildDocker.container(base = image,
+ exp = fremakeYaml["experiment"],
+ libs = fremakeYaml["container_addlibs"],
+ RUNenv = RUNenv,
+ target = target)
+
+ dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
+ dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile",
+ freMakefile.getTmpDir() + "/linkline.sh")
+
+ for c in fremakeYaml['src']:
+ dockerBuild.writeDockerfileMkmf(c)
+
+ dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh")
+
+ ## Run the dockerfile; build the container
+ dockerBuild.build(containerBuild,containerRun)
+
+ #freCheckout.cleanup()
+ #buildDockerfile(fremakeYaml,image)
+
+ if baremetalRun:
+ if __name__ == '__main__':
+ # Create a multiprocessing Pool
+ pool = Pool(processes=nparallel)
+ # process data_inputs iterable with pool
+ pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
+
+@click.command()
+def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+ '''
+ Decorator for calling _fremake_run - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose)
+
+if __name__ == "__main__":
+ fremake_run()
diff --git a/lib/fre/make/tests/AM5_example/am5.yaml b/lib/fre/make/tests/AM5_example/am5.yaml
new file mode 100644
index 00000000..359755ec
--- /dev/null
+++ b/lib/fre/make/tests/AM5_example/am5.yaml
@@ -0,0 +1,104 @@
+# reusable variables
+fre_properties:
+ - &AM5_VERSION "am5f7b12r1"
+ - &FRE_STEM !join [am5/, *AM5_VERSION]
+
+ # amip
+ - &EXP_AMIP_START "1979"
+ - &EXP_AMIP_END "2020"
+ - &ANA_AMIP_START "1980"
+ - &ANA_AMIP_END "2020"
+
+ - &PP_AMIP_CHUNK96 "1yr"
+ - &PP_AMIP_CHUNK384 "1yr"
+ - &PP_XYINTERP96 "180,288"
+ - &PP_XYINTERP384 "720,1152"
+
+ # climo
+ - &EXP_CLIMO_START96 "0001"
+ - &EXP_CLIMO_END96 "0011"
+ - &ANA_CLIMO_START96 "0002"
+ - &ANA_CLIMO_END96 "0011"
+
+ - &EXP_CLIMO_START384 "0001"
+ - &EXP_CLIMO_END384 "0006"
+ - &ANA_CLIMO_START384 "0002"
+ - &ANA_CLIMO_END384 "0006"
+
+ # coupled
+ - &PP_CPLD_CHUNK_A "5yr"
+ - &PP_CPLD_CHUNK_B "20yr"
+
+ # grids
+ - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+
+ # compile information
+ - &release "f1a1r1"
+ - &INTEL "intel-classic"
+ - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+ - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+
+build:
+ # compile information
+ compileYaml: "compile.yaml"
+ platformYaml: "yaml_include/platforms.yaml"
+
+shared:
+ # directories shared across tools
+ directories: &shared_directories
+ history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+ pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+ analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+ ptmp_dir: "/xtmp/$USER/ptmp"
+ fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
+
+ # shared pp settings
+ postprocess:
+ settings: &shared_settings
+ history_segment: "P1Y"
+ site: "ppan"
+ switches: &shared_switches
+ do_statics: True
+ do_timeavgs: True
+ clean_work: True
+ do_refinediag: False
+ do_atmos_plevel_masking: True
+ do_preanalysis: False
+ do_analysis: True
+
+experiments:
+ - name: "c96L65_am5f7b12r1_amip"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
+ - name: "c96L65_am5f7b12r1_pdclim1850F"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ - name: "c96L65_am5f7b12r1_pdclim2010F"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ - name: "c96L65_am5f7b12r1_pdclim2010AERF"
+ pp:
+ - "yaml_include/pp.c96_clim.yaml"
+ - name: "c384L65_am5f7b12r1_amip"
+ pp:
+ - "yaml_include/pp.c384_amip.yaml"
+ - name: "c384L65_am5f7b12r1_pdclim2010F"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ - name: "c384L65_am5f7b12r1_pdclim1850F"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ - name: "c384L65_am5f7b12r1_pdclim2010AERF"
+ pp:
+ - "yaml_include/pp.c384_clim.yaml"
+ - name: "c384L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg"
+ pp:
+ - "yaml_include/pp.c384_amip.yaml"
+ - "yaml_include/pp.om4.yaml"
+ - name: "c96L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
+ - "yaml_include/pp.om4.yaml"
+ - name: "c96L65_am5f7b12r1_amip_cosp"
+ pp:
+ - "yaml_include/pp.c96_amip.yaml"
diff --git a/lib/fre/make/tests/AM5_example/compile.yaml b/lib/fre/make/tests/AM5_example/compile.yaml
new file mode 100644
index 00000000..5f9a361b
--- /dev/null
+++ b/lib/fre/make/tests/AM5_example/compile.yaml
@@ -0,0 +1,67 @@
+compile:
+ experiment: "am5"
+ container_addlibs:
+ baremetal_linkerflags:
+ src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF"
+ branch: "2022.01"
+ cppdefs: "-DHAVE_GETTID -Duse_libMPI -Duse_netCDF"
+ otherFlags: *FMSincludes
+ - component: "am5_phys"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/am5_phys.git"
+ branch: "2022.01"
+ otherFlags: *FMSincludes
+ - component: "GFDL_atmos_cubed_sphere"
+ requires: ["FMS", "am5_phys"]
+ repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML"
+ branch: "2022.01"
+ paths: ["GFDL_atmos_cubed_sphere/driver/GFDL",
+ "GFDL_atmos_cubed_sphere/model",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90",
+ "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90",
+ "GFDL_atmos_cubed_sphere/tools",
+ "GFDL_atmos_cubed_sphere/GFDL_tools"]
+ otherFlags: *FMSincludes
+ - component: "atmos_drivers"
+ requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere"]
+ repo: "https://github.com/NOAA-GFDL/atmos_drivers.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE"
+ branch: "2022.01"
+ paths: ["atmos_drivers/coupled"]
+ otherFlags: *FMSincludes
+ - component: "ice_sis"
+ requires: ["FMS", "ice_param", "mom6"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/ice_sis.git"
+ branch: "2021.02"
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
+ - component: "ice_param"
+ repo: "https://github.com/NOAA-GFDL/ice_param.git"
+ cppdefs: "-Duse_yaml -Duse_libMPI -Duse_netCDF"
+ branch: "2021.02"
+ requires: ["FMS", "mom6"]
+ otherFlags: !join [*FMSincludes," ", *momIncludes]
+ - component: "land_lad2"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git"
+ branch: "2022.01"
+ branch: "land_lad2_2021.02"
+ doF90Cpp: True
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "mom6"
+ requires: ["FMS"]
+ paths: ["mom6/MOM6-examples/src/MOM6/config_src/dynamic", "mom6/MOM6-examples/src/MOM6/config_src/coupled_driver", "mom6/MOM6-examples/src/MOM6/src/*/", "mom6/MOM6-examples/src/MOM6/src/*/*/", "mom6/ocean_BGC/generic_tracers", "mom6/ocean_BGC/mocsy/src"]
+ branch: ["2021.02","dev/gfdl/2018.04.06"]
+ repo: ["https://github.com/NOAA-GFDL/ocean_BGC.git","https://github.com/NOAA-GFDL/MOM6-examples.git"]
+ makeOverrides: 'OPENMP=""'
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
+ - component: "FMScoupler"
+ paths: ["FMScoupler/full", "FMScoupler/shared"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ branch: "2022.01"
+ requires: ["FMS", "atmos_drivers", "am5_phys", "land_lad2", "ice_sis", "ice_param", "mom6"]
+ otherFlags: !join [*FMSincludes, " ", *momIncludes]
diff --git a/lib/fre/make/tests/AM5_example/yaml_include/platforms.yaml b/lib/fre/make/tests/AM5_example/yaml_include/platforms.yaml
new file mode 100644
index 00000000..60d1aad2
--- /dev/null
+++ b/lib/fre/make/tests/AM5_example/yaml_include/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/lib/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml
new file mode 100644
index 00000000..117c66c6
--- /dev/null
+++ b/lib/fre/make/tests/AM5_example/yaml_include/pp.c96_amip.yaml
@@ -0,0 +1,88 @@
+# local reusable variable overrides
+fre_properties:
+ - &custom_interp "180,360"
+
+# directory overrides
+#c96_amip_directories:
+directories:
+ <<: *shared_directories
+ ptmp_dir: "/ptmp/$USER"
+ pp_grid_spec: *GRID_SPEC96
+
+#c96_amip_postprocess:
+postprocess:
+ # pp setting overrides
+ settings:
+ <<: *shared_settings
+ pp_start: *ANA_AMIP_START
+ pp_stop: *ANA_AMIP_END
+ pp_chunk_a: *PP_AMIP_CHUNK96
+ pp_components: "atmos atmos_scalar"
+ switches:
+ <<: *shared_switches
+ do_statics: False
+
+ # main pp instructions
+ components:
+ - type: "atmos_cmip"
+ sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *custom_interp
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos"
+ sources: "atmos_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_level_cmip"
+ sources: "atmos_level_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_level"
+ sources: "atmos_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_month_aer"
+ sources: "atmos_month_aer"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
+ - type: "atmos_diurnal"
+ sources: "atmos_diurnal"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order2"
+ inputRealm: 'atmos'
+ - type: "atmos_scalar"
+ sources: "atmos_scalar"
+ - type: "aerosol_cmip"
+ xyInterp: *PP_XYINTERP96
+ sources: "aerosol_month_cmip"
+ sourceGrid: "cubedsphere"
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
+ - type: "land"
+ sources: "land_month"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'land'
+ - type: "land_cmip"
+ sources: "land_month_cmip"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'land'
+ - type: "tracer_level"
+ sources: "atmos_tracer"
+ sourceGrid: "cubedsphere"
+ xyInterp: *PP_XYINTERP96
+ interpMethod: "conserve_order1"
+ inputRealm: 'atmos'
diff --git a/lib/fre/make/tests/ESM4_example/compile.yaml b/lib/fre/make/tests/ESM4_example/compile.yaml
new file mode 100644
index 00000000..ee51658d
--- /dev/null
+++ b/lib/fre/make/tests/ESM4_example/compile.yaml
@@ -0,0 +1,124 @@
+compile:
+ experiment: "esm4"
+ container_addlibs:
+ baremetal_linkerflags:
+ src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=500"
+ branch: *FMS_GIT_TAG
+ - component: "atmos_phys"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/atmos_phys.git"
+ branch: *ATM_PHYS_GIT_TAG
+ otherFlags: *FMSincludes
+ - component: "atmos_dyn"
+ requires: ["FMS", "atmos_phys"]
+ repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ branch: *ATM_FV3_GIT_TAG
+ paths: [ "atmos_dyn/driver/GFDL",
+ "atmos_dyn/model",
+ "atmos_dyn/model_nh_null",
+ "atmos_dyn/GFDL_tools",
+ "atmos_dyn/driver/SHiELD/cloud_diagnosis.F90",
+ "atmos_dyn/driver/SHiELD/gfdl_cloud_microphys.F90",
+ "atmos_dyn/tools" ]
+ - component: "atmos_drivers"
+ requires: ["FMS", "atmos_phys", "atmos_dyn"]
+ repo: "https://github.com/NOAA-GFDL/atmos_drivers.git"
+ cppdefs: "-DSPMD -DCLIMATE_NUDGE"
+ branch: *ATM_DRV_GIT_TAG
+ otherFlags: *FMSincludes
+ paths: ["atmos_drivers/coupled"]
+ - component: "lm4p"
+ requires: ["FMS"]
+ repo: "https://gitlab.gfdl.noaa.gov/FMS/lm4p.git"
+ branch: *LAND_GIT_TAG
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "mom6"
+ requires: ["FMS"]
+ repo: ["https://github.com/NOAA-GFDL/MOM6-examples.git",
+ "https://github.com/NOAA-GFDL/ocean_BGC.git" ]
+ branch: [ "dev/gfdl", *OCEAN_BGC_GIT_TAG ] # cant use property for mom6 since its a commit hash instead of a branch
+ otherFlags: !join [ *FMSincludes, " ", *MOMincludes ]
+ additionalInstructions: |
+ pushd mom6/MOM6-examples
+ git checkout 40e3937 # this is just the value of MOM6_GIT_TAG property, can't seem to use variable
+ popd
+ # this step might be covered by initial checkout since the default is recursive
+ git submodule update --recursive --init mom6/MOM6-examples/src/MOM6 mom6/MOM6-examples/src/SIS2 mom6/MOM6-examples/src/icebergs
+ # checkout dev/gfdl on the icebergs submodule
+ pushd mom6/MOM6-examples/src/icebergs
+ git checkout dev/gfdl
+ popd
+ # this is different than the MOM6_GIT_TAG above since its pulling a submodule not the whole repo
+ if [[ $MOM6_GIT_FIX ]]; then
+ echo WARNING: Checking out MOM6_GIT_FIX, set to: $MOM6_GIT_FIX
+ pushd mom6/MOM6-examples/src/MOM6/
+ git checkout $MOM6_GIT_FIX
+ popd
+ fi
+ if [[ $SIS2_GIT_FIX ]]; then
+ echo WARNING: Checking out SIS2_GIT_FIX, set to: $SIS2_GIT_FIX
+ pushd mom6/MOM6-examples/src/SIS2/
+ git checkout $SIS2_GIT_FIX
+ popd
+ fi
+ # link in dataset
+ pushd mom6/MOM6-examples
+ # skip the check for orion
+ #set platform_domain = `perl -T -e "use Net::Domain(hostdomain) ; print hostdomain"`
+ #if ("${platform_domain}" =~ *"MsState"* ) then
+ # ln -s /work/noaa/gfdlscr/pdata/gfdl/gfdl_O/datasets/ .datasets
+ #else
+ ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets
+ #endif
+ popd
+ test -e mom6/.datasets
+ if [[ $status != 0 ]]; then
+ echo ""; echo "" ; echo " WARNING: .datasets link in MOM6 examples directory is invalid"; echo ""; echo ""
+ fi
+ cppdefs: "-DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2"
+ paths: [ "mom6/MOM6-examples/src/MOM6/config_src/infra/FMS2",
+ "mom6/MOM6-examples/src/MOM6/config_src/memory/dynamic_symmetric",
+ "mom6/MOM6-examples/src/MOM6/config_src/drivers/FMS_cap",
+ "mom6/MOM6-examples/src/MOM6/src/ALE",
+ "mom6/MOM6-examples/src/MOM6/src/core",
+ "mom6/MOM6-examples/src/MOM6/src/diagnostics",
+ "mom6/MOM6-examples/src/MOM6/src/equation_of_state",
+ "mom6/MOM6-examples/src/MOM6/src/framework",
+ "mom6/MOM6-examples/src/MOM6/src/ice_shelf",
+ "mom6/MOM6-examples/src/MOM6/src/initialization",
+ "mom6/MOM6-examples/src/MOM6/src/ocean_data_assim",
+ "mom6/MOM6-examples/src/MOM6/src/parameterizations",
+ "mom6/MOM6-examples/src/MOM6/src/tracer",
+ "mom6/MOM6-examples/src/MOM6/src/user",
+ "mom6/MOM6-examples/src/MOM6/config_src/external/ODA_hooks",
+ "mom6/MOM6-examples/src/MOM6/config_src/external/database_comms",
+ "mom6/MOM6-examples/src/MOM6/config_src/external/drifters",
+ "mom6/MOM6-examples/src/MOM6/config_src/external/stochastic_physics",
+ "mom6/MOM6-examples/src/MOM6/config_src/external/stochastic_physics",
+ "mom6/ocean_BGC/generic_tracers",
+ "mom6/ocean_BGC/mocsy/src" ]
+ - component: "sis2"
+ requires: ["FMS", "mom6"]
+ repo: "https://github.com/NOAA-GFDL/ice_param.git"
+ branch: "2024.01"
+ cppdefs: "-DUSE_FMS2_IO"
+ otherFlags: !join [ *FMSincludes, " ", *MOMincludes ]
+ paths: [ "mom6/MOM6-examples/src/SIS2/config_src/dynamic_symmetric",
+ "mom6/MOM6-examples/src/SIS2/config_src/external/Icepack_interfaces",
+ "mom6/MOM6-examples/src/SIS2/src",
+ "mom6/MOM6-examples/src/icebergs/src",
+ "sis2" ]
+ - component: "coupler"
+ requires: ["FMS", "atmos_dyn", "atmos_drivers", "atmos_phys", "lm4p", "sis2", "mom6"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler"
+ branch: "2024.01"
+ otherFlags: !join [ *FMSincludes, " ", *MOMincludes ]
+ paths: [ "coupler/shared",
+ "coupler/full" ]
+
diff --git a/lib/fre/make/tests/ESM4_example/esm4.yaml b/lib/fre/make/tests/ESM4_example/esm4.yaml
new file mode 100644
index 00000000..436672ab
--- /dev/null
+++ b/lib/fre/make/tests/ESM4_example/esm4.yaml
@@ -0,0 +1,86 @@
+# esm4.2, based off ESM4p2_piControl_spinup_J_rts.xml
+# this needs -npc (non-parallel checkout) flag during checkout script creation for additional checkouts to work properly
+fre_properties:
+ - &RELEASE "2024.01"
+ - &FMS_GIT_TAG "2024.01"
+ - &ATM_PHYS_GIT_TAG "2024.01-alpha6"
+ - &ATM_FV3_GIT_TAG "2023.03"
+ - &ATM_DRV_GIT_TAG "2023.04"
+ - &LAND_GIT_TAG "2024.01"
+ - &ICE_PARAM_GIT_TAG "2023.04"
+ - &ESM4_VERSION "2024.01"
+ - &OCEAN_BGC_GIT_TAG "dev4.2_benthic"
+ - &MOM6_DATE "20231130"
+ - &MOM6_GIT_TAG "40e3937"
+ - &MOM6_GIT_FIX ""
+ - &SIS2_GIT_FIX ""
+ # compile
+ - &FRE_STEM !join [fre/FMS, *RELEASE, _mom6_, *MOM6_DATE]
+ - &INTEL "intel-classic"
+ - &FMSincludes "-IFMS/include"
+ - &MOMincludes "-Imom6/MOM6-examples/src/MOM6/src/framework"
+ # post processing
+ - &OM4_ANALYSIS "ice_ocean_SIS2/OM4_05"
+ - &PROD_SIMTIME "5" # simulation length in years
+ - &PP_CMIP_CHUNK_A "5yr" # smaller chunk length for pp/analysis
+ - &PP_CMIP_CHUNK_B "10yr" # bigger chunk length for pp/analysis
+ - &PP_START_YEAR "0001" # starting year
+ - &PROD_RUNTIME "16:00:00" # Maximum wall clock per simulation
+ - &PROD_SEGTIME "03:00:00" # Wall clock per segment (usually 1 year)
+ # MDBI settings
+ - &EXP_CPLD_START "0001" # simulation start year
+ - &EXP_CPLD_END "1000" # simulation end year
+ # this property wasn't actually used in the xml, but seems to be intended for specifying reference files to verify regression tests
+ - &reference_tag "FMS2022.03_mom6_20220703"
+ # these properties modify the build
+ - &BUILD_DATE "" # included as part of the src directory path
+ - &MODIFIER "" # appended to compile experiment name, can likely remove since compile experiments are no longer a thing
+ - &PLATINFO "" # used for awg_input
+ - &LIBS_ROOT "esm4.2_compile$(MODIFIER)" # used below
+ - &SRC_DIR !join [ $root/, *BUILD_DATE, /, *LIBS_ROOT, /src] # this ends up being used for input paths
+ - &MOM6_EXAMPLES !join [ $root/, *BUILD_DATE, /, *LIBS_ROOT, /src/mom6] # also for input paths
+ # input paths
+ - &AWG_INPUT_HOME_GAEA "awg_include"
+ - &AWG_INPUT_HOME_NESCC "awg_include"
+ - &AWG_INPUT_HOME_GFDL !join [ /nbhome/$USER/, *FRE_STEM, *PLATINFO, /$(name)/mdt_xml/awg_include]
+ - &USER_FILES_F2toF5 "/gpfs/f5/gfdl_f/world-shared/Niki.Zadeh/archive/input/f2_user_files_in_xmls"
+ - &append_to_setup_csh "" # The start year of forcing dataset. FRE hack to set fyear
+
+build:
+ compileYaml: "compile.yaml"
+ platformYaml: "platforms.yaml"
+
+shared:
+ # directories shared across tools
+ # shamelessly stolen from am5 example
+ directories: &shared_directories
+ history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+ pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+ analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+ ptmp_dir: "/xtmp/$USER/ptmp"
+ fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
+
+ # shared pp settings
+ # also shamelessly stolen from am5 example
+ postprocess:
+ settings: &shared_settings
+ history_segment: "P1Y"
+ site: "ppan"
+ switches: &shared_switches
+ do_statics: True
+ do_timeavgs: True
+ clean_work: True
+ do_refinediag: False
+ do_atmos_plevel_masking: True
+ do_preanalysis: False
+ do_analysis: True
+
+experiments:
+ - name: "ESM4p2_piControl_spinup_J"
+ pp:
+ - name: "ESM4p2_piControl_spinup_Jb"
+ pp:
+ - name: "ESM4p2_piControl_spinup_J_redoyr450_btmdiags"
+ pp:
+ - name: "ESM4p2_piControl_spinup_J_redoyr450"
+ pp:
diff --git a/lib/fre/make/tests/ESM4_example/platforms.yaml b/lib/fre/make/tests/ESM4_example/platforms.yaml
new file mode 100644
index 00000000..14d4dfff
--- /dev/null
+++ b/lib/fre/make/tests/ESM4_example/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL,".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/tests/SHiELD_example/SHiELD.yaml b/lib/fre/make/tests/SHiELD_example/SHiELD.yaml
new file mode 100644
index 00000000..9f063d0a
--- /dev/null
+++ b/lib/fre/make/tests/SHiELD_example/SHiELD.yaml
@@ -0,0 +1,10 @@
+platformYaml: platforms.yaml
+compileYaml: compile.yaml
+fv3_release: main
+phy_release: main
+fms_release: "2023.02"
+drivers_release: main
+coupler_release: "2023.02"
+FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+INTEL: intel-classic
diff --git a/lib/fre/make/tests/SHiELD_example/compile.yaml b/lib/fre/make/tests/SHiELD_example/compile.yaml
new file mode 100644
index 00000000..a83bb1ce
--- /dev/null
+++ b/lib/fre/make/tests/SHiELD_example/compile.yaml
@@ -0,0 +1,38 @@
+experiment: shield_nh
+compileInclude: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+container_addlibs: ["bacio","sp","w3emc","w3nco"]
+baremetal_addlibs: ["-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_4", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_8", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/sp-2.5.0-7bumbmx/lib64 -lsp_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3emc-2.10.0-zmuykep/lib64 -lw3emc_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3nco-2.4.1-76qm6h2/lib -lw3nco_d"]
+src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DHAVE_SCHED_GETAFFINITY -DINTERNAL_FILE_NML -DGFS_PHYS -DGFS_CONSTANTS -DHAVE_GETTID"
+ branch: "$(fms_release)"
+ - component: "SHiELD_physics"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/SHiELD_physics.git"
+ branch: "$(phy_release)"
+ paths: [SHiELD_physics/gsmphys,
+ SHiELD_physics/GFS_layer,
+ SHiELD_physics/IPD_layer]
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ otherFlags: "$(FMSincludes)"
+ - component: "fv3"
+ requires: ["FMS", "SHiELD_physics"]
+ repo: ["https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git",
+ "https://github.com/NOAA-GFDL/atmos_drivers.git"]
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ branch: ["$(fv3_release)","$(drivers_release)"]
+ paths: [SHiELD_physics/FV3GFS/,
+ fv3/atmos_drivers/SHiELD/atmos_model.F90,
+ fv3/GFDL_atmos_cubed_sphere/driver/SHiELD/atmosphere.F90,
+ fv3/GFDL_atmos_cubed_sphere/tools/,
+ fv3/GFDL_atmos_cubed_sphere/model/,
+ fv3/GFDL_atmos_cubed_sphere/GFDL_tools/fv_diag_column.F90]
+ otherFlags: "$(FMSincludes)"
+ - component: "FMScoupler"
+ requires: ["FMS", "SHiELD_physics", "fv3"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND"
+ branch: "$(coupler_release)"
+ paths: ["FMScoupler/SHiELD/coupler_main.F90"]
+ otherFlags: "$(FMSincludes)"
diff --git a/lib/fre/make/tests/SHiELD_example/platforms.yaml b/lib/fre/make/tests/SHiELD_example/platforms.yaml
new file mode 100644
index 00000000..9f72043b
--- /dev/null
+++ b/lib/fre/make/tests/SHiELD_example/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2022.2.1","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/SHiELDtest
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: ["$(INTEL)/2023.1.0","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
+ modelRoot: ${HOME}/fremake_canopy/SHiELDtest
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.12.1"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/tests/__init__.py b/lib/fre/make/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/make/tests/null_example/compile.yaml b/lib/fre/make/tests/null_example/compile.yaml
new file mode 100644
index 00000000..ab5052a0
--- /dev/null
+++ b/lib/fre/make/tests/null_example/compile.yaml
@@ -0,0 +1,48 @@
+compile:
+ experiment: "null_model_full"
+ container_addlibs:
+ baremetal_linkerflags:
+ src:
+ - component: "FMS"
+ repo: "https://github.com/NOAA-GFDL/FMS.git"
+ cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" # gettid flag is platform specific
+ otherFlags: "-fallow-argument-mismatch" # only needed for gcc
+ branch: *branch
+ - component: "atmos_null"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/atmos_null.git"
+ branch: *branch
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "land_null"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/land_null.git"
+ branch: *branch
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "ice_param"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/ice_param.git"
+ branch: *branch
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "ocean_null"
+ requires: ["FMS"]
+ repo: "https://github.com/NOAA-GFDL/ocean_null.git"
+ branch: *branch
+ otherFlags: *FMSincludes
+ - component: "ice_null"
+ requires: ["FMS", "ice_param", "ocean_null"]
+ repo: "https://github.com/NOAA-GFDL/ice_null.git"
+ branch: *branch
+ cppdefs: "-DINTERNAL_FILE_NML"
+ otherFlags: *FMSincludes
+ - component: "coupler"
+ requires: ["FMS", "ocean_null", "atmos_null", "land_null", "ice_param", "ocean_null", "ice_null"]
+ repo: "https://github.com/NOAA-GFDL/FMScoupler.git"
+ branch: *branch
+ otherFlags: *FMSincludes
+ cppdefs: "-D_USE_LEGACY_LAND_ -Duse_AM3_physics -DINTERNAL_FILE_NML"
+ paths: [ "coupler/full",
+ "coupler/shared" ]
+
diff --git a/lib/fre/make/tests/null_example/null_model.yaml b/lib/fre/make/tests/null_example/null_model.yaml
new file mode 100644
index 00000000..2dcbcad4
--- /dev/null
+++ b/lib/fre/make/tests/null_example/null_model.yaml
@@ -0,0 +1,15 @@
+# reusable variables
+fre_properties:
+ - &VERSION "full"
+ - &FRE_STEM !join [null_model/, *VERSION]
+ # compile information
+ - &branch "main"
+ - &INTEL "intel-classic"
+ - &FMSincludes "-IFMS/include"
+
+build:
+ compileYaml: "compile.yaml"
+ platformYaml: "platforms.yaml"
+
+experiments:
+ - name: "null_model_full"
diff --git a/lib/fre/make/tests/null_example/platforms.yaml b/lib/fre/make/tests/null_example/platforms.yaml
new file mode 100644
index 00000000..60d1aad2
--- /dev/null
+++ b/lib/fre/make/tests/null_example/platforms.yaml
@@ -0,0 +1,26 @@
+platforms:
+ - name: ncrc5.intel
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: ncrc5.intel23
+ compiler: intel
+ modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
+ modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
+ fc: ftn
+ cc: cc
+ mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"]
+ modelRoot: ${HOME}/fremake_canopy/test
+ - name: hpcme.2023
+ compiler: intel
+ RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"]
+ modelRoot: /apps
+ fc: mpiifort
+ cc: mpiicc
+ container: True
+ containerBuild: "podman"
+ containerRun: "apptainer"
diff --git a/lib/fre/make/tests/test_create_makefile.py b/lib/fre/make/tests/test_create_makefile.py
new file mode 100644
index 00000000..36188b33
--- /dev/null
+++ b/lib/fre/make/tests/test_create_makefile.py
@@ -0,0 +1,72 @@
+"""
+Test fre make create-makefile
+"""
+import os
+import shutil
+from pathlib import Path
+from fre.make import createMakefile
+
+# SET-UP
+test_dir = Path("fre/make/tests")
+NM_EXAMPLE = Path("null_example")
+YAMLFILE = "null_model.yaml"
+BM_PLATFORM = ["ncrc5.intel23"]
+CONTAINER_PLATFORM = ["hpcme.2023"]
+TARGET = ["debug"]
+EXPERIMENT = "null_model_full"
+
+# Create output location
+out = f"{test_dir}/makefile_out"
+if Path(out).exists():
+ # remove
+ shutil.rmtree(out)
+ # create output directory
+ Path(out).mkdir(parents=True,exist_ok=True)
+else:
+ Path(out).mkdir(parents=True,exist_ok=True)
+
+# Set output directory as home for fre make output
+#os.environ["HOME"]=str(Path(out))
+
+def test_modelyaml_exists():
+ """
+ Check the model yaml exists
+ """
+ assert Path(f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}").exists()
+
+def test_compileyaml_exists():
+ """
+ Check the compile yaml exists
+ """
+ assert Path(f"{test_dir}/{NM_EXAMPLE}/compile.yaml").exists()
+
+def test_platformyaml_exists():
+ """
+ Check the platform yaml exists
+ """
+ assert Path(f"{test_dir}/{NM_EXAMPLE}/platforms.yaml").exists()
+
+def test_bm_makefile_creation():
+ """
+ Check the makefile is created when a bare-metal platform is used
+ """
+ # Set output directory as home for fre make output
+ os.environ["HOME"]=str(Path(out))
+
+ bm_plat = BM_PLATFORM[0]
+ targ = TARGET[0]
+ yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}"
+
+ createMakefile.makefile_create(yamlfile_path,BM_PLATFORM,TARGET)
+
+ assert Path(f"{out}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists()
+
+def test_container_makefile_creation():
+ """
+ Check the makefile is created when the container platform is used
+ """
+ container_plat = CONTAINER_PLATFORM[0]
+ yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}"
+ createMakefile.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET)
+
+ assert Path(f"tmp/{container_plat}/Makefile").exists()
diff --git a/lib/fre/pp/README.md b/lib/fre/pp/README.md
new file mode 100644
index 00000000..173c1055
--- /dev/null
+++ b/lib/fre/pp/README.md
@@ -0,0 +1,148 @@
+
+
+
+
+
+
+
+## **Quickstart instructions to postprocess FMS history output on PP/AN or gaea**
+
+1. Checkout postprocessing workflow template
+This will clone the postprocessing repository into `/home/$USER/cylc-src/EXPNAME__PLATFORM__TARGET`.
+```
+module load fre/canopy
+fre pp checkout -e EXPNAME -p PLATFORM -t TARGET
+```
+
+2. Configure pp template with either XML or pp.yaml
+
+```
+fre pp configure-xml -e EXPNAME -p PLATFORM -t TARGET -x XML
+```
+or
+```
+fre pp configure-yaml -e EXPNAME -p PLATFORM -t TARGET -y YAML
+
+```
+
+3. (OPTIONAL BUT RECCOMENDED) Create `history-manifest` for config validation
+
+Create a `history-manifest` of a single tar file archive first for use in the validation.
+This list represents the available source files within the history tar archives, and enables the
+validation procedure to catch a wider variety of potential errors. This can be done like so-
+```
+tar -tf /archive/$USER/path/to/history/files/YYYYMMDD.nc.tar | grep -v tile[2-6] | sort > /home/$USER/cylc-src/EXPNAME__PLATFORM__TARGET/history-manifest
+```
+
+4. Validate the configuration
+```
+fre pp validate -e EXPNAME -p PLATFORM -t TARGET
+```
+
+Warnings related to directories are probably valid and should be fixed in `rose-suite.conf`, or created as necessary via `mkdir`.
+
+If you are running postprocessing gaea, you'll need to change the `SITE` variable in `rose-suite.conf` from `ppan` to `gaea`.
+
+5. Install the workflow
+
+```
+fre pp install -e EXPNAME -p PLATFORM -t TARGET
+```
+
+If you are attempting this on gaea, you'll need to make two one-time changes before installing.
+- Currently, `cylc`, `rose`, and `isodatetime` must be in your PATH for new shells. One approach to do this is
+to symlink the fms-user-installed fre-cli cylc/rose/isodatetime scripts into your local `~/bin` directory,
+and then add that `~/bin` directory to your PATH in your `.bashrc` or `.cshrc`. (If you don't do this, Cylc tasks
+will fail complaining those 3 tools are not available.)
+
+```
+cd ~/bin
+ln -s /ncrc/home2/Flexible.Modeling.System/conda/envs/fre-cli/bin/{cylc,rose,isodatetime} .
+echo 'setenv PATH ${PATH}:~/bin' >> ~/.cshrc
+```
+- Currently, the cylc available on gaea (through `module load cylc` or the `PATH` trick above) does not
+include any global configuration, so you'll need to create a file `~/.cylc/flow/global.cylc` that contains the following.
+If you don't do this, Cylc will use your home directory for the scratch space and rapidly fill your quota.)
+
+```
+[install]
+ [[symlink dirs]]
+ [[[localhost]]]
+ run = /gpfs/f5/scratch/gfdl_f/$USER
+```
+
+6. Run the workflow
+
+```
+fre pp run -e EXPNAME -p PLATFORM -t TARGET
+```
+
+7. Report status of workflow progress
+
+```
+fre pp status -e EXPNAME -p PLATFORM -t TARGET
+```
+
+8. Launch GUI
+
+```
+TODO: fre pp gui?
+
+The full GUI can be launched on jhan or jhanbigmem (an107 or an201).
+
+cylc gui --ip=`hostname -f` --port=`jhp 1` --no-browser
+```
+
+## **PP Wrapper Usage**
+
+Here
+
+### **PP Wrapper Decision Tree**
+
+
+### **Tests**
+
+To run `fre pp` tests, return to root directory of the fre-cli repo and call just those tests with
+
+ python -m pytest fre/pp/tests/[test script.py]
+
+Or run all tests with
+
+ python -m pytests fre/pp/tests
diff --git a/lib/fre/pp/__init__.py b/lib/fre/pp/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/fre/pp/checkoutScript.py b/lib/fre/pp/checkoutScript.py
new file mode 100644
index 00000000..57036634
--- /dev/null
+++ b/lib/fre/pp/checkoutScript.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+# Author: Bennett Chang
+# Description:
+
+import os
+import subprocess
+from subprocess import PIPE
+from subprocess import STDOUT
+import re
+import click
+
+#############################################
+
+package_dir = os.path.dirname(os.path.abspath(__file__))
+
+#############################################
+
+def _checkoutTemplate(experiment, platform, target, branch='main'):
+ """
+ Checkout the workflow template files from the repo
+ """
+ # Create the directory if it doesn't exist
+ directory = os.path.expanduser("~/cylc-src")
+ os.makedirs(directory, exist_ok=True)
+
+ # Change the current working directory
+ os.chdir(directory)
+
+ # Set the name of the directory
+ name = f"{experiment}__{platform}__{target}"
+
+ # Clone the repository with depth=1; check for errors
+ click.echo("cloning experiment into directory " + directory + "/" + name)
+ clonecmd = (
+ f"git clone -b {branch} --single-branch --depth=1 --recursive "
+ f"https://github.com/NOAA-GFDL/fre-workflows.git {name}" )
+ preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory."
+ click.echo(clonecmd)
+ cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT)
+ if not cloneproc.returncode == 0:
+ if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None:
+ argstring = f" -e {experiment} -p {platform} -t {target}"
+ stop_report = (
+ "Error in checkoutTemplate: the workflow definition specified by -e/-p/-t already"
+ f" exists at the location ~/cylc-src/{name}!\n"
+ f"In the future, we will confirm that ~/cylc-src/{name} is usable and will check "
+ "whether it is up-to-date.\n"
+ "But for now, if you wish to proceed, you must delete the workflow definition.\n"
+ "To start over, try:\n"
+ f"\t cylc stop {name}\n"
+ f"\t cylc clean {name}\n"
+ f"\t rm -r ~/cylc-src/{name}" )
+ click.echo(stop_report)
+ return 1
+ else:
+ #if not identified, just print the error
+ click.echo(clonecmd)
+ click.echo(cloneproc.stdout)
+ return 1
+
+#############################################
+
+@click.command()
+def checkoutTemplate(experiment, platform, target, branch="main"):
+ '''
+ Wrapper script for calling checkoutTemplate - allows the decorated version
+ of the function to be separate from the undecorated version
+ '''
+ return _checkoutTemplate(experiment, platform, target, branch)
+
+
+if __name__ == '__main__':
+ checkoutTemplate()
diff --git a/lib/fre/pp/configure_script_xml.py b/lib/fre/pp/configure_script_xml.py
new file mode 100644
index 00000000..899b4830
--- /dev/null
+++ b/lib/fre/pp/configure_script_xml.py
@@ -0,0 +1,696 @@
+#!/usr/bin/env python3
+'''
+ Primary Usage: fre-bronx-to-canopy -x XML -e EXP -p PLATFORM -t TARGET
+
+ The Bronx-to-Canopy XML converter overwrites 3 files:
+ - rose-suite.conf
+ - app/remap-pp-components/rose-app.conf
+ - app/regrid-xy/rose-app.conf
+'''
+
+# std lib
+import re
+import os
+import subprocess
+import logging
+
+# third party
+import click
+import metomi.rose.config
+import metomi.isodatetime.parsers
+
+#############################################
+
+LOGGING_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
+CYLC_PATH = '/home/fms/fre-canopy/system-settings/bin'
+CYLC_REFINED_SCRIPTS = ["check4ptop.pl",
+ "module_init_3_1_6.pl",
+ "plevel_mask.ncl",
+ "refineDiag_atmos.csh",
+ "refine_fields.pl",
+ "surface_albedo.ncl",
+ "tasminmax.ncl",
+ "tracer_refine.ncl",
+ "refineDiag_atmos_cmip6.csh"
+ ]
+CYLC_REFINED_DIR = "\\$CYLC_WORKFLOW_RUN_DIR/etc/refineDiag"
+PREANALYSIS_SCRIPT = "refineDiag_data_stager_globalAve.csh"
+
+
+def freq_from_legacy(legacy_freq):
+ """Return ISO8601 duration given Bronx-style frequencies
+
+ Arguments:
+ 1. legacy_freq[str]: The Bronx frequency
+ """
+ lookup = {
+ 'annual': 'P1Y',
+ 'monthly': 'P1M',
+ 'seasonal': 'P3M',
+ 'daily': 'P1D',
+ '120hr': 'P120D',
+ '12hr': 'PT12H',
+ '8hr': 'PT8H',
+ '6hr': 'PT6H',
+ '4hr': 'PT4H',
+ '3hr': 'PT3H',
+ '2hr': 'PT2H',
+ '1hr': 'PT1H',
+ 'hourly': 'PT1H',
+ '30min': 'PT30M'
+ }
+ return lookup[legacy_freq]
+
+def chunk_from_legacy(legacy_chunk):
+ """Return ISO8601 duration given Bronx-style chunk
+
+ Arguments:
+ 1. legacy_chunk[str]: The Bronx chunk
+ """
+ regex = re.compile( r'(\d+)(\w+)' )
+ match = regex.match( legacy_chunk )
+ if not match:
+ logging.error( "Could not convert Bronx chunk to ISO8601 duration: %s",
+ legacy_chunk )
+ raise ValueError
+
+ time_unit = match.group(2)
+ if time_unit not in ['yr','mo']:
+ logging.error("Unknown time units %s", match.group(2) )
+ raise ValueError
+
+ time_quant = match.group(1)
+ ret_val=f'P{time_quant}'
+ if time_unit == "yr":
+ ret_val+='Y'#return f'P{time_quant}Y'
+ elif time_unit == 'mo':
+ ret_val+='M'#return f'P{time_quant}M'
+ return ret_val
+
+def frelist_xpath(xml, platform, target, experiment,
+# do_analysis, historydir, refinedir, ppdir,
+# do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual,
+ xpath):
+ """Returns filepaths of FRE XML elements that use X-path notation
+ using Bronx's 'frelist' command via subprocess module
+
+ Arguments:
+ 1. args[str]: Argparse user-input arguments
+ 2. xpath[str]: X-path (XML) notation required by 'frelist'
+ """
+
+ cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} --evaluate '{xpath}'"
+ logging.info("running cmd:\n %s",cmd) #logging.info(f"running cmd:\n {cmd}")
+ logging.info(">> %s",xpath)
+ process = subprocess.run(cmd,
+ shell=True,
+ check=False, #if True, retrieving std err difficult...
+ capture_output=True,
+ universal_newlines=True)
+
+ logging.info("stdout: \n%s",process.stdout.strip())
+ logging.info("stderr: \n%s",process.stderr.strip())
+ logging.info("returncode: \n%s",process.returncode)
+
+
+ result = process.stdout.strip()
+ if process.returncode > 0:
+ raise subprocess.CalledProcessError(
+ returncode=process.returncode,
+ cmd=cmd,
+ output=process.stdout.strip(),
+ stderr=process.stderr.strip() )
+ return result
+
+def duration_to_seconds(duration):
+ """Returns the conversion of a chunk duration to seconds
+
+ Arguments:
+ 1. duration[str]: The original chunk duration
+ """
+ dur = metomi.isodatetime.parsers.DurationParser().parse(duration)
+ return dur.get_seconds()
+
+
+def main(xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir,
+ do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual):
+ """The meat of the converter
+
+ Arguments:
+ 1. args[argparse Namespace]: Arguments given at the command line
+
+ Tasks:
+ 1. Generate key-value pairs for the rose-suite.conf.
+ 2. Generate content for the regrid-xy app
+ 3. Generate content for the remap-pp-components app
+ """
+
+ ##########################################################################
+ # Set up default configurations for regrid-xy and remap-pp-components
+ ##########################################################################
+ rose_remap = metomi.rose.config.ConfigNode()
+ rose_remap.set(keys=['command', 'default'], value='remap-pp-components')
+
+ rose_regrid_xy = metomi.rose.config.ConfigNode()
+ rose_regrid_xy.set(keys=['command', 'default'], value='regrid-xy')
+
+ ##########################################################################
+ # Create the rose-suite config and begin setting up key-value pairs
+ # Note: All strings inside the rose-suite configuration MUST be quoted.
+ # Note: The exception to the 'quote' rule is boolean values.
+ # Note: Addition of quotes will appear as "'" in the code.
+ ##########################################################################
+ rose_suite = metomi.rose.config.ConfigNode()
+ rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"')
+ rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True')
+ rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"')
+ rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False')
+ rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True')
+ rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True')
+ rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False')
+ rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True')
+ rose_suite.set(keys=['template variables', 'FRE_ANALYSIS_HOME'],
+ value='"/home/fms/local/opt/fre-analysis/test"')
+
+ # not sure about these
+ rose_suite.set(keys=['template variables', 'PP_DEFAULT_XYINTERP'], value='"360,180"')
+ rose_suite.set(keys=['template variables', 'DO_ANALYSIS'], value='True')
+
+ rose_suite.set(keys=['template variables', 'EXPERIMENT'],
+ value=f"'{experiment}'") #value=f"'{}'".format(experiment))
+ rose_suite.set(keys=['template variables', 'PLATFORM'],
+ value=f"'{platform}'")#value="'{}'".format(platform))
+ rose_suite.set(keys=['template variables', 'TARGET'],
+ value=f"'{target}'")#value="'{}'".format(target))
+
+ #regex_fre_property = re.compile(r'\$\((\w+)') #notyetimplemented
+ #all_components = set() #notyetimplemented
+
+ ##########################################################################
+ # Run 'frelist' in the background to fetch the default history directory,
+ # the default history_refineDiag directory, and the default PP directory,
+ # all of which have been set in the XML. If the custom arguments for these
+ # directory variables have not been set by the user at the command line,
+ # the XML's default paths will be inserted into rose-suite.conf.
+ ##########################################################################
+ if historydir is None:
+ logging.info("Running frelist for historydir assignment, this may fail")
+ logging.info("If so, try the frelist call manually and use the historydir argument")
+
+ fetch_history_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d archive"
+ logging.info(">> %s", fetch_history_cmd)
+
+ fetch_history_process = subprocess.run(fetch_history_cmd,
+ shell=True,
+ check=True,
+ capture_output=True,
+ universal_newlines=True)
+ historydir = fetch_history_process.stdout.strip() + '/history'
+ logging.info(historydir)
+
+
+
+ # Q: should respond to do_refineDiag and refinedir args of this function?
+ historydir_refined = historydir + '_refineDiag'
+
+ if ppdir is None:
+ logging.info("Running frelist for ppdir assignment...")
+ logging.info("If this fails, try the frelist call manually and use the ppdir argument")
+ fetch_pp_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d postProcess"
+ logging.info(">> %s", fetch_pp_cmd)
+
+ fetch_pp_process = subprocess.run(fetch_pp_cmd,
+ shell=True,
+ check=True,
+ capture_output=True,
+ universal_newlines=True)
+ ppdir = fetch_pp_process.stdout.strip()
+ logging.info(ppdir)
+
+
+ # Q: shouldn't there be a CLI analysis dir arg while we're here?
+ # basically, this is borderline on the same level as the ppDir and historydir fields.
+ #if do_analysis:
+ logging.info("Running frelist for analysis_dir assignment...")
+ fetch_analysis_dir_cmd = f"frelist -x {xml} -p {platform} -t {target} {experiment} -d analysis"
+ logging.info(">> %s", fetch_analysis_dir_cmd)
+
+ fetch_analysis_dir_process = subprocess.run(fetch_analysis_dir_cmd,
+ shell=True,
+ check=True,
+ capture_output=True,
+ universal_newlines=True)
+ analysis_dir = fetch_analysis_dir_process.stdout.strip()
+ logging.info(analysis_dir)
+ #else:
+ # logging.info('not doing analysis.')
+
+
+ grid_spec = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir, do_refinediag,
+ #pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'input/dataFile[@label="gridSpec"]')
+ sim_time = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir, do_refinediag,
+ #pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'runtime/production/@simTime')
+ sim_units = frelist_xpath(xml, platform, target, experiment,
+ #do_analysis,
+ #historydir, refinedir, ppdir, do_refinediag,
+ #pp_start, pp_stop, validate,
+ #verbose, quiet, dual,
+ 'runtime/production/@units')
+
+ rose_suite.set(keys=['template variables', 'HISTORY_DIR'],
+ value=f"'{historydir}'") #value="'{}'".format(historydir))
+ # set some dirs to something else to allow bronx dual-pps easily
+ if dual:
+ rose_suite.set(keys=['template variables', 'PP_DIR'],
+ value=f"'{ppdir}_canopy'")
+ rose_suite.set(keys=['template variables', 'ANALYSIS_DIR'],
+ value=f"'{analysis_dir}_canopy'")
+ else:
+ rose_suite.set(keys=['template variables', 'PP_DIR'],
+ value=f"'{ppdir}'")
+ rose_suite.set(keys=['template variables', 'ANALYSIS_DIR'],
+ value=f"'{analysis_dir}'")
+ rose_suite.set(keys=['template variables', 'PP_GRID_SPEC'],
+ value=f"'{grid_spec}'")
+
+ ##########################################################################
+ # Process the refineDiag scripts into the rose-suite configuration from
+ # the
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Welcome to Fre-Cli’s documentation!
+
+
+Contents:
+-
+
- Setup + +
- Usage
-
+
- Calling
fre
+ - Tools + +
+ - Calling
- Developer Usage + +
- Documentation-Documentation + +
- Badges +
Indices
+-
+
- +
- +
- +
