From 58b135716e80f921c970da7caac43e03b32b97c2 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Fri, 23 Aug 2024 14:12:45 -0500 Subject: [PATCH 01/23] Initial significant changes to app to allow tracking and storing data for concurrent sessions. Downloads into session-specific folders works, as does script monitoring and cancelling. Radar download status needs to be addressed, and all variables within RadarServer object need to be moved to dcc.Store in a dict. --- app.py | 283 ++++++++++++++++++++---------------- config.py | 109 +++++++++++++- layout_components.py | 4 +- scripts/Nexrad.py | 8 +- scripts/hodo_plot.py | 7 +- scripts/munger.py | 27 ++-- scripts/obs_placefile.py | 21 +-- scripts/update_dir_list.py | 9 +- scripts/update_hodo_page.py | 30 ++-- utils.py | 59 ++++---- 10 files changed, 355 insertions(+), 202 deletions(-) diff --git a/app.py b/app.py index 4515dd1..f1425e3 100644 --- a/app.py +++ b/app.py @@ -34,7 +34,8 @@ from botocore.client import Config # bootstrap is what helps styling for a better presentation import dash_bootstrap_components as dbc -import config as cfg +import config +from config import app import layout_components as lc from scripts.obs_placefile import Mesowest @@ -107,15 +108,15 @@ def __init__(self): #self.make_simulation_times() # This will generate a logfile. Something we'll want to turn on in the future. self.log = self.create_logfile() - UpdateHodoHTML('None') # set up the hodo page with no images + #UpdateHodoHTML('None', '', '') # set up the hodo page with no images def create_logfile(self): """ Creates an initial logfile. Stored in the data dir for now. Call is sa.log.info or sa.log.error or sa.log.warning or sa.log.exception """ - os.makedirs(cfg.LOG_DIR, exist_ok=True) - logging.basicConfig(filename=f"{cfg.LOG_DIR}/logfile.txt", + os.makedirs(config.LOG_DIR, exist_ok=True) + logging.basicConfig(filename=f"{config.LOG_DIR}/logfile.txt", format='%(levelname)s %(asctime)s :: %(message)s', datefmt="%Y-%m-%d %H:%M:%S") log = logging.getLogger() @@ -135,13 +136,13 @@ def create_radar_dict(self) -> None: 'asos_one': asos_one, 'asos_two': asos_two, 'radar': radar.upper(), 'file_list': []} - def copy_grlevel2_cfg_file(self) -> None: + def copy_grlevel2_cfg_file(self, cfg) -> None: """ Ensures a grlevel2.cfg file is copied into the polling directory. This file is required for GR2Analyst to poll for radar data. """ - source = cfg.BASE_DIR / 'grlevel2.cfg' - destination = cfg.POLLING_DIR / 'grlevel2.cfg' + source = f"{cfg['BASE_DIR']}/grlevel2.cfg" + destination = f"{cfg['POLLING_DIR']}/grlevel2.cfg" try: shutil.copyfile(source, destination) except Exception as e: @@ -302,12 +303,12 @@ def _clamp(n, minimum, maximum): math.cos(d/R) - math.sin(phi_new) * math.sin(phi_out)) return math.degrees(phi_out), math.degrees(lambda_out) - def shift_placefiles(self) -> None: + def shift_placefiles(self, PLACEFILES_DIR) -> None: """ # While the _shifted placefiles should be purged for each run, just ensure we're # only querying the "original" placefiles to shift (exclude any with _shifted.txt) """ - filenames = glob(f"{cfg.PLACEFILES_DIR}/*.txt") + filenames = glob(f"{PLACEFILES_DIR}/*.txt") filenames = [x for x in filenames if "shifted" not in x] for file_ in filenames: with open(file_, 'r', encoding='utf-8') as f: @@ -373,12 +374,13 @@ def datetime_object_from_timestring(self, dt_str: str) -> datetime: utc_file_time = file_time.replace(tzinfo=pytz.UTC) return utc_file_time - def remove_files_and_dirs(self) -> None: + def remove_files_and_dirs(self, cfg) -> None: """ Cleans up files and directories from the previous simulation so these datasets are not included in the current simulation. """ - dirs = [cfg.RADAR_DIR, cfg.POLLING_DIR, cfg.HODOGRAPHS_DIR, cfg.MODEL_DIR] + dirs = [cfg['RADAR_DIR'], cfg['POLLING_DIR'], cfg['HODOGRAPHS_DIR'], cfg['MODEL_DIR'], + cfg['PLACEFILES_DIR']] for directory in dirs: for root, dirs, files in os.walk(directory, topdown=False): for name in files: @@ -391,11 +393,7 @@ def remove_files_and_dirs(self) -> None: # ----------------------------- Initialize the app -------------------------------------------- ################################################################################################ - sa = RadarSimulator() -app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG], - suppress_callback_exceptions=True, update_title=None) -app.title = "Radar Simulator" ################################################################################################ # ----------------------------- Build the layout --------------------------------------------- @@ -409,7 +407,8 @@ def remove_files_and_dirs(self) -> None: dcc.Dropdown(np.arange(1, sa.days_in_month+1), 16, id='start_day', clearable=False)])) playback_time_options = dbc.Col(html.Div([ - dcc.Dropdown(options={'label': 'Sim not started', 'value': ''}, id='change_time', disabled=True, clearable=False)])) + dcc.Dropdown(options={'label': 'Sim not started', 'value': ''}, id='change_time', + disabled=True, clearable=False)])) playback_time_options_col = dbc.Col(html.Div([lc.change_playback_time_label, lc.spacer_mini, playback_time_options])) @@ -425,39 +424,62 @@ def remove_files_and_dirs(self) -> None: playback_controls, lc.spacer_mini, ]),style=lc.section_box_pad)) -app.layout = dbc.Container([ - # testing directory size monitoring - dcc.Interval(id='directory_monitor', disabled=False, interval=2*1000), - dcc.Interval(id='playback_timer', disabled=True, interval=15*1000), - # dcc.Store(id='model_dir_size'), - # dcc.Store(id='radar_dir_size'), - dcc.Store(id='tradar'), - dcc.Store(id='dummy'), - dcc.Store(id='playback_running_store', data=False), - dcc.Store(id='playback_start_store'), - dcc.Store(id='playback_end_store'), - dcc.Store(id='playback_clock_store'), - lc.top_section, lc.top_banner, - dbc.Container([ - dbc.Container([ - html.Div([html.Div([lc.step_select_time_section, lc.spacer, - dbc.Row([ - lc.sim_year_section, lc.sim_month_section, sim_day_selection, - lc.sim_hour_section, lc.sim_minute_section, lc.sim_duration_section, - lc.spacer, lc.step_time_confirm])], style={'padding': '1em'}), - ], style=lc.section_box)]) - ]), lc.spacer, - lc.full_radar_select_section, lc.spacer_mini, - lc.map_section, - lc.full_transpose_section, - lc.scripts_button, - lc.status_section, - lc.spacer,lc.toggle_placefiles_btn,lc.spacer_mini, - lc.full_links_section, lc.spacer, - simulation_playback_section, - html.Div(id='playback_speed_dummy', style={'display': 'none'}), - lc.radar_id, lc.bottom_section - ])# end of app.layout +@app.callback( + Output('dynamic_container', 'children'), + Output('layout_has_initialized', 'data'), + Input('directory_monitor', 'n_intervals'), + State('layout_has_initialized', 'data'), + State('dynamic_container', 'children') +) +def generate_layout(n_intervals, layout_has_initialized, children): + """ + Dynamically generate the layout, which was started in the config file to set up + the unique session id. This callback should only be executed once at page load in. + Thereafter, layout_has_initialized will be set to True + """ + if not layout_has_initialized['added']: + if children is None: + children = [] + + new_items = dbc.Container([ + dcc.Interval(id='playback_timer', disabled=True, interval=15*1000), + dcc.Store(id='tradar'), + dcc.Store(id='dummy'), + dcc.Store(id='playback_running_store', data=False), + dcc.Store(id='playback_start_store'), + dcc.Store(id='playback_end_store'), + dcc.Store(id='playback_clock_store'), + lc.top_section, lc.top_banner, + dbc.Container([ + dbc.Container([ + html.Div([html.Div([lc.step_select_time_section, lc.spacer, + dbc.Row([ + lc.sim_year_section, lc.sim_month_section, sim_day_selection, + lc.sim_hour_section, lc.sim_minute_section, lc.sim_duration_section, + lc.spacer, lc.step_time_confirm])], style={'padding': '1em'}), + ], style=lc.section_box)]) + ]), lc.spacer, + lc.full_radar_select_section, lc.spacer_mini, + lc.map_section, + lc.full_transpose_section, + lc.scripts_button, + lc.status_section, + lc.spacer,lc.toggle_placefiles_btn,lc.spacer_mini, + lc.full_links_section, lc.spacer, + simulation_playback_section, + html.Div(id='playback_speed_dummy', style={'display': 'none'}), + lc.radar_id, lc.bottom_section + ]) + + # Append the new component to the current list of children + children = list(children) + children.append(new_items) + + layout_has_initialized['added'] = True + + return children, layout_has_initialized + + return children, layout_has_initialized ################################################################################################ ################################################################################################ @@ -576,7 +598,7 @@ def transpose_radar(value): # ----------------------------- Run Scripts button -------------------------------------------- ################################################################################################ -def query_radar_files(): +def query_radar_files(cfg): """ Get the radar files from the AWS bucket. This is a preliminary step to build the progess bar. """ @@ -585,9 +607,10 @@ def query_radar_files(): sa.radar_files_dict = {} for _r, radar in enumerate(sa.radar_list): radar = radar.upper() - args = [radar, f'{sa.event_start_str}', str(sa.event_duration), str(False)] + args = [radar, f'{sa.event_start_str}', str(sa.event_duration), str(False), + cfg['RADAR_DIR']] sa.log.info(f"Passing {args} to Nexrad.py") - results = utils.exec_script(cfg.NEXRAD_SCRIPT_PATH, args) + results = utils.exec_script(Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if results['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: sa.log.warning(f"User cancelled query_radar_files()") break @@ -610,11 +633,11 @@ def run_hodo_script(args) -> None: start and playback start """ print(args) - subprocess.run(["python", cfg.HODO_SCRIPT_PATH] + args, check=True) + subprocess.run(["python", config.HODO_SCRIPT_PATH] + args, check=True) def call_function(func, *args, **kwargs): - if len(args) > 0: + if len(args) > 1: sa.log.info(f"Sending {args[1]} to {args[0]}") result = func(*args, **kwargs) @@ -626,29 +649,31 @@ def call_function(func, *args, **kwargs): return result -def run_with_cancel_button(): +def run_with_cancel_button(cfg): """ This version of the script-launcher trying to work in cancel button """ + UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + sa.scripts_progress = 'Setting up files and times' # determine actual event time, playback time, diff of these two sa.make_simulation_times() # clean out old files and directories try: - sa.remove_files_and_dirs() + sa.remove_files_and_dirs(cfg) except Exception as e: sa.log.exception("Error removing files and directories: ", exc_info=True) # based on list of selected radars, create a dictionary of radar metadata try: sa.create_radar_dict() - sa.copy_grlevel2_cfg_file() + sa.copy_grlevel2_cfg_file(cfg) except Exception as e: sa.log.exception("Error creating radar dict or config file: ", exc_info=True) # Create initial dictionary of expected radar files if len(sa.radar_list) > 0: - res = call_function(query_radar_files) + res = call_function(query_radar_files, cfg) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return @@ -663,37 +688,42 @@ def run_with_cancel_button(): sa.log.exception("Error defining new radar: ", exc_info=True) # Radar download - args = [radar, str(sa.event_start_str), str(sa.event_duration), str(True)] - res = call_function(utils.exec_script, cfg.NEXRAD_SCRIPT_PATH, args) + args = [radar, str(sa.event_start_str), str(sa.event_duration), str(True), + cfg['RADAR_DIR']] + res = call_function(utils.exec_script, Path(cfg['NEXRAD_SCRIPT_PATH']), + args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # Munger - args = [radar, str(sa.playback_start_str), str(sa.event_duration), - str(sa.simulation_seconds_shift), new_radar] - res = call_function(utils.exec_script, cfg.MUNGER_SCRIPT_FILEPATH, args) + args = [radar, str(sa.playback_start_str), str(sa.event_duration), + str(sa.simulation_seconds_shift), cfg['RADAR_DIR'], cfg['POLLING_DIR'], + cfg['L2MUNGER_FILEPATH'], cfg['DEBZ_FILEPATH'], new_radar] + res = call_function(utils.exec_script, Path(cfg['MUNGER_SCRIPT_FILEPATH']), + args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # this gives the user some radar data to poll while other scripts are running try: - UpdateDirList(new_radar, 'None', initialize=True) + UpdateDirList(new_radar, 'None', cfg['POLLING_DIR'], initialize=True) except Exception as e: print(f"Error with UpdateDirList ", e) sa.log.exception(f"Error with UpdateDirList ", exc_info=True) - + # Surface observations - args = [str(sa.lat), str(sa.lon), sa.event_start_str, str(sa.event_duration)] - res = call_function(utils.exec_script, cfg.OBS_SCRIPT_PATH, args) + args = [str(sa.lat), str(sa.lon), sa.event_start_str, cfg['PLACEFILES_DIR'], + str(sa.event_duration)] + res = call_function(utils.exec_script, Path(cfg['OBS_SCRIPT_PATH']), args, + cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # NSE placefiles - #args = [str(sa.event_start_time), str(sa.event_duration), str(sa.scripts_path), - # str(sa.data_dir), str(sa.placefiles_dir)] - args = [str(sa.event_start_time), str(sa.event_duration), str(cfg.SCRIPTS_DIR), - str(cfg.DATA_DIR), str(cfg.PLACEFILES_DIR)] - res = call_function(utils.exec_script, cfg.NSE_SCRIPT_PATH, args) + args = [str(sa.event_start_time), str(sa.event_duration), cfg['SCRIPTS_DIR'], + cfg['DATA_DIR'], cfg['PLACEFILES_DIR']] + res = call_function(utils.exec_script, Path(cfg['NSE_SCRIPT_PATH']), args, + cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return @@ -701,7 +731,7 @@ def run_with_cancel_button(): # script needs to execute every time, even if a user doesn't select a radar # to transpose to. sa.log.info(f"Entering function run_transpose_script") - run_transpose_script() + run_transpose_script(cfg['PLACEFILES_DIR']) # Hodographs for radar, data in sa.radar_dict.items(): @@ -712,20 +742,23 @@ def run_with_cancel_button(): sa.log.exception("Error getting radar metadata: ", exc_info=True) # Execute hodograph script - args = [radar, sa.new_radar, asos_one, asos_two, str(sa.simulation_seconds_shift)] - res = call_function(utils.exec_script, cfg.HODO_SCRIPT_PATH, args) + args = [radar, sa.new_radar, asos_one, asos_two, str(sa.simulation_seconds_shift), + cfg['RADAR_DIR'], cfg['HODOGRAPHS_DIR']] + res = call_function(utils.exec_script, Path(cfg['HODO_SCRIPT_PATH']), args, + cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return try: - UpdateHodoHTML('None') + UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) except Exception as e: print("Error updating hodo html: ", e) sa.log.exception("Error updating hodo html: ", exc_info=True) @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), - [Input('run_scripts_btn', 'n_clicks')], + [Input('run_scripts_btn', 'n_clicks'), + State('configs', 'data')], prevent_initial_call=True, running=[ (Output('start_year', 'disabled'), True, False), @@ -745,7 +778,7 @@ def run_with_cancel_button(): (Output('change_time', 'disabled'), True, False), # wait to enable change time dropdown (Output('cancel_scripts', 'disabled'), False, True), ]) -def launch_simulation(n_clicks) -> None: +def launch_simulation(n_clicks, configs) -> None: """ This function is called when the "Run Scripts" button is clicked. It will execute the necessary scripts to simulate radar operations, create hodographs, and transpose placefiles. @@ -753,10 +786,10 @@ def launch_simulation(n_clicks) -> None: if n_clicks == 0: raise PreventUpdate else: - if cfg.PLATFORM == 'WINDOWS': + if config.PLATFORM == 'WINDOWS': sa.make_simulation_times() else: - run_with_cancel_button() + run_with_cancel_button(configs) ################################################################################################ # ----------------------------- Monitoring and reporting script status ------------------------ @@ -764,16 +797,17 @@ def launch_simulation(n_clicks) -> None: @app.callback( Output('dummy', 'data'), - [Input('cancel_scripts', 'n_clicks')], + [Input('cancel_scripts', 'n_clicks'), + State('session_id', 'data')], prevent_initial_call=True) -def cancel_scripts(n_clicks) -> None: +def cancel_scripts(n_clicks, SESSION_ID) -> None: """ This function is called when the "Cancel Scripts" button is clicked. It will cancel all Args: n_clicks (int): incremented whenever the "Cancel Scripts" button is clicked """ if n_clicks > 0: - utils.cancel_all(sa) + utils.cancel_all(sa, SESSION_ID) @app.callback( @@ -784,10 +818,11 @@ def cancel_scripts(n_clicks) -> None: Output('model_table', 'data'), Output('model_status_warning', 'children'), Output('show_script_progress', 'children', allow_duplicate=True), - [Input('directory_monitor', 'n_intervals')], + [Input('directory_monitor', 'n_intervals'), + State('configs', 'data')], prevent_initial_call=True ) -def monitor(_n): +def monitor(_n, cfg): """ This function is called every second by the directory_monitor interval. It (1) checks the status of the various scripts and reports them to the front-end application and @@ -797,40 +832,42 @@ def monitor(_n): screen_output = "" seen_scripts = [] for p in processes: - # Returns get_data or process (the two scripts launched by nse.py) - name = p['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] - - # Scripts executed as python modules will be like [python, -m, script.name] - if p['cmdline'][1] == '-m': - # Should return Nexrad, munger, nse, etc. - name = p['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] - if p['name'] == 'wgrib2': - name = 'wgrib2' - - if name in cfg.scripts_list and name not in seen_scripts: - runtime = time.time() - p['create_time'] - screen_output += f"{name}: running for {round(runtime,1)} s. " - seen_scripts.append(name) + process_session_id = p['session_id'] + if process_session_id == cfg['SESSION_ID']: + # Returns get_data or process (the two scripts launched by nse.py) + name = p['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] + + # Scripts executed as python modules will be like [python, -m, script.name] + if p['cmdline'][1] == '-m': + # Should return Nexrad, munger, nse, etc. + name = p['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] + if p['name'] == 'wgrib2': + name = 'wgrib2' + + if name in config.scripts_list and name not in seen_scripts: + runtime = time.time() - p['create_time'] + screen_output += f"{name}: running for {round(runtime,1)} s. " + seen_scripts.append(name) # Radar file download status radar_dl_completion, radar_files = utils.radar_monitor(sa) # Radar mungering/transposing status - munger_completion = utils.munger_monitor(sa) + munger_completion = utils.munger_monitor(sa, cfg) # Surface placefile status - placefile_stats = utils.surface_placefile_monitor(sa) + placefile_stats = utils.surface_placefile_monitor(sa, cfg) placefile_status_string = f"{placefile_stats[0]}/{placefile_stats[1]} files found" # Hodographs. Currently hard-coded to expect 2 files for every radar and radar file. - num_hodograph_images = len(glob(f"{cfg.HODO_IMAGES}/*.png")) + num_hodograph_images = len(glob(f"{cfg['HODOGRAPHS_DIR']}/*.png")) hodograph_completion = 0 if len(radar_files) > 0: hodograph_completion = 100 * \ (num_hodograph_images / (2*len(radar_files))) # NSE placefiles - model_list, model_warning = utils.nse_status_checker(sa) + model_list, model_warning = utils.nse_status_checker(sa, cfg) return (radar_dl_completion, hodograph_completion, munger_completion, placefile_status_string, model_list, model_warning, screen_output) @@ -841,11 +878,11 @@ def monitor(_n): # whether to also perform a spatial shift occurrs within self.shift_placefiles where # a check for sa.new_radar != None takes place. -def run_transpose_script() -> None: +def run_transpose_script(PLACEFILES_DIR) -> None: """ Wrapper function to the shift_placefiles script """ - sa.shift_placefiles() + sa.shift_placefiles(PLACEFILES_DIR) ################################################################################################ # ----------------------------- Toggle Placefiles Section -------------------------------------- @@ -881,8 +918,9 @@ def toggle_placefiles_section(n) -> dict: Output('end_readout', 'style'), Output('change_time', 'options'), Input('playback_btn', 'n_clicks'), + State('configs', 'data'), prevent_initial_call=True) -def initiate_playback(_nclick): +def initiate_playback(_nclick, cfg): """ Enables/disables interval component that elapses the playback time @@ -894,13 +932,13 @@ def initiate_playback(_nclick): end = sa.playback_end_str style = lc.playback_times_style options = sa.playback_dropdown_dict - if cfg.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str) + if config.PLATFORM != 'WINDOWS': + UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str) + UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) else: for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar,sa.playback_clock_str) + UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) return btn_text, btn_disabled, False, playback_running, start, style, end, style, options @@ -914,9 +952,10 @@ def initiate_playback(_nclick): [Input('pause_resume_playback_btn', 'n_clicks'), Input('playback_timer', 'n_intervals'), Input('change_time', 'value'), - Input('playback_running_store', 'data') + Input('playback_running_store', 'data'), + State('configs', 'data') ], prevent_initial_call=True) -def manage_clock_(nclicks, _n_intervals, new_time, _playback_running): +def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg): """ Test """ @@ -937,13 +976,13 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running): if sa.playback_clock < sa.playback_end: sa.playback_clock_str = sa.date_time_string(sa.playback_clock) readout_time = datetime.strftime(sa.playback_clock, '%Y-%m-%d %H:%M:%S') - if cfg.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str) + if config.PLATFORM != 'WINDOWS': + UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str) + UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) else: for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar,sa.playback_clock_str) + UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) else: pass @@ -976,13 +1015,13 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running): sa.playback_clock = sa.playback_clock.replace(tzinfo=timezone.utc) sa.playback_clock_str = new_time readout_time = datetime.strftime(sa.playback_clock, '%Y-%m-%d %H:%M:%S') - if cfg.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str) + if config.PLATFORM != 'WINDOWS': + UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str) + UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) else: for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar,sa.playback_clock_str) + UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) if triggered_id == 'playback_running_store': pass @@ -1107,11 +1146,11 @@ def get_duration(duration) -> int: ################################################################################################ if __name__ == '__main__': - if cfg.CLOUD: + if config.CLOUD: app.run_server(host="0.0.0.0", port=8050, threaded=True, debug=True, use_reloader=False, dev_tools_hot_reload=False) else: - if cfg.PLATFORM == 'DARWIN': + if config.PLATFORM == 'DARWIN': app.run(host="0.0.0.0", port=8051, threaded=True, debug=True, use_reloader=False, dev_tools_hot_reload=False) else: diff --git a/config.py b/config.py index ef800ce..7e2f32a 100644 --- a/config.py +++ b/config.py @@ -4,8 +4,16 @@ from pathlib import Path import os import sys +import time +import uuid -#BASE_DIR = Path.cwd() +import flask +from dash import Dash, dcc, html, State, Input, Output +import dash_bootstrap_components as dbc + +######################################################################################## +# Define the initial base directory +######################################################################################## BASE_DIR = Path('/data/cloud-radar-server') LINK_BASE = "https://rssic.nws.noaa.gov/assets" CLOUD = True @@ -26,7 +34,99 @@ CLOUD = False PLATFORM = 'WINDOWS' +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# This and LINK_BASE will need to be moved into the dcc.Store object. All references to +# these locations in layout_components will also need to be moved into app.py after +# directory paths are read in from +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +PLACEFILES_LINKS = f'{LINK_BASE}/placefiles' + +######################################################################################## +# Initialize the application layout. A unique session ID will be generated on each page +# load. +######################################################################################## +server = flask.Flask(__name__) +app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG], + suppress_callback_exceptions=True, update_title=None, server=server) +app.title = "Radar Simulator" + +def init_layout(): + """ + Initialize the layout with a unique session id. The 'dynamic container' is used + within the app to build out the rest of the application layout on page load + """ + session_id = f'{time.time_ns()//1000}_{uuid.uuid4()}' + return dbc.Container([ + # Elements used to store and track the session id + dcc.Store(id='session_id', data=session_id, storage_type='session'), + dcc.Interval(id='broadcast_session_id', interval=1, n_intervals=0, max_intervals=1), + dcc.Store(id='configs', data={}), + + # Elements needed to set up the layout on page load by app.py + dcc.Interval(id='directory_monitor', interval=1000), + dcc.Store(id='layout_has_initialized', data={'added': False}), + html.Div(id='dynamic_container') + ]) + +app.layout = init_layout + +@app.callback( + Output('configs', 'data'), + Input('broadcast_session_id', 'n_intervals'), + State('session_id', 'data') +) +def broadcast_session_id(n_intervals, session_id): + """ + Callback executed once on page load to query the session id in dcc.Store component. + Creates a dictionary of directory paths and stores in a separate dcc.Store component. + + We cannot pass pathlib.Path objects in this dictionary since they're not + JSON-serializable. + """ + if n_intervals > 0: + dirs = { + 'BASE_DIR': f'{BASE_DIR}', + 'ASSETS_DIR': f'{BASE_DIR}/assets/{session_id}', + 'DATA_DIR': f'{BASE_DIR}/data/{session_id}' + } + dirs['SESSION_ID'] = session_id + dirs['PLACEFILES_DIR'] = f"{dirs['ASSETS_DIR']}/placefiles" + dirs['HODOGRAPHS_DIR'] = f"{dirs['ASSETS_DIR']}/hodographs" + dirs['HODOGRAPHS_PAGE'] = f"{dirs['ASSETS_DIR']}/hodographs.html" + dirs['HODO_HTML_PAGE'] = dirs['HODOGRAPHS_PAGE'] + dirs['POLLING_DIR'] = f"{dirs['ASSETS_DIR']}/polling" + dirs['MODEL_DIR'] = f"{dirs['DATA_DIR']}/model_data" + dirs['RADAR_DIR'] = f"{dirs['DATA_DIR']}/radar" + dirs['LOG_DIR'] = f"{dirs['DATA_DIR']}/logs" + + # Need to be updated + dirs['LINK_BASE'] = f"https://rssic.nws.noaa.gov/assets/{session_id}" + dirs['PLACEFILES_LINKS'] = f"{dirs['LINK_BASE']}/placefiles" + dirs['HODO_HTML_LINK'] = f"{dirs['LINK_BASE']}/hodographs.html" + + # Static directories (not dependent on session id) + dirs['SCRIPTS_DIR'] = f'{BASE_DIR}/scripts' + dirs['OBS_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/obs_placefile.py' + dirs['HODO_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/hodo_plot.py' + dirs['NEXRAD_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/Nexrad.py' + dirs['L2MUNGER_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/l2munger' + dirs['MUNGER_SCRIPT_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/munger.py' + dirs['MUNGE_DIR'] = f'{dirs['SCRIPTS_DIR']}/munge' + dirs['NSE_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/nse.py' + dirs['DEBZ_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/debz.py' + dirs['CSV_PATH'] = f'{BASE_DIR}/radars.csv' + + os.makedirs(dirs['MODEL_DIR'], exist_ok=True) + os.makedirs(dirs['RADAR_DIR'], exist_ok=True) + os.makedirs(dirs['HODOGRAPHS_DIR'], exist_ok=True) + os.makedirs(dirs['PLACEFILES_DIR'], exist_ok=True) + os.makedirs(dirs['POLLING_DIR'], exist_ok=True) + os.makedirs(dirs['LOG_DIR'], exist_ok=True) + + return dirs + +''' ASSETS_DIR = BASE_DIR / 'assets' PLACEFILES_DIR = ASSETS_DIR / 'placefiles' @@ -61,7 +161,10 @@ os.makedirs(DATA_DIR, exist_ok=True) os.makedirs(HODO_IMAGES, exist_ok=True) os.makedirs(PLACEFILES_DIR, exist_ok=True) - -# +''' +DATA_DIR = BASE_DIR / 'data' +LOG_DIR = DATA_DIR / 'logs' +# Names (without extensions) of various pre-processing scripts. Needed for script +# monitoring and/or cancelling. scripts_list = ["Nexrad", "munger", "obs_placefile", "nse", "wgrib2", "get_data", "process", "hodo_plot"] diff --git a/layout_components.py b/layout_components.py index c2f5c5d..78d7c12 100644 --- a/layout_components.py +++ b/layout_components.py @@ -227,8 +227,8 @@ fig.update_layout( mapbox={'accesstoken': MAP_TOKEN, - #'style': "carto-darkmatter", - 'style': "mapbox://styles/mapbox/dark-v10", + 'style': "carto-darkmatter", + #'style': "mapbox://styles/mapbox/dark-v10", 'center': {'lon': -94.4, 'lat': 38.2}, 'zoom': 3.8}) diff --git a/scripts/Nexrad.py b/scripts/Nexrad.py index 0cf760c..54cab49 100644 --- a/scripts/Nexrad.py +++ b/scripts/Nexrad.py @@ -15,7 +15,7 @@ import botocore from botocore.client import Config -from config import RADAR_DIR +#from config import RADAR_DIR class NexradDownloader: """ @@ -32,7 +32,7 @@ class NexradDownloader: """ - def __init__(self, radar_id, start_tstr, duration, download): + def __init__(self, radar_id, start_tstr, duration, download, RADAR_DIR): super().__init__() self.radar_id = radar_id self.start_tstr = start_tstr @@ -45,7 +45,7 @@ def __init__(self, radar_id, start_tstr, duration, download): user_agent_extra='Resource')).Bucket('noaa-nexrad-level2') self.prefix_day_one, self.prefix_day_two = self.make_prefix() - self.download_directory = RADAR_DIR / self.radar_id / 'downloads' + self.download_directory = Path(f"{RADAR_DIR}/{self.radar_id}/downloads") os.makedirs(self.download_directory, exist_ok=True) self.process_files() sys.stdout.write(json.dumps(self.radar_files_dict)) @@ -124,4 +124,4 @@ def process_files(self) -> None: download_flag = sys.argv[4] if type(download_flag) == str: download_flag = ast.literal_eval(download_flag) - NexradDownloader(sys.argv[1], sys.argv[2], sys.argv[3], download_flag) + NexradDownloader(sys.argv[1], sys.argv[2], sys.argv[3], download_flag, sys.argv[5]) diff --git a/scripts/hodo_plot.py b/scripts/hodo_plot.py index bc9edbc..5443984 100644 --- a/scripts/hodo_plot.py +++ b/scripts/hodo_plot.py @@ -18,7 +18,7 @@ from multiprocessing import Pool, freeze_support from pathlib import Path -from config import RADAR_DIR, HODO_IMAGES +#from config import RADAR_DIR, HODO_IMAGES import scripts.hodo_resources as hr from dotenv import load_dotenv @@ -42,10 +42,9 @@ asos_two = None timeshift_seconds = int(sys.argv[5]) -#BASE_DIR = Path('/data/cloud-radar-server') -#RADAR_DIR = BASE_DIR / 'data' / 'radar' -#HODO_IMAGES = BASE_DIR / 'assets'/ 'hodographs' +RADAR_DIR = Path(sys.argv[6]) +HODO_IMAGES = Path(sys.argv[7]) THIS_RADAR = RADAR_DIR / radar_id os.makedirs(THIS_RADAR, exist_ok=True) DOWNLOADS = THIS_RADAR / 'downloads' diff --git a/scripts/munger.py b/scripts/munger.py index 6ac4120..3be5612 100644 --- a/scripts/munger.py +++ b/scripts/munger.py @@ -19,8 +19,9 @@ from datetime import datetime, timedelta, timezone import time import pytz +from pathlib import Path -from config import RADAR_DIR, POLLING_DIR, L2MUNGER_FILEPATH, DEBZ_FILEPATH +#from config import RADAR_DIR, POLLING_DIR, L2MUNGER_FILEPATH, DEBZ_FILEPATH class Munger(): """ @@ -44,19 +45,24 @@ class Munger(): If None, will use original radar location """ - def __init__(self, original_rda, playback_start, duration, timeshift, new_rda='None'): + def __init__(self, original_rda, playback_start, duration, timeshift, RADAR_DIR, POLLING_DIR, + L2MUNGER_FILEPATH, DEBZ_FILEPATH, new_rda='None'): self.original_rda = original_rda.upper() print(self.original_rda) - self.source_directory = RADAR_DIR / self.original_rda / 'downloads' + self.source_directory = Path(f"{RADAR_DIR}/{self.original_rda}/downloads") os.makedirs(self.source_directory, exist_ok=True) self.playback_start = datetime.strptime(playback_start,"%Y-%m-%d %H:%M").replace(tzinfo=pytz.UTC) self.duration = duration self.seconds_shift = int(timeshift) # Needed for data passed in via command line. self.new_rda = new_rda - self.this_radar_polling_dir = POLLING_DIR / self.original_rda + self.polling_dir = Path(POLLING_DIR) + self.assets_dir = self.polling_dir.parent + self.this_radar_polling_dir = Path(f"{POLLING_DIR}/{self.original_rda}") if self.new_rda != 'None': - self.this_radar_polling_dir = POLLING_DIR / self.new_rda.upper() + self.this_radar_polling_dir = Path(f"{POLLING_DIR}/{self.new_rda.upper()}") + self.l2munger_filepath = Path(L2MUNGER_FILEPATH) + self.debz_filepath = Path(DEBZ_FILEPATH) os.makedirs(self.this_radar_polling_dir, exist_ok=True) self.copy_l2munger_executable() @@ -70,9 +76,9 @@ def copy_l2munger_executable(self) -> None: The compiled l2munger executable needs to be in the source radar files directory to work properly """ - chmod_cmd = f'chmod 775 {L2MUNGER_FILEPATH}' + chmod_cmd = f'chmod 775 {self.l2munger_filepath}' os.system(chmod_cmd) - cp_cmd = f'cp {L2MUNGER_FILEPATH} {self.source_directory}' + cp_cmd = f'cp {self.l2munger_filepath} {self.source_directory}' os.system(cp_cmd) @@ -102,7 +108,7 @@ def uncompress_files(self) -> None: if 'V0' in filename_str: # Keep existing logic for .V06 and .V08 files - command_str = f'python {DEBZ_FILEPATH} {filename_str} {filename_str}.uncompressed' + command_str = f'python {self.debz_filepath} {filename_str} {filename_str}.uncompressed' os.system(command_str) else: print(f'File type not recognized: {filename_str}') @@ -163,7 +169,7 @@ def munge_files(self) -> None: munges radar files to start at the reference time also changes RDA location """ - fout = open('/data/cloud-radar-server/assets/file_times.txt', 'w', encoding='utf-8') + fout = open(f'{self.assets_dir}/file_times.txt', 'w', encoding='utf-8') os.chdir(self.source_directory) self.source_files = list(self.source_directory.glob('*uncompressed')) for uncompressed_file in self.uncompressed_files: @@ -198,5 +204,6 @@ def munge_files(self) -> None: playback_end_time = playback_start_time + timedelta(minutes=int(DURATION)) Munger(ORIG_RDA, playback_start_str, DURATION, seconds_shift, NEW_RDA) else: - Munger(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) + Munger(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], + sys.argv[6], sys.argv[7], sys.argv[8], sys.argv[9]) #original_rda, playback_start, duration, timeshift, new_rda, playback_speed=1.5 diff --git a/scripts/obs_placefile.py b/scripts/obs_placefile.py index ca34dab..1c759f4 100644 --- a/scripts/obs_placefile.py +++ b/scripts/obs_placefile.py @@ -16,7 +16,7 @@ import pytz import requests from dotenv import load_dotenv -from config import PLACEFILES_DIR +#from config import PLACEFILES_DIR load_dotenv() API_TOKEN = os.getenv("SYNOPTIC_API_TOKEN") #PLACEFILES_DIR = os.path.join(os.getcwd(),'assets','placefiles') @@ -82,7 +82,8 @@ class Mesowest(): """ - def __init__(self,lat,lon, event_timestr, duration=None): + def __init__(self,lat,lon, event_timestr, PLACEFILES_DIR, duration=None): + self.placefiles_dir = Path(PLACEFILES_DIR) self.lat = float(lat) self.lon = float(lon) self.event_timestr = event_timestr @@ -306,23 +307,23 @@ def build_placefile(self): self.all_placefile += f'{obj_head} Threshold: {other_zoom}\n{rt_txt} End:\n\n' - with open(os.path.join(PLACEFILES_DIR, 'temp.txt'), 'w', encoding='utf8') as outfile: + with open(os.path.join(self.placefiles_dir, 'temp.txt'), 'w', encoding='utf8') as outfile: outfile.write(self.placefile) - with open(os.path.join(PLACEFILES_DIR, 'wind.txt'), 'w', encoding='utf8') as outfile: + with open(os.path.join(self.placefiles_dir, 'wind.txt'), 'w', encoding='utf8') as outfile: outfile.write(self.wind_placefile) - with open(os.path.join(PLACEFILES_DIR, 'dwpt.txt'), 'w', encoding='utf8') as outfile: + with open(os.path.join(self.placefiles_dir, 'dwpt.txt'), 'w', encoding='utf8') as outfile: outfile.write(self.dewpoint_placefile) - with open(os.path.join(PLACEFILES_DIR, 'latest_surface_observations.txt'), 'w', encoding='utf8') as outfile: + with open(os.path.join(self.placefiles_dir, 'latest_surface_observations.txt'), 'w', encoding='utf8') as outfile: outfile.write(self.all_placefile) - with open(os.path.join(PLACEFILES_DIR, 'latest_surface_observations.txt'),'r',encoding='utf8') as fin: + with open(os.path.join(self.placefiles_dir, 'latest_surface_observations.txt'),'r',encoding='utf8') as fin: data = fin.readlines() - with open(os.path.join(PLACEFILES_DIR, 'latest_surface_observations_lg.txt'), 'w', encoding='utf8') as largefout: - with open(os.path.join(PLACEFILES_DIR, 'latest_surface_observations_xlg.txt'), 'w', encoding='utf8') as xlargefout: + with open(os.path.join(self.placefiles_dir, 'latest_surface_observations_lg.txt'), 'w', encoding='utf8') as largefout: + with open(os.path.join(self.placefiles_dir, 'latest_surface_observations_xlg.txt'), 'w', encoding='utf8') as xlargefout: for line in data: if 'Font: 1' in line: largefout.write('Font: 1, 14, 1, "Arial"\n') @@ -517,5 +518,5 @@ def build_object(self,new_str,short,this_dict): if __name__ == "__main__": - test = Mesowest(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + test = Mesowest(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) #test = Mesowest(42.9634, -85.6681, '2024-06-01 23:15:20 UTC', 60) diff --git a/scripts/update_dir_list.py b/scripts/update_dir_list.py index 16666dd..ae05034 100644 --- a/scripts/update_dir_list.py +++ b/scripts/update_dir_list.py @@ -8,7 +8,8 @@ import sys from datetime import datetime import pytz -from config import POLLING_DIR +from pathlib import Path +#from config import POLLING_DIR class UpdateDirList(): @@ -32,10 +33,10 @@ class UpdateDirList(): - def __init__(self, radar: str, current_playback_timestr: str, initialize: bool = False): + def __init__(self, radar: str, current_playback_timestr: str, POLLING_DIR: str, initialize: bool = False): self.radar = radar.upper() self.current_playback_timestr = current_playback_timestr - self.this_radar_polling_directory = POLLING_DIR / self.radar + self.this_radar_polling_directory = Path(f'{POLLING_DIR}/{self.radar}') print(self.this_radar_polling_directory) self.dirlist_file = self.this_radar_polling_directory / 'dir.list' self.current_playback_time = None @@ -98,4 +99,4 @@ def update_dirlist(self) -> None: if __name__ == "__main__": #this_radar = 'KGRR' #this_playback_time = '2024-06-01 23:15' - UpdateDirList(sys.argv[1],sys.argv[2],sys.argv[3]) + UpdateDirList(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4]) diff --git a/scripts/update_hodo_page.py b/scripts/update_hodo_page.py index c216cee..c21c24e 100644 --- a/scripts/update_hodo_page.py +++ b/scripts/update_hodo_page.py @@ -8,17 +8,15 @@ from datetime import datetime import pytz -#HODO_DIR = '/data/cloud-radar-server/assets/hodographs' -#HODOGRAPHS_PAGE = '/data/cloud-radar-server/assets/hodographs.html' dir_parts = Path.cwd().parts if 'C:\\' in dir_parts: PLATFORM = 'windows' - HODOGRAPHS_DIR = 'C:/data/scripts/cloud-radar-server/assets/hodographs' - HODOGRAPHS_PAGE = 'C:/data/scripts/cloud-radar-server/assets/hodographs.html' - #link_base = "http://localhost:8050/assets" - #cloud = False -else: - from config import HODOGRAPHS_DIR, HODOGRAPHS_PAGE +# HODOGRAPHS_DIR = 'C:/data/scripts/cloud-radar-server/assets/hodographs' +# HODOGRAPHS_PAGE = 'C:/data/scripts/cloud-radar-server/assets/hodographs.html' +# #link_base = "http://localhost:8050/assets" +# #cloud = False +#else: +# from config import HODOGRAPHS_DIR, HODOGRAPHS_PAGE HEAD = """ @@ -58,8 +56,10 @@ class UpdateHodoHTML(): If True, the page will be initialized with a message that graphics are not available If False, the page will be updated with "available" hodographs based on the current playback time """ - def __init__(self, clock_str: str): + def __init__(self, clock_str: str, hodographs_dir: str, hodographs_page: str): self.clock_str = clock_str + self.hodographs_dir = hodographs_dir + self.hodographs_page = hodographs_page if self.clock_str == 'None': self.initialize_hodo_page() else: @@ -78,7 +78,7 @@ def make_valid_hodo_list(self) -> list: Returns a list of valid hodographs based on the current playback time """ valid_hodo_list = [] - self.image_files = [f for f in os.listdir(HODOGRAPHS_DIR) if f.endswith('.png') or f.endswith('.jpg')] + self.image_files = [f for f in os.listdir(self.hodographs_dir) if f.endswith('.png') or f.endswith('.jpg')] self.image_files.sort() try: self.first_image_path = self.image_files[0] @@ -98,7 +98,7 @@ def initialize_hodo_page(self) -> None: """ Initializes the hodographs.html page with a message that graphics are not available """ - with open(HODOGRAPHS_PAGE, 'w', encoding='utf-8') as fout: + with open(self.hodographs_page, 'w', encoding='utf-8') as fout: fout.write(HEAD_NOLIST) fout.write('

\n') fout.write('

Graphics not available, check back later!

\n') @@ -112,7 +112,7 @@ def update_hodo_page(self) -> None: if len(self.valid_hodo_list) == 0: self.initialize_hodo_page() else: - with open(HODOGRAPHS_PAGE, 'w', encoding='utf-8') as fout: + with open(self.hodographs_page, 'w', encoding='utf-8') as fout: fout.write(HEAD) for filename in self.valid_hodo_list: file_time = datetime.strptime(filename[-19:-4], '%Y%m%d_%H%M%S').replace(tzinfo=pytz.UTC).timestamp() @@ -237,12 +237,12 @@ def update_hodographs_loop_page(self) -> None: """ - with open(HODOGRAPHS_PAGE, 'w', encoding='utf-8') as fout: + with open(self.hodographs_page, 'w', encoding='utf-8') as fout: fout.write(html_content) if __name__ == "__main__": if PLATFORM == 'windows': - UpdateHodoHTML('2024-09-01 23:15') + UpdateHodoHTML('2024-09-01 23:15', sys.argv[2], sys.argv[3]) else: - UpdateHodoHTML(sys.argv[1]) + UpdateHodoHTML(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/utils.py b/utils.py index a9a76e0..633c009 100644 --- a/utils.py +++ b/utils.py @@ -5,10 +5,9 @@ from glob import glob import psutil import pandas as pd -import json -import config as cfg +import config -def exec_script(script_path, args): +def exec_script(script_path, args, session_id): """ Generalized function to run application scripts. subprocess.run() or similar is required for tracking of spawned python processes and termination if requested @@ -22,15 +21,15 @@ def exec_script(script_path, args): PYTHON = r"C:\\Users\\lee.carlaw\\environments\\cloud-radar\\Scripts\python.exe" output = {} + env = os.environ.copy() + env['session_id'] = session_id # add the unique session id to the process try: # Execute scripts as python module to allow config import from higher-level dir: # python -m scripts.script-name parts = script_path.parts arg = f"{script_path.parts[-2]}.{parts[-1]}".replace(".py", "") process = subprocess.Popen([PYTHON, '-m', arg] + args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - #process = subprocess.Popen([PYTHON, script_path] + args, stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) + stderr=subprocess.PIPE, env=env) output['stdout'], output['stderr'] = process.communicate() output['returncode'] = process.returncode except Exception as e: @@ -50,15 +49,20 @@ def get_app_processes(): info = proc.info if ('python' in info['name'] or 'wgrib2' in info['name']) and \ len(info['cmdline']) > 1: + + # Tag process with its unique session id (set in exec_script above) + info['session_id'] = proc.environ().get('session_id') processes.append(info) except: pass return processes -def cancel_all(sa): +def cancel_all(sa, session_id): """ This function is invoked when the user clicks the Cancel button in the app. See app.cancel_scripts. + + Updated to only kill processes associated with this unique session id """ processes = get_app_processes() @@ -71,18 +75,20 @@ def cancel_all(sa): # shouldn't be? # ****************************************************************************** for process in processes: - name = process['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] - if process['cmdline'][1] == '-m': - name = process['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] - if process['name'] == 'wgrib2': name = 'wgrib2' - - if name in cfg.scripts_list: - sa.log.info(f"Killing process: {name} with pid: {process['pid']}") - os.kill(process['pid'], signal.SIGTERM) - - if len(process['cmdline']) >= 3 and 'multiprocessing' in process['cmdline'][2]: - sa.log.info(f"Killing spawned multi-process with pid: {process['pid']}") - os.kill(process['pid'], signal.SIGTERM) + process_session_id = process['session_id'] + if process_session_id == session_id: + name = process['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] + if process['cmdline'][1] == '-m': + name = process['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] + if process['name'] == 'wgrib2': name = 'wgrib2' + + if name in config.scripts_list: + sa.log.info(f"Killing process: {name} with pid: {process['pid']}") + os.kill(process['pid'], signal.SIGTERM) + + if len(process['cmdline']) >= 3 and 'multiprocessing' in process['cmdline'][2]: + sa.log.info(f"Killing spawned multi-process with pid: {process['pid']}") + os.kill(process['pid'], signal.SIGTERM) def calc_completion_percentage(expected_files, files_on_system): @@ -104,34 +110,31 @@ def radar_monitor(sa): percent_complete = calc_completion_percentage(expected_files, files_on_system) return percent_complete, files_on_system -def munger_monitor(sa): +def munger_monitor(sa, cfg): expected_files = list(sa.radar_files_dict.values()) # Are the mungered files always .gz? - #files_on_system = glob(f"{sa.polling_dir}/**/*.gz", recursive=True) - files_on_system = glob(f"{cfg.POLLING_DIR}/**/*.gz", recursive=True) + files_on_system = glob(f"{cfg['POLLING_DIR']}/**/*.gz", recursive=True) percent_complete = calc_completion_percentage(expected_files, files_on_system) return percent_complete -def surface_placefile_monitor(_sa): +def surface_placefile_monitor(_sa, cfg): filenames = [ 'wind.txt', 'temp.txt', 'latest_surface_observations.txt', 'latest_surface_observations_lg.txt', 'latest_surface_observations_xlg.txt' ] - #expected_files = [f"{sa.placefiles_dir}/{i}" for i in filenames] - expected_files = [f"{cfg.PLACEFILES_DIR}/{i}" for i in filenames] + expected_files = [f"{cfg['PLACEFILES_DIR']}/{i}" for i in filenames] files_on_system = [x for x in expected_files if os.path.exists(x)] #percent_complete = calc_completion_percentage(expected_files, files_on_system) return len(files_on_system), len(expected_files) -def nse_status_checker(_sa): +def nse_status_checker(_sa, cfg): """ Read in model status text file and query associated file sizes. """ - #filename = f"{sa.data_dir}/model_data/model_list.txt" - filename = f"{cfg.DATA_DIR}/model_data/model_list.txt" + filename = f"{cfg['MODEL_DIR']}/model_list.txt" output = [] warning_text = "" if os.path.exists(filename): From ef7542164f66945e758ec5de2a34dd23877a2d81 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sat, 24 Aug 2024 20:49:45 -0500 Subject: [PATCH 02/23] Initial updates to move RadarSimulator object variables to dict in dcc.Store. Radar download and mungering dealt with. Monitoring broken for now. --- app.py | 354 ++++++++++++++++++++++++++++++++----------- config.py | 1 + layout_components.py | 7 +- 3 files changed, 271 insertions(+), 91 deletions(-) diff --git a/app.py b/app.py index f1425e3..ae3674a 100644 --- a/app.py +++ b/app.py @@ -57,6 +57,78 @@ TIME_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z" TOKEN = 'INSERT YOUR MAPBOX TOKEN HERE' +def date_time_string(dt) -> str: + """ + Converts a datetime object to a string. + """ + return datetime.strftime(dt, "%Y-%m-%d %H:%M") + +def make_simulation_times(sa) -> dict: + """ + playback_start_time: datetime object + - the time the simulation starts. + - set to (current UTC time rounded to nearest 30 minutes then minus 2hrs) + - This is "recent enough" for GR2Analyst to poll data + playback_timer: datetime object + - the "current" displaced realtime during the playback + event_start_time: datetime object + - the historical time the actual event started. + - based on user inputs of the event start time + simulation_time_shift: timedelta object + the difference between the playback start time and the event start time + simulation_seconds_shift: int + the difference between the playback start time and the event start time in seconds + + Variables ending with "_str" are the string representations of the datetime objects + """ + + sa['playback_start'] = datetime.now(pytz.utc) - timedelta(hours=2) + sa['playback_start'] = sa['playback_start'].replace(second=0, microsecond=0) + if sa['playback_start'].minute < 30: + sa['playback_start'] = sa['playback_start'].replace(minute=0) + else: + sa['playback_start'] = sa['playback_start'].replace(minute=30) + + sa['playback_start_str'] = date_time_string(sa['playback_start']) + + sa['playback_end'] = sa['playback_start'] + timedelta(minutes=int(sa['event_duration'])) + sa['playback_end_str'] = date_time_string(sa['playback_end']) + + sa['playback_clock'] = sa['playback_start'] + timedelta(seconds=600) + sa['playback_clock_str'] = date_time_string(sa['playback_clock']) + + sa['event_start_time'] = datetime(sa['event_start_year'], sa['event_start_month'], + sa['event_start_day'], sa['event_start_hour'], + sa['event_start_minute'], second=0, + tzinfo=timezone.utc) + # a timedelta object is not JSON serializable, so cannot be included in the output + # dictionary stored in the dcc.Store object. All references to simulation_time_shift + # will need to use the simulation_seconds_shift reference instead. + simulation_time_shift = sa['playback_start'] - sa['event_start_time'] + sa['simulation_seconds_shift'] = round(simulation_time_shift.total_seconds()) + sa['event_start_str'] = date_time_string(sa['event_start_time']) + increment_list = [] + for t in range(0, int(sa['event_duration']/5) + 1 , 1): + new_time = sa['playback_start'] + timedelta(seconds=t*300) + new_time_str = date_time_string(new_time) + increment_list.append(new_time_str) + + sa['playback_dropdown_dict'] = [{'label': increment, 'value': increment} for increment in increment_list] + return sa + +def create_radar_dict(sa) -> dict: + """ + Creates dictionary of radar sites and their metadata to be used in the simulation. + """ + for _i, radar in enumerate(sa['radar_list']): + sa['lat'] = lc.df[lc.df['radar'] == radar]['lat'].values[0] + sa['lon'] = lc.df[lc.df['radar'] == radar]['lon'].values[0] + asos_one = lc.df[lc.df['radar'] == radar]['asos_one'].values[0] + asos_two = lc.df[lc.df['radar'] == radar]['asos_two'].values[0] + sa['radar_dict'][radar.upper()] = {'lat': sa['lat'], 'lon': sa['lon'], + 'asos_one': asos_one, 'asos_two': asos_two, + 'radar': radar.upper(), 'file_list': []} + ################################################################################################ # ----------------------------- Define class RadarSimulator ----------------------------------- ################################################################################################ @@ -109,7 +181,7 @@ def __init__(self): # This will generate a logfile. Something we'll want to turn on in the future. self.log = self.create_logfile() #UpdateHodoHTML('None', '', '') # set up the hodo page with no images - + def create_logfile(self): """ Creates an initial logfile. Stored in the data dir for now. Call is @@ -233,12 +305,12 @@ def change_playback_time(self,dseconds) -> str: return self.playback_clock_str - def get_days_in_month(self) -> None: - """ - Helper function to determine number of days to display in the dropdown - """ - self.days_in_month = calendar.monthrange( - self.event_start_year, self.event_start_month)[1] + #def get_days_in_month(self) -> None: + # """ + # Helper function to determine number of days to display in the dropdown + # """ + # self.days_in_month = calendar.monthrange( + # self.event_start_year, self.event_start_month)[1] def get_timestamp(self, file: str) -> float: """ @@ -402,10 +474,6 @@ def remove_files_and_dirs(self, cfg) -> None: ################################################################################################ ################################################################################################ ################################################################################################ -sim_day_selection = dbc.Col(html.Div([ - lc.step_day, - dcc.Dropdown(np.arange(1, sa.days_in_month+1), 16, id='start_day', clearable=False)])) - playback_time_options = dbc.Col(html.Div([ dcc.Dropdown(options={'label': 'Sim not started', 'value': ''}, id='change_time', disabled=True, clearable=False)])) @@ -427,17 +495,64 @@ def remove_files_and_dirs(self, cfg) -> None: @app.callback( Output('dynamic_container', 'children'), Output('layout_has_initialized', 'data'), + Output('sim_settings', 'data'), Input('directory_monitor', 'n_intervals'), State('layout_has_initialized', 'data'), - State('dynamic_container', 'children') + State('dynamic_container', 'children'), + State('sim_settings', 'data') ) -def generate_layout(n_intervals, layout_has_initialized, children): +def generate_layout(n_intervals, layout_has_initialized, children, sim_settings): """ Dynamically generate the layout, which was started in the config file to set up the unique session id. This callback should only be executed once at page load in. Thereafter, layout_has_initialized will be set to True """ if not layout_has_initialized['added']: + + # Initialize variables + sim_settings['event_start_year'] = 2024 + sim_settings['event_start_month'] = 7 + sim_settings['days_in_month'] = 30 + sim_settings['event_start_day'] = 12 + sim_settings['event_start_hour'] = 0 + sim_settings['event_start_minute'] = 30 + sim_settings['event_duration'] = 60 + sim_settings['timestring'] = None + sim_settings['number_of_radars'] = 1 + sim_settings['radar_list'] = [] + sim_settings['playback_dropdown_dict'] = {} + sim_settings['radar_dict'] = {} + sim_settings['radar_files_dict'] = {} + #sim_settings['radar'] = None + sim_settings['lat'] = None + sim_settings['lon'] = None + sim_settings['new_radar'] = 'None' + sim_settings['new_lat'] = None + sim_settings['new_lon'] = None + sim_settings['scripts_progress'] = 'Scripts not started' + #self.base_dir = Path.cwd() + sim_settings['playback_initiated'] = False + sim_settings['playback_speed'] = 1.0 + sim_settings['playback_start'] = 'Not Ready' + sim_settings['playback_end'] = 'Not Ready' + sim_settings['playback_start_str'] = 'Not Ready' + sim_settings['playback_end_str'] = 'Not Ready' + sim_settings['playback_current_time'] = 'Not Ready' + sim_settings['playback_clock'] = None + sim_settings['playback_clock_str'] = None + sim_settings['simulation_running'] = False + sim_settings['playback_paused'] = False + + # Settings for date dropdowns moved here to avoid specifying different values in + # the layout + now = datetime.now(pytz.utc) + sim_year_section = dbc.Col(html.Div([lc.step_year, dcc.Dropdown(np.arange(1992, now.year + 1), sim_settings['event_start_year'], id='start_year', clearable=False),])) + sim_month_section = dbc.Col(html.Div([lc.step_month, dcc.Dropdown(np.arange(1, 13), sim_settings['event_start_month'], id='start_month', clearable=False),])) + sim_day_selection = dbc.Col(html.Div([lc.step_day, dcc.Dropdown(np.arange(1, 31), sim_settings['event_start_day'], id='start_day', clearable=False)])) + sim_hour_section = dbc.Col(html.Div([lc.step_hour, dcc.Dropdown(np.arange(0, 24), sim_settings['event_start_hour'], id='start_hour', clearable=False),])) + sim_minute_section = dbc.Col(html.Div([lc.step_minute, dcc.Dropdown([0, 15, 30, 45], sim_settings['event_start_minute'], id='start_minute', clearable=False),])) + sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown(np.arange(0, 240, 15), sim_settings['event_duration'], id='duration', clearable=False),])) + if children is None: children = [] @@ -454,8 +569,8 @@ def generate_layout(n_intervals, layout_has_initialized, children): dbc.Container([ html.Div([html.Div([lc.step_select_time_section, lc.spacer, dbc.Row([ - lc.sim_year_section, lc.sim_month_section, sim_day_selection, - lc.sim_hour_section, lc.sim_minute_section, lc.sim_duration_section, + sim_year_section, sim_month_section, sim_day_selection, + sim_hour_section, sim_minute_section, sim_duration_section, lc.spacer, lc.step_time_confirm])], style={'padding': '1em'}), ], style=lc.section_box)]) ]), lc.spacer, @@ -477,9 +592,9 @@ def generate_layout(n_intervals, layout_has_initialized, children): layout_has_initialized['added'] = True - return children, layout_has_initialized + return children, layout_has_initialized, sim_settings - return children, layout_has_initialized + return children, layout_has_initialized, sim_settings ################################################################################################ ################################################################################################ @@ -491,44 +606,64 @@ def generate_layout(n_intervals, layout_has_initialized, children): ################################################################################################ @app.callback( - [Output('show_radar_selection_feedback', 'children'), + Output('show_radar_selection_feedback', 'children'), Output('confirm_radars_btn', 'children'), - Output('confirm_radars_btn', 'disabled')], + Output('confirm_radars_btn', 'disabled'), + Output('sim_settings', 'data', allow_duplicate=True), Input('radar_quantity', 'value'), Input('graph', 'clickData'), + State('sim_settings', 'data'), prevent_initial_call=True ) -def display_click_data(quant_str: str, click_data: dict): +def display_click_data(quant_str: str, click_data: dict, sim_settings: dict): """ Any time a radar site is clicked, this function will trigger and update the radar list. + + The allow_duplicate=True addition seems to cause this callback to fire repeatedly + event when no user input has triggered. Necessitated some workarounds. """ # initially have to make radar selections and can't finalize select_action = 'Make' btn_deactivated = True - triggered_id = ctx.triggered_id - if triggered_id == 'radar_quantity': - sa.number_of_radars = int(quant_str[0:1]) - sa.radar_list = [] - sa.radar_dict = {} - return f'Use map to select {quant_str}', f'{select_action} selections', True + #triggered_id = ctx.triggered_id + sim_settings['number_of_radars'] = int(quant_str[0:1]) + + # This block was getting triggered repeatedly when adding allow_duplicate=True. + #if triggered_id == 'radar_quantity' and len(sim_settings['radar_list']) != 0: + # sa.number_of_radars = int(quant_str[0:1]) + # sa.radar_list = [] + # sa.radar_dict = {} + + # sim_settings['number_of_radars'] = int(quant_str[0:1]) + # sim_settings['radar_list'] = [] + # sim_settings['radar_dict'] = {} + + # return f'Use map to select {quant_str}', f'{select_action} selections', True, sim_settings + add_to_list = False try: - sa.radar = click_data['points'][0]['customdata'] - print(f"Selected radar: {sa.radar}") + radar = click_data['points'][0]['customdata'] + if len(sim_settings['radar_list']) > 0: + if radar != sim_settings['radar_list'][-1] and radar not in sim_settings['radar_list']: + add_to_list = True + else: + add_to_list = True except (KeyError, IndexError, TypeError): - return 'No radar selected ...', f'{select_action} selections', True - - sa.radar_list.append(sa.radar) - if len(sa.radar_list) > sa.number_of_radars: - sa.radar_list = sa.radar_list[-sa.number_of_radars:] - if len(sa.radar_list) == sa.number_of_radars: + return 'No radar selected ...', f'{select_action} selections', True, sim_settings + + #if triggered_id != 'radar_quantity': + if add_to_list: + sim_settings['radar_list'].append(radar) + if len(sim_settings['radar_list']) > sim_settings['number_of_radars']: + sim_settings['radar_list'] = sim_settings['radar_list'][-sim_settings['number_of_radars']:] + if len(sim_settings['radar_list']) == sim_settings['number_of_radars']: select_action = 'Finalize' btn_deactivated = False - print(f"Radar list: {sa.radar_list}") - listed_radars = ', '.join(sa.radar_list) - return listed_radars, f'{select_action} selections', btn_deactivated + #print(f"Radar list: {sim_settings['radar_list']}") + listed_radars = ', '.join(sim_settings['radar_list']) + return listed_radars, f'{select_action} selections', btn_deactivated, sim_settings @app.callback( @@ -598,16 +733,17 @@ def transpose_radar(value): # ----------------------------- Run Scripts button -------------------------------------------- ################################################################################################ -def query_radar_files(cfg): +def query_radar_files(cfg, sim_settings): """ Get the radar files from the AWS bucket. This is a preliminary step to build the progess bar. """ # Need to reset the expected files dictionary with each call. Otherwise, if a user # cancels a request, the previously-requested files will still be in the dictionary. - sa.radar_files_dict = {} - for _r, radar in enumerate(sa.radar_list): + #sa.radar_files_dict = {} + sim_settings['radar_files_dict'] = {} + for _r, radar in enumerate(sim_settings['radar_list']): radar = radar.upper() - args = [radar, f'{sa.event_start_str}', str(sa.event_duration), str(False), + args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), str(False), cfg['RADAR_DIR']] sa.log.info(f"Passing {args} to Nexrad.py") results = utils.exec_script(Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) @@ -617,7 +753,8 @@ def query_radar_files(cfg): json_data = results['stdout'].decode('utf-8') sa.log.info(f"Nexrad.py returned with {json_data}") - sa.radar_files_dict.update(json.loads(json_data)) + #sa.radar_files_dict.update(json.loads(json_data)) + sim_settings['radar_files_dict'].update(json.loads(json_data)) return results @@ -649,15 +786,17 @@ def call_function(func, *args, **kwargs): return result -def run_with_cancel_button(cfg): +def run_with_cancel_button(cfg, sim_settings): """ This version of the script-launcher trying to work in cancel button """ UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) sa.scripts_progress = 'Setting up files and times' + sim_settings['scripts_progress'] = 'Setting up files and times' # determine actual event time, playback time, diff of these two sa.make_simulation_times() + make_simulation_times(sim_settings) # clean out old files and directories try: sa.remove_files_and_dirs(cfg) @@ -667,29 +806,30 @@ def run_with_cancel_button(cfg): # based on list of selected radars, create a dictionary of radar metadata try: sa.create_radar_dict() + create_radar_dict(sim_settings) sa.copy_grlevel2_cfg_file(cfg) except Exception as e: sa.log.exception("Error creating radar dict or config file: ", exc_info=True) # Create initial dictionary of expected radar files - if len(sa.radar_list) > 0: - res = call_function(query_radar_files, cfg) + if len(sim_settings['radar_list']) > 0: + res = call_function(query_radar_files, cfg, sim_settings) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return - for _r, radar in enumerate(sa.radar_list): + for _r, radar in enumerate(sim_settings['radar_list']): radar = radar.upper() try: - if sa.new_radar == 'None': + if sim_settings['new_radar'] == 'None': new_radar = radar else: - new_radar = sa.new_radar.upper() + new_radar = sim_settings['new_radar'].upper() except Exception as e: sa.log.exception("Error defining new radar: ", exc_info=True) # Radar download - args = [radar, str(sa.event_start_str), str(sa.event_duration), str(True), - cfg['RADAR_DIR']] + args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), + str(True), cfg['RADAR_DIR']] res = call_function(utils.exec_script, Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: @@ -758,7 +898,8 @@ def run_with_cancel_button(cfg): @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), [Input('run_scripts_btn', 'n_clicks'), - State('configs', 'data')], + State('configs', 'data'), + State('sim_settings', 'data')], prevent_initial_call=True, running=[ (Output('start_year', 'disabled'), True, False), @@ -778,7 +919,7 @@ def run_with_cancel_button(cfg): (Output('change_time', 'disabled'), True, False), # wait to enable change time dropdown (Output('cancel_scripts', 'disabled'), False, True), ]) -def launch_simulation(n_clicks, configs) -> None: +def launch_simulation(n_clicks, configs, sim_settings) -> None: """ This function is called when the "Run Scripts" button is clicked. It will execute the necessary scripts to simulate radar operations, create hodographs, and transpose placefiles. @@ -789,7 +930,7 @@ def launch_simulation(n_clicks, configs) -> None: if config.PLATFORM == 'WINDOWS': sa.make_simulation_times() else: - run_with_cancel_button(configs) + run_with_cancel_button(configs, sim_settings) ################################################################################################ # ----------------------------- Monitoring and reporting script status ------------------------ @@ -819,10 +960,11 @@ def cancel_scripts(n_clicks, SESSION_ID) -> None: Output('model_status_warning', 'children'), Output('show_script_progress', 'children', allow_duplicate=True), [Input('directory_monitor', 'n_intervals'), - State('configs', 'data')], + State('configs', 'data'), + State('sim_settings', 'data')], prevent_initial_call=True ) -def monitor(_n, cfg): +def monitor(_n, cfg, sim_settings): """ This function is called every second by the directory_monitor interval. It (1) checks the status of the various scripts and reports them to the front-end application and @@ -1054,34 +1196,41 @@ def update_playback_speed(selected_speed): ################################################################################################ # ----------------------------- Time Selection Summary and Callbacks -------------------------- ################################################################################################ - @app.callback( Output('show_time_data', 'children'), + Output('sim_settings', 'data', allow_duplicate=True), Input('start_year', 'value'), Input('start_month', 'value'), Input('start_day', 'value'), Input('start_hour', 'value'), Input('start_minute', 'value'), Input('duration', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' ) -def get_sim(_yr, _mo, _dy, _hr, _mn, _dur) -> str: +def get_sim(_yr, _mo, _dy, _hr, _mn, _dur, sim_settings) -> str: """ Changes to any of the Inputs above will trigger this callback function to update the time summary displayed on the page. Variables already have been stored in sa object for use in scripts so don't need to be explicitly returned here. """ - sa.make_simulation_times() - line1 = f'{sa.event_start_str}Z ____ {sa.event_duration} minutes' - return line1 + sim_settings = make_simulation_times(sim_settings) + line1 = f'{sim_settings['event_start_str']}Z ____ {sim_settings['event_duration']} minutes' + return line1, sim_settings -@app.callback(Output('start_year', 'value'), Input('start_year', 'value')) -def get_year(start_year) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('start_year', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_year(start_year, sim_settings) -> int: """ - Updates the start year variable in the sa object + Updates the start year variable """ - sa.event_start_year = start_year - return sa.event_start_year + sim_settings['event_start_year'] = start_year + return sim_settings @app.callback( @@ -1097,49 +1246,74 @@ def update_day_dropdown(selected_year, selected_month): return day_options -@app.callback(Output('start_month', 'value'), Input('start_month', 'value')) -def get_month(start_month) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('start_month', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_month(start_month, sim_settings) -> int: """ - Updates the start month variable in the sa object + Updates the start month variable """ - sa.event_start_month = start_month - return sa.event_start_month + sim_settings['event_start_month'] = start_month + return sim_settings -@app.callback(Output('start_day', 'value'), Input('start_day', 'value')) -def get_day(start_day) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('start_day', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_day(start_day, sim_settings) -> int: """ - Updates the start day variable in the sa object + Updates the start day variable """ - sa.event_start_day = start_day - return sa.event_start_day + sim_settings['event_start_day'] = start_day + return sim_settings -@app.callback(Output('start_hour', 'value'), Input('start_hour', 'value')) -def get_hour(start_hour) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('start_hour', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_hour(start_hour, sim_settings) -> int: """ - Updates the start hour variable in the sa object + Updates the start hour variable """ - sa.event_start_hour = start_hour - return sa.event_start_hour + sim_settings['event_start_hour'] = start_hour + return sim_settings -@app.callback(Output('start_minute', 'value'), Input('start_minute', 'value')) -def get_minute(start_minute) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('start_minute', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_minute(start_minute, sim_settings) -> int: """ - Updates the start minute variable in the sa object + Updates the start minute variable """ - sa.event_start_minute = start_minute - return sa.event_start_minute + sim_settings['event_start_minute'] = start_minute + return sim_settings -@app.callback(Output('duration', 'value'), Input('duration', 'value')) -def get_duration(duration) -> int: +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('duration', 'value'), + State('sim_settings', 'data'), + prevent_initial_call='initial_duplicate' +) +def get_duration(duration, sim_settings) -> int: """ - Updates the event duration (in minutes) in the sa object + Updates the event duration (in minutes) """ - sa.event_duration = duration - return sa.event_duration + sim_settings['event_duration'] = duration + return sim_settings ################################################################################################ # ----------------------------- Start app ----------------------------------------------------- diff --git a/config.py b/config.py index 7e2f32a..a393ca4 100644 --- a/config.py +++ b/config.py @@ -61,6 +61,7 @@ def init_layout(): dcc.Store(id='session_id', data=session_id, storage_type='session'), dcc.Interval(id='broadcast_session_id', interval=1, n_intervals=0, max_intervals=1), dcc.Store(id='configs', data={}), + dcc.Store(id='sim_settings', data={}), # Elements needed to set up the layout on page load by app.py dcc.Interval(id='directory_monitor', interval=1000), diff --git a/layout_components.py b/layout_components.py index 78d7c12..619e36e 100644 --- a/layout_components.py +++ b/layout_components.py @@ -14,7 +14,7 @@ load_dotenv() MAP_TOKEN = os.getenv("MAPBOX_TOKEN") -now = datetime.now(pytz.utc) +#now = datetime.now(pytz.utc) df = pd.read_csv('radars.csv', dtype={'lat': float, 'lon': float}) # df = pd.read_csv('radars_no_tdwr.csv', dtype={'lat': float, 'lon': float}) @@ -143,6 +143,10 @@ step_minute = html.Div(children="Minute", style=time_headers) step_duration = html.Div(children="Duration", style=time_headers) +# Date settings (sim_year_section, sim_month_section, etc.) moved to app.py +''' +# Moved these into the app in order to control defaults more easily and remove the +# potential to specify different values in two locations sim_year_section = dbc.Col(html.Div([step_year, dcc.Dropdown(np.arange(1992, now.year + 1), now.year, id='start_year', clearable=False),])) @@ -158,6 +162,7 @@ sim_duration_section = dbc.Col(html.Div([ step_duration, dcc.Dropdown(np.arange(0, 240, 15), 60, id='duration', clearable=False),])) +''' CONFIRM_TIMES_TEXT = "Confirm start time and duration -->" confirm_times_section = dbc.Col( From ca0d6d4825299a7e7717d4ce6bd0c2e2d0767dcd Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sat, 24 Aug 2024 22:07:10 -0500 Subject: [PATCH 03/23] Move logging to top of app. One log file for everything for the time being. --- app.py | 93 ++++++++++++++++++++++++++++++++++++++++++------------- config.py | 4 +-- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/app.py b/app.py index ae3674a..2d75b37 100644 --- a/app.py +++ b/app.py @@ -55,7 +55,49 @@ # Regular expressions. First one finds lat/lon pairs, second finds the timestamps. LAT_LON_REGEX = "[0-9]{1,2}.[0-9]{1,100},[ ]{0,1}[|\\s-][0-9]{1,3}.[0-9]{1,100}" TIME_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z" -TOKEN = 'INSERT YOUR MAPBOX TOKEN HERE' +#TOKEN = 'INSERT YOUR MAPBOX TOKEN HERE' + +# Configure logging +""" +Idea is to move all of these functions to some other utility script within the main dir +""" +def create_logfile(LOG_DIR): + """ + Generate the main logfile for the download and processing scripts. + """ + logging.basicConfig( + filename=f'{LOG_DIR}/scripts.txt', # Log file location + level=logging.INFO, # Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format='%(levelname)s %(asctime)s :: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S" + ) + +def copy_grlevel2_cfg_file(cfg) -> None: + """ + Ensures a grlevel2.cfg file is copied into the polling directory. + This file is required for GR2Analyst to poll for radar data. + """ + source = f"{cfg['BASE_DIR']}/grlevel2.cfg" + destination = f"{cfg['POLLING_DIR']}/grlevel2.cfg" + try: + shutil.copyfile(source, destination) + except Exception as e: + print(f"Error copying {source} to {destination}: {e}") + +def remove_files_and_dirs(cfg) -> None: + """ + Cleans up files and directories from the previous simulation so these datasets + are not included in the current simulation. + """ + dirs = [cfg['RADAR_DIR'], cfg['POLLING_DIR'], cfg['HODOGRAPHS_DIR'], cfg['MODEL_DIR'], + cfg['PLACEFILES_DIR']] + for directory in dirs: + for root, dirs, files in os.walk(directory, topdown=False): + for name in files: + if name != 'grlevel2.cfg': + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) def date_time_string(dt) -> str: """ @@ -499,9 +541,10 @@ def remove_files_and_dirs(self, cfg) -> None: Input('directory_monitor', 'n_intervals'), State('layout_has_initialized', 'data'), State('dynamic_container', 'children'), - State('sim_settings', 'data') + State('sim_settings', 'data'), + State('configs', 'data') ) -def generate_layout(n_intervals, layout_has_initialized, children, sim_settings): +def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, configs): """ Dynamically generate the layout, which was started in the config file to set up the unique session id. This callback should only be executed once at page load in. @@ -594,6 +637,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings) return children, layout_has_initialized, sim_settings + create_logfile(configs['LOG_DIR']) return children, layout_has_initialized, sim_settings ################################################################################################ @@ -743,16 +787,16 @@ def query_radar_files(cfg, sim_settings): sim_settings['radar_files_dict'] = {} for _r, radar in enumerate(sim_settings['radar_list']): radar = radar.upper() - args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), str(False), - cfg['RADAR_DIR']] - sa.log.info(f"Passing {args} to Nexrad.py") + args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), + str(False), cfg['RADAR_DIR']] + logging.info(f"{cfg['SESSION_ID']} :: Passing {args} to Nexrad.py") results = utils.exec_script(Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if results['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: - sa.log.warning(f"User cancelled query_radar_files()") + logging.warning(f"{cfg['SESSION_ID']} :: User cancelled query_radar_files()") break json_data = results['stdout'].decode('utf-8') - sa.log.info(f"Nexrad.py returned with {json_data}") + logging.info(f"{cfg['SESSION_ID']} :: Nexrad.py returned with {json_data}") #sa.radar_files_dict.update(json.loads(json_data)) sim_settings['radar_files_dict'].update(json.loads(json_data)) @@ -774,15 +818,16 @@ def run_hodo_script(args) -> None: def call_function(func, *args, **kwargs): - if len(args) > 1: - sa.log.info(f"Sending {args[1]} to {args[0]}") + # For the main script calls + if len(args) > 2: + logging.info(f"Sending {args[1]} to {args[0]}") result = func(*args, **kwargs) if len(result['stderr']) > 0: - sa.log.error(result['stderr'].decode('utf-8')) + logging.error(result['stderr'].decode('utf-8')) if 'exception' in result: - sa.log.error(f"Exception {result['exception']} occurred in {func.__name__}") + logging.error(f"Exception {result['exception']} occurred in {func.__name__}") return result @@ -792,24 +837,27 @@ def run_with_cancel_button(cfg, sim_settings): """ UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - sa.scripts_progress = 'Setting up files and times' + #sa.scripts_progress = 'Setting up files and times' sim_settings['scripts_progress'] = 'Setting up files and times' # determine actual event time, playback time, diff of these two - sa.make_simulation_times() - make_simulation_times(sim_settings) + #sa.make_simulation_times() + sim_settings = make_simulation_times(sim_settings) # clean out old files and directories try: - sa.remove_files_and_dirs(cfg) + remove_files_and_dirs(cfg) + #sa.remove_files_and_dirs(cfg) except Exception as e: - sa.log.exception("Error removing files and directories: ", exc_info=True) + logging.exception("Error removing files and directories: ", exc_info=True) # based on list of selected radars, create a dictionary of radar metadata try: sa.create_radar_dict() create_radar_dict(sim_settings) + sa.copy_grlevel2_cfg_file(cfg) + copy_grlevel2_cfg_file(cfg) except Exception as e: - sa.log.exception("Error creating radar dict or config file: ", exc_info=True) + logging.exception("Error creating radar dict or config file: ", exc_info=True) # Create initial dictionary of expected radar files if len(sim_settings['radar_list']) > 0: @@ -825,16 +873,17 @@ def run_with_cancel_button(cfg, sim_settings): else: new_radar = sim_settings['new_radar'].upper() except Exception as e: - sa.log.exception("Error defining new radar: ", exc_info=True) + logging.exception("Error defining new radar: ", exc_info=True) # Radar download - args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), - str(True), cfg['RADAR_DIR']] + args = [radar, str(sim_settings['event_start_str']), + str(sim_settings['event_duration']), str(True), cfg['RADAR_DIR']] res = call_function(utils.exec_script, Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return + ''' # Munger args = [radar, str(sa.playback_start_str), str(sa.event_duration), str(sa.simulation_seconds_shift), cfg['RADAR_DIR'], cfg['POLLING_DIR'], @@ -894,7 +943,7 @@ def run_with_cancel_button(cfg, sim_settings): except Exception as e: print("Error updating hodo html: ", e) sa.log.exception("Error updating hodo html: ", exc_info=True) - + ''' @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), [Input('run_scripts_btn', 'n_clicks'), diff --git a/config.py b/config.py index a393ca4..b913018 100644 --- a/config.py +++ b/config.py @@ -55,7 +55,7 @@ def init_layout(): Initialize the layout with a unique session id. The 'dynamic container' is used within the app to build out the rest of the application layout on page load """ - session_id = f'{time.time_ns()//1000}_{uuid.uuid4()}' + session_id = f'{time.time_ns()//1000}_{uuid.uuid4().hex}' return dbc.Container([ # Elements used to store and track the session id dcc.Store(id='session_id', data=session_id, storage_type='session'), @@ -99,7 +99,7 @@ def broadcast_session_id(n_intervals, session_id): dirs['POLLING_DIR'] = f"{dirs['ASSETS_DIR']}/polling" dirs['MODEL_DIR'] = f"{dirs['DATA_DIR']}/model_data" dirs['RADAR_DIR'] = f"{dirs['DATA_DIR']}/radar" - dirs['LOG_DIR'] = f"{dirs['DATA_DIR']}/logs" + dirs['LOG_DIR'] = f"{dirs['BASE_DIR']}/data/logs" # Need to be updated dirs['LINK_BASE'] = f"https://rssic.nws.noaa.gov/assets/{session_id}" From 80ac13ccbafac2afb36aa305e4ee2bd8bd5779a5 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sat, 24 Aug 2024 22:37:30 -0500 Subject: [PATCH 04/23] Getting munger and session-based radar monitoring working again. --- app.py | 23 +++++++++++++++-------- utils.py | 17 +++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/app.py b/app.py index 2d75b37..8b8a63c 100644 --- a/app.py +++ b/app.py @@ -555,11 +555,11 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, # Initialize variables sim_settings['event_start_year'] = 2024 sim_settings['event_start_month'] = 7 - sim_settings['days_in_month'] = 30 - sim_settings['event_start_day'] = 12 + sim_settings['event_start_day'] = 16 sim_settings['event_start_hour'] = 0 sim_settings['event_start_minute'] = 30 sim_settings['event_duration'] = 60 + sim_settings['days_in_month'] = 30 sim_settings['timestring'] = None sim_settings['number_of_radars'] = 1 sim_settings['radar_list'] = [] @@ -800,6 +800,11 @@ def query_radar_files(cfg, sim_settings): #sa.radar_files_dict.update(json.loads(json_data)) sim_settings['radar_files_dict'].update(json.loads(json_data)) + # Write radar metadata for this simulation to a text file. More complicated updating the + # dcc.Store object with this information since this function isn't a callback. + with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'w') as jsonfile: + json.dump(sim_settings['radar_files_dict'], jsonfile) + return results @@ -883,16 +888,18 @@ def run_with_cancel_button(cfg, sim_settings): if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return - ''' # Munger - args = [radar, str(sa.playback_start_str), str(sa.event_duration), - str(sa.simulation_seconds_shift), cfg['RADAR_DIR'], cfg['POLLING_DIR'], - cfg['L2MUNGER_FILEPATH'], cfg['DEBZ_FILEPATH'], new_radar] + args = [radar, str(sim_settings['playback_start_str']), + str(sim_settings['event_duration']), + str(sim_settings['simulation_seconds_shift']), cfg['RADAR_DIR'], + cfg['POLLING_DIR'],cfg['L2MUNGER_FILEPATH'], cfg['DEBZ_FILEPATH'], + new_radar] res = call_function(utils.exec_script, Path(cfg['MUNGER_SCRIPT_FILEPATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return - + + ''' # this gives the user some radar data to poll while other scripts are running try: UpdateDirList(new_radar, 'None', cfg['POLLING_DIR'], initialize=True) @@ -1041,7 +1048,7 @@ def monitor(_n, cfg, sim_settings): seen_scripts.append(name) # Radar file download status - radar_dl_completion, radar_files = utils.radar_monitor(sa) + radar_dl_completion, radar_files = utils.radar_monitor(cfg) # Radar mungering/transposing status munger_completion = utils.munger_monitor(sa, cfg) diff --git a/utils.py b/utils.py index 633c009..4667252 100644 --- a/utils.py +++ b/utils.py @@ -6,6 +6,7 @@ import psutil import pandas as pd import config +import json def exec_script(script_path, args, session_id): """ @@ -98,15 +99,19 @@ def calc_completion_percentage(expected_files, files_on_system): return percent_complete -def radar_monitor(sa): +def radar_monitor(cfg): """ - Reads in dictionary of radar files passed from Nexrad.py. Looks for associated - radar files on the system and compares to the total expected number and broadcasts a - percentage to the radar_status progress bar. + Reads in dictionary of radar files. Looks for associated radar files on the system + and compares to the total expected number and broadcasts a percentage to the + radar_status progress bar. """ - expected_files = list(sa.radar_files_dict.values()) + filename = f'{cfg['RADAR_DIR']}/radarinfo.json' + expected_files = [] + if os.path.exists(filename): + with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'r') as jsonfile: + expected_files = list(json.load(jsonfile).values()) + files_on_system = [x for x in expected_files if os.path.exists(x)] - percent_complete = calc_completion_percentage(expected_files, files_on_system) return percent_complete, files_on_system From 1bc2e9ca6f07b04c4851a3278acc69c486f0ce03 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sat, 24 Aug 2024 22:49:40 -0500 Subject: [PATCH 05/23] Minor tweaks. Through the UpdateDirList and radar section now. --- app.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/app.py b/app.py index 8b8a63c..7386143 100644 --- a/app.py +++ b/app.py @@ -797,29 +797,28 @@ def query_radar_files(cfg, sim_settings): json_data = results['stdout'].decode('utf-8') logging.info(f"{cfg['SESSION_ID']} :: Nexrad.py returned with {json_data}") - #sa.radar_files_dict.update(json.loads(json_data)) sim_settings['radar_files_dict'].update(json.loads(json_data)) # Write radar metadata for this simulation to a text file. More complicated updating the # dcc.Store object with this information since this function isn't a callback. - with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'w') as jsonfile: - json.dump(sim_settings['radar_files_dict'], jsonfile) + with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'w') as json_file: + json.dump(sim_settings['radar_files_dict'], json_file) return results - -def run_hodo_script(args) -> None: - """ - Runs the hodo script with the necessary arguments. - radar: str - the original radar, tells script where to find raw radar data - sa.new_radar: str - Either 'None' or the new radar to transpose to - asos_one: str - the first ASOS station to use for hodographs - asos_two: str - the second ASOS station to use for hodographs as a backup - sa.simulation_seconds_shift: str - time shift (seconds) between the event - start and playback start - """ - print(args) - subprocess.run(["python", config.HODO_SCRIPT_PATH] + args, check=True) +# !!! Not used? Can delete? !!! +#def run_hodo_script(args) -> None: +# """ +# Runs the hodo script with the necessary arguments. +# radar: str - the original radar, tells script where to find raw radar data +# sa.new_radar: str - Either 'None' or the new radar to transpose to +# asos_one: str - the first ASOS station to use for hodographs +# asos_two: str - the second ASOS station to use for hodographs as a backup +# sa.simulation_seconds_shift: str - time shift (seconds) between the event +# start and playback start +# """ +# print(args) +# subprocess.run(["python", config.HODO_SCRIPT_PATH] + args, check=True) def call_function(func, *args, **kwargs): @@ -864,12 +863,15 @@ def run_with_cancel_button(cfg, sim_settings): except Exception as e: logging.exception("Error creating radar dict or config file: ", exc_info=True) - # Create initial dictionary of expected radar files if len(sim_settings['radar_list']) > 0: + + # Create initial dictionary of expected radar files. + # TO DO: report back issues with radar downloads (e.g. 0 files found) res = call_function(query_radar_files, cfg, sim_settings) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return + # Radar downloading and mungering steps for _r, radar in enumerate(sim_settings['radar_list']): radar = radar.upper() try: @@ -899,14 +901,14 @@ def run_with_cancel_button(cfg, sim_settings): if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return - ''' # this gives the user some radar data to poll while other scripts are running try: UpdateDirList(new_radar, 'None', cfg['POLLING_DIR'], initialize=True) except Exception as e: print(f"Error with UpdateDirList ", e) - sa.log.exception(f"Error with UpdateDirList ", exc_info=True) + logging.exception(f"Error with UpdateDirList ", exc_info=True) + ''' # Surface observations args = [str(sa.lat), str(sa.lon), sa.event_start_str, cfg['PLACEFILES_DIR'], str(sa.event_duration)] From 5146ff113f238a28b2f60e28c53a94c999cbc2ac Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sat, 24 Aug 2024 22:50:27 -0500 Subject: [PATCH 06/23] . --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 7386143..7c400bc 100644 --- a/app.py +++ b/app.py @@ -986,7 +986,7 @@ def launch_simulation(n_clicks, configs, sim_settings) -> None: raise PreventUpdate else: if config.PLATFORM == 'WINDOWS': - sa.make_simulation_times() + make_simulation_times(sim_settings) else: run_with_cancel_button(configs, sim_settings) From 2df91fd0952e212b5bd7de73c34627b342c6ce45 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 11:12:09 -0500 Subject: [PATCH 07/23] Get transposing logic to work again. Working through nse placefiles step now. Munger and hodograph monitors inactive right now. --- app.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 144 insertions(+), 21 deletions(-) diff --git a/app.py b/app.py index 7c400bc..4c7a505 100644 --- a/app.py +++ b/app.py @@ -72,6 +72,121 @@ def create_logfile(LOG_DIR): datefmt="%Y-%m-%d %H:%M:%S" ) +def shift_placefiles(PLACEFILES_DIR, sa) -> None: + """ + # While the _shifted placefiles should be purged for each run, just ensure we're + # only querying the "original" placefiles to shift (exclude any with _shifted.txt) + """ + filenames = glob(f"{PLACEFILES_DIR}/*.txt") + filenames = [x for x in filenames if "shifted" not in x] + for file_ in filenames: + with open(file_, 'r', encoding='utf-8') as f: + data = f.readlines() + outfilename = f"{file_[0:file_.index('.txt')]}_shifted.txt" + outfile = open(outfilename, 'w', encoding='utf-8') + + for line in data: + new_line = line + + if sa['simulation_seconds_shift'] is not None and \ + any(x in line for x in ['Valid', 'TimeRange']): + new_line = shift_time(line, sa['simulation_seconds_shift']) + + # Shift this line in space. Only perform if both an original and transpose + # radar have been specified. + if sa['new_radar'] != 'None' and sa['radar'] is not None: + regex = re.findall(LAT_LON_REGEX, line) + if len(regex) > 0: + idx = regex[0].index(',') + plat, plon = float(regex[0][0:idx]), float(regex[0][idx+1:]) + lat_out, lon_out = move_point(plat, plon, sa['lat'], sa['lon'], + sa['new_lat'], sa['new_lon']) + new_line = line.replace(regex[0], f"{lat_out}, {lon_out}") + + outfile.write(new_line) + outfile.close() + +def shift_time(line: str, simulation_seconds_shift: int) -> str: + """ + Shifts the time-associated lines in a placefile. + These look for 'Valid' and 'TimeRange'. + """ + simulation_time_shift = timedelta(seconds=simulation_seconds_shift) + new_line = line + if 'Valid:' in line: + idx = line.find('Valid:') + # Leave off \n character + valid_timestring = line[idx+len('Valid:')+1:-1] + dt = datetime.strptime(valid_timestring, '%H:%MZ %a %b %d %Y') + new_validstring = datetime.strftime(dt + simulation_time_shift, + '%H:%MZ %a %b %d %Y') + new_line = line.replace(valid_timestring, new_validstring) + + if 'TimeRange' in line: + regex = re.findall(TIME_REGEX, line) + dt = datetime.strptime(regex[0], '%Y-%m-%dT%H:%M:%SZ') + new_datestring_1 = datetime.strftime(dt + simulation_time_shift, + '%Y-%m-%dT%H:%M:%SZ') + dt = datetime.strptime(regex[1], '%Y-%m-%dT%H:%M:%SZ') + new_datestring_2 = datetime.strftime(dt + simulation_time_shift, + '%Y-%m-%dT%H:%M:%SZ') + new_line = line.replace(f"{regex[0]} {regex[1]}", + f"{new_datestring_1} {new_datestring_2}") + return new_line + +def move_point(plat, plon, lat, lon, new_radar_lat, new_radar_lon): + """ + Shift placefiles to a different radar site. Maintains the original azimuth and range + from a specified RDA and applies it to a new radar location. + + Parameters: + ----------- + plat: float + Original placefile latitude + plon: float + Original palcefile longitude + + lat and lon is the lat/lon pair for the original radar + new_lat and new_lon is for the transposed radar. These values are set in + the transpose_radar function after a user makes a selection in the + new_radar_selection dropdown. + + """ + def _clamp(n, minimum, maximum): + """ + Helper function to make sure we're not taking the square root of a negative + number during the calculation of `c` below. + """ + return max(min(maximum, n), minimum) + + # Compute the initial distance from the original radar location + phi1, phi2 = math.radians(lat), math.radians(plat) + d_phi = math.radians(plat - lat) + d_lambda = math.radians(plon - lon) + + a = math.sin(d_phi/2)**2 + (math.cos(phi1) * + math.cos(phi2) * math.sin(d_lambda/2)**2) + a = _clamp(a, 0, a) + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + d = R * c + + # Compute the bearing + y = math.sin(d_lambda) * math.cos(phi2) + x = (math.cos(phi1) * math.sin(phi2)) - (math.sin(phi1) * math.cos(phi2) * + math.cos(d_lambda)) + theta = math.atan2(y, x) + bearing = (math.degrees(theta) + 360) % 360 + + # Apply this distance and bearing to the new radar location + phi_new, lambda_new = math.radians(new_radar_lat), math.radians(new_radar_lon) + phi_out = math.asin((math.sin(phi_new) * math.cos(d/R)) + (math.cos(phi_new) * + math.sin(d/R) * math.cos(math.radians(bearing)))) + lambda_out = lambda_new + math.atan2(math.sin(math.radians(bearing)) * + math.sin(d/R) * math.cos(phi_new), + math.cos(d/R) - math.sin(phi_new) * math.sin(phi_out)) + return math.degrees(phi_out), math.degrees(lambda_out) + + def copy_grlevel2_cfg_file(cfg) -> None: """ Ensures a grlevel2.cfg file is copied into the polling directory. @@ -566,7 +681,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, sim_settings['playback_dropdown_dict'] = {} sim_settings['radar_dict'] = {} sim_settings['radar_files_dict'] = {} - #sim_settings['radar'] = None + sim_settings['radar'] = None sim_settings['lat'] = None sim_settings['lon'] = None sim_settings['new_radar'] = 'None' @@ -699,6 +814,7 @@ def display_click_data(quant_str: str, click_data: dict, sim_settings: dict): #if triggered_id != 'radar_quantity': if add_to_list: sim_settings['radar_list'].append(radar) + sim_settings['radar'] = radar if len(sim_settings['radar_list']) > sim_settings['number_of_radars']: sim_settings['radar_list'] = sim_settings['radar_list'][-sim_settings['number_of_radars']:] if len(sim_settings['radar_list']) == sim_settings['number_of_radars']: @@ -732,8 +848,9 @@ def toggle_map_display(map_n, confirm_n) -> dict: Output('run_scripts_btn', 'disabled') ], Input('confirm_radars_btn', 'n_clicks'), Input('radar_quantity', 'value'), + State('sim_settings', 'data'), prevent_initial_call=True) -def finalize_radar_selections(clicks: int, _quant_str: str) -> dict: +def finalize_radar_selections(clicks: int, _quant_str: str, sim_settings: dict) -> dict: """ This will display the transpose section on the page if the user has selected a single radar. """ @@ -743,7 +860,7 @@ def finalize_radar_selections(clicks: int, _quant_str: str) -> dict: if triggered_id == 'radar_quantity': return disp_none, disp_none, disp_none, True if clicks > 0: - if sa.number_of_radars == 1 and len(sa.radar_list) == 1: + if sim_settings['number_of_radars'] == 1 and len(sim_settings['radar_list']) == 1: return lc.section_box_pad, disp_none, {'display': 'block'}, False return lc.section_box_pad, {'display': 'block'}, disp_none, False @@ -753,25 +870,30 @@ def finalize_radar_selections(clicks: int, _quant_str: str) -> dict: @app.callback( Output('tradar', 'data'), - Input('new_radar_selection', 'value')) -def transpose_radar(value): + Output('sim_settings', 'data', allow_duplicate=True), + Input('new_radar_selection', 'value'), + State('sim_settings', 'data'), + prevent_initial_call=True +) +def transpose_radar(value, sim_settings): """ If a user switches from a selection BACK to "None", without this, the application - will not update sa.new_radar to None. Instead, it'll be the previous selection. + will not update new_radar to None. Instead, it'll be the previous selection. Since we always evaluate "value" after every user selection, always set new_radar initially to None. Added tradar as a dcc.Store as this callback didn't seem to execute otherwise. The tradar store value is not used (currently), as everything is stored in sa.whatever. """ - sa.new_radar = 'None' + sim_settings['new_radar'] = 'None' if value != 'None': - sa.new_radar = value - sa.new_lat = lc.df[lc.df['radar'] == sa.new_radar]['lat'].values[0] - sa.new_lon = lc.df[lc.df['radar'] == sa.new_radar]['lon'].values[0] - return f'{sa.new_radar}' - return 'None' + new_radar = value + sim_settings['new_radar'] = new_radar + sim_settings['new_lat'] = lc.df[lc.df['radar'] == new_radar]['lat'].values[0] + sim_settings['new_lon'] = lc.df[lc.df['radar'] == new_radar]['lon'].values[0] + return f'{new_radar}', sim_settings + return 'None', sim_settings ################################################################################################ # ----------------------------- Run Scripts button -------------------------------------------- @@ -838,7 +960,7 @@ def call_function(func, *args, **kwargs): def run_with_cancel_button(cfg, sim_settings): """ This version of the script-launcher trying to work in cancel button - """ + """ UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) #sa.scripts_progress = 'Setting up files and times' @@ -908,9 +1030,9 @@ def run_with_cancel_button(cfg, sim_settings): print(f"Error with UpdateDirList ", e) logging.exception(f"Error with UpdateDirList ", exc_info=True) - ''' # Surface observations - args = [str(sa.lat), str(sa.lon), sa.event_start_str, cfg['PLACEFILES_DIR'], + args = [str(sim_settings['lat']), str(sim_settings['lon']), + sim_settings['event_start_str'], cfg['PLACEFILES_DIR'], str(sa.event_duration)] res = call_function(utils.exec_script, Path(cfg['OBS_SCRIPT_PATH']), args, cfg['SESSION_ID']) @@ -918,8 +1040,8 @@ def run_with_cancel_button(cfg, sim_settings): return # NSE placefiles - args = [str(sa.event_start_time), str(sa.event_duration), cfg['SCRIPTS_DIR'], - cfg['DATA_DIR'], cfg['PLACEFILES_DIR']] + args = [str(sim_settings['event_start_time']), str(sim_settings['event_duration']), + cfg['SCRIPTS_DIR'], cfg['DATA_DIR'], cfg['PLACEFILES_DIR']] res = call_function(utils.exec_script, Path(cfg['NSE_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: @@ -928,9 +1050,10 @@ def run_with_cancel_button(cfg, sim_settings): # Since there will always be a timeshift associated with a simulation, this # script needs to execute every time, even if a user doesn't select a radar # to transpose to. - sa.log.info(f"Entering function run_transpose_script") - run_transpose_script(cfg['PLACEFILES_DIR']) + logging.info(f"Entering function run_transpose_script") + run_transpose_script(cfg['PLACEFILES_DIR'], sim_settings) + ''' # Hodographs for radar, data in sa.radar_dict.items(): try: @@ -1078,11 +1201,11 @@ def monitor(_n, cfg, sim_settings): # whether to also perform a spatial shift occurrs within self.shift_placefiles where # a check for sa.new_radar != None takes place. -def run_transpose_script(PLACEFILES_DIR) -> None: +def run_transpose_script(PLACEFILES_DIR, sim_settings) -> None: """ Wrapper function to the shift_placefiles script """ - sa.shift_placefiles(PLACEFILES_DIR) + shift_placefiles(PLACEFILES_DIR, sim_settings) ################################################################################################ # ----------------------------- Toggle Placefiles Section -------------------------------------- From 3f348c6aa34ad5c2cae8c5786e8087f74f1d324d Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 11:29:09 -0500 Subject: [PATCH 08/23] Initial download/processing scripts and monitoring/status functions appear to be working. --- app.py | 24 ++++++++++++------------ utils.py | 24 ++++++++++++++---------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/app.py b/app.py index 4c7a505..7df0b5b 100644 --- a/app.py +++ b/app.py @@ -960,7 +960,7 @@ def call_function(func, *args, **kwargs): def run_with_cancel_button(cfg, sim_settings): """ This version of the script-launcher trying to work in cancel button - """ + """ UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) #sa.scripts_progress = 'Setting up files and times' @@ -1052,19 +1052,19 @@ def run_with_cancel_button(cfg, sim_settings): # to transpose to. logging.info(f"Entering function run_transpose_script") run_transpose_script(cfg['PLACEFILES_DIR'], sim_settings) - - ''' + # Hodographs - for radar, data in sa.radar_dict.items(): + for radar, data in sim_settings['radar_dict'].items(): try: asos_one = data['asos_one'] asos_two = data['asos_two'] except KeyError as e: - sa.log.exception("Error getting radar metadata: ", exc_info=True) + logging.exception("Error getting radar metadata: ", exc_info=True) # Execute hodograph script - args = [radar, sa.new_radar, asos_one, asos_two, str(sa.simulation_seconds_shift), - cfg['RADAR_DIR'], cfg['HODOGRAPHS_DIR']] + args = [radar, sim_settings['new_radar'], asos_one, asos_two, + str(sim_settings['simulation_seconds_shift']), cfg['RADAR_DIR'], + cfg['HODOGRAPHS_DIR']] res = call_function(utils.exec_script, Path(cfg['HODO_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: @@ -1075,7 +1075,7 @@ def run_with_cancel_button(cfg, sim_settings): except Exception as e: print("Error updating hodo html: ", e) sa.log.exception("Error updating hodo html: ", exc_info=True) - ''' + @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), [Input('run_scripts_btn', 'n_clicks'), @@ -1173,13 +1173,13 @@ def monitor(_n, cfg, sim_settings): seen_scripts.append(name) # Radar file download status - radar_dl_completion, radar_files = utils.radar_monitor(cfg) + radar_dl_completion, radar_files = utils.radar_monitor(cfg['RADAR_DIR']) # Radar mungering/transposing status - munger_completion = utils.munger_monitor(sa, cfg) + munger_completion = utils.munger_monitor(cfg['RADAR_DIR'], cfg['POLLING_DIR']) # Surface placefile status - placefile_stats = utils.surface_placefile_monitor(sa, cfg) + placefile_stats = utils.surface_placefile_monitor(cfg['PLACEFILES_DIR']) placefile_status_string = f"{placefile_stats[0]}/{placefile_stats[1]} files found" # Hodographs. Currently hard-coded to expect 2 files for every radar and radar file. @@ -1190,7 +1190,7 @@ def monitor(_n, cfg, sim_settings): (num_hodograph_images / (2*len(radar_files))) # NSE placefiles - model_list, model_warning = utils.nse_status_checker(sa, cfg) + model_list, model_warning = utils.nse_status_checker(cfg['MODEL_DIR']) return (radar_dl_completion, hodograph_completion, munger_completion, placefile_status_string, model_list, model_warning, screen_output) diff --git a/utils.py b/utils.py index 4667252..278ba03 100644 --- a/utils.py +++ b/utils.py @@ -99,47 +99,51 @@ def calc_completion_percentage(expected_files, files_on_system): return percent_complete -def radar_monitor(cfg): +def radar_monitor(RADAR_DIR): """ Reads in dictionary of radar files. Looks for associated radar files on the system and compares to the total expected number and broadcasts a percentage to the radar_status progress bar. """ - filename = f'{cfg['RADAR_DIR']}/radarinfo.json' + filename = f'{RADAR_DIR}/radarinfo.json' expected_files = [] if os.path.exists(filename): - with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'r') as jsonfile: + with open(f'{RADAR_DIR}/radarinfo.json', 'r') as jsonfile: expected_files = list(json.load(jsonfile).values()) files_on_system = [x for x in expected_files if os.path.exists(x)] percent_complete = calc_completion_percentage(expected_files, files_on_system) return percent_complete, files_on_system -def munger_monitor(sa, cfg): - expected_files = list(sa.radar_files_dict.values()) +def munger_monitor(RADAR_DIR, POLLING_DIR): + filename = f'{RADAR_DIR}/radarinfo.json' + expected_files = [] + if os.path.exists(filename): + with open(f'{RADAR_DIR}/radarinfo.json', 'r') as jsonfile: + expected_files = list(json.load(jsonfile).values()) # Are the mungered files always .gz? - files_on_system = glob(f"{cfg['POLLING_DIR']}/**/*.gz", recursive=True) + files_on_system = glob(f"{POLLING_DIR}/**/*.gz", recursive=True) percent_complete = calc_completion_percentage(expected_files, files_on_system) return percent_complete -def surface_placefile_monitor(_sa, cfg): +def surface_placefile_monitor(PLACEFILES_DIR): filenames = [ 'wind.txt', 'temp.txt', 'latest_surface_observations.txt', 'latest_surface_observations_lg.txt', 'latest_surface_observations_xlg.txt' ] - expected_files = [f"{cfg['PLACEFILES_DIR']}/{i}" for i in filenames] + expected_files = [f"{PLACEFILES_DIR}/{i}" for i in filenames] files_on_system = [x for x in expected_files if os.path.exists(x)] #percent_complete = calc_completion_percentage(expected_files, files_on_system) return len(files_on_system), len(expected_files) -def nse_status_checker(_sa, cfg): +def nse_status_checker(MODEL_DIR): """ Read in model status text file and query associated file sizes. """ - filename = f"{cfg['MODEL_DIR']}/model_list.txt" + filename = f"{MODEL_DIR}/model_list.txt" output = [] warning_text = "" if os.path.exists(filename): From f63d551ca1cdc37b5b154bed6dff260f4df73461 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 14:55:16 -0500 Subject: [PATCH 09/23] Turn off global RadarSimulator object and references to sa. throughout application. Getting simulation/clock controls working again, but not quite there...issues with passing updated clock info/datetimes in manage_clock_ function. --- app.py | 220 +++++++++++++++++++++---------------------------- scripts/nse.py | 2 +- utils.py | 9 +- 3 files changed, 103 insertions(+), 128 deletions(-) diff --git a/app.py b/app.py index 7df0b5b..e02f316 100644 --- a/app.py +++ b/app.py @@ -622,7 +622,7 @@ def remove_files_and_dirs(self, cfg) -> None: # ----------------------------- Initialize the app -------------------------------------------- ################################################################################################ -sa = RadarSimulator() +#sa = RadarSimulator() ################################################################################################ # ----------------------------- Build the layout --------------------------------------------- @@ -687,7 +687,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, sim_settings['new_radar'] = 'None' sim_settings['new_lat'] = None sim_settings['new_lon'] = None - sim_settings['scripts_progress'] = 'Scripts not started' + #'] = 'Scripts not started' #self.base_dir = Path.cwd() sim_settings['playback_initiated'] = False sim_settings['playback_speed'] = 1.0 @@ -883,7 +883,7 @@ def transpose_radar(value, sim_settings): initially to None. Added tradar as a dcc.Store as this callback didn't seem to execute otherwise. The - tradar store value is not used (currently), as everything is stored in sa.whatever. + tradar store value is not used (currently). """ sim_settings['new_radar'] = 'None' @@ -905,7 +905,7 @@ def query_radar_files(cfg, sim_settings): """ # Need to reset the expected files dictionary with each call. Otherwise, if a user # cancels a request, the previously-requested files will still be in the dictionary. - #sa.radar_files_dict = {} + # radar_files_dict = {} sim_settings['radar_files_dict'] = {} for _r, radar in enumerate(sim_settings['radar_list']): radar = radar.upper() @@ -963,24 +963,18 @@ def run_with_cancel_button(cfg, sim_settings): """ UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - #sa.scripts_progress = 'Setting up files and times' - sim_settings['scripts_progress'] = 'Setting up files and times' + #sim_settings['scripts_progress'] = 'Setting up files and times' # determine actual event time, playback time, diff of these two - #sa.make_simulation_times() - sim_settings = make_simulation_times(sim_settings) + #sim_settings = make_simulation_times(sim_settings) # clean out old files and directories try: remove_files_and_dirs(cfg) - #sa.remove_files_and_dirs(cfg) except Exception as e: logging.exception("Error removing files and directories: ", exc_info=True) # based on list of selected radars, create a dictionary of radar metadata try: - sa.create_radar_dict() create_radar_dict(sim_settings) - - sa.copy_grlevel2_cfg_file(cfg) copy_grlevel2_cfg_file(cfg) except Exception as e: logging.exception("Error creating radar dict or config file: ", exc_info=True) @@ -1033,14 +1027,14 @@ def run_with_cancel_button(cfg, sim_settings): # Surface observations args = [str(sim_settings['lat']), str(sim_settings['lon']), sim_settings['event_start_str'], cfg['PLACEFILES_DIR'], - str(sa.event_duration)] + str(sim_settings['event_duration'])] res = call_function(utils.exec_script, Path(cfg['OBS_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # NSE placefiles - args = [str(sim_settings['event_start_time']), str(sim_settings['event_duration']), + args = [str(sim_settings['event_start_str']), str(sim_settings['event_duration']), cfg['SCRIPTS_DIR'], cfg['DATA_DIR'], cfg['PLACEFILES_DIR']] res = call_function(utils.exec_script, Path(cfg['NSE_SCRIPT_PATH']), args, cfg['SESSION_ID']) @@ -1074,7 +1068,29 @@ def run_with_cancel_button(cfg, sim_settings): UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) except Exception as e: print("Error updating hodo html: ", e) - sa.log.exception("Error updating hodo html: ", exc_info=True) + logging.exception("Error updating hodo html: ", exc_info=True) + +@app.callback( + Output('sim_settings', 'data', allow_duplicate=True), + Input('run_scripts_btn', 'n_clicks'), + State('sim_settings', 'data'), + prevent_initial_call=True +) +def set_simulation_times(n_clicks, sim_settings): + """ + This setter callback will ensure the simulation control settings (playback times) + are broadcast to the sim_settings dictionary in dcc.Store. + """ + if n_clicks == 0: sim_settings['script_btn_clicks'] = 0 + + # User has clicked the run_scripts_btn. B/c of the allow_duplicate=True in output, + # this callback fires repeatedly with ctx.triggered_id = run_scripts_button. This is + # a workaround to determine when the scripts button has actually been clicked. + if n_clicks > sim_settings['script_btn_clicks']: + sim_settings = make_simulation_times(sim_settings) + sim_settings['script_btn_clicks'] = n_clicks + + return sim_settings @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), @@ -1100,7 +1116,7 @@ def run_with_cancel_button(cfg, sim_settings): (Output('change_time', 'disabled'), True, False), # wait to enable change time dropdown (Output('cancel_scripts', 'disabled'), False, True), ]) -def launch_simulation(n_clicks, configs, sim_settings) -> None: +def launch_simulation(n_clicks, configs, sim_settings): """ This function is called when the "Run Scripts" button is clicked. It will execute the necessary scripts to simulate radar operations, create hodographs, and transpose placefiles. @@ -1129,7 +1145,7 @@ def cancel_scripts(n_clicks, SESSION_ID) -> None: n_clicks (int): incremented whenever the "Cancel Scripts" button is clicked """ if n_clicks > 0: - utils.cancel_all(sa, SESSION_ID) + utils.cancel_all(SESSION_ID) @app.callback( @@ -1198,9 +1214,8 @@ def monitor(_n, cfg, sim_settings): # ----------------------------- Transpose placefiles in time and space ------------------------ ################################################################################################ # A time shift will always be applied in the case of a simulation. Determination of -# whether to also perform a spatial shift occurrs within self.shift_placefiles where -# a check for sa.new_radar != None takes place. - +# whether to also perform a spatial shift occurrs within shift_placefiles where a check for +# new_radar != None takes place. def run_transpose_script(PLACEFILES_DIR, sim_settings) -> None: """ Wrapper function to the shift_placefiles script @@ -1242,8 +1257,9 @@ def toggle_placefiles_section(n) -> dict: Output('change_time', 'options'), Input('playback_btn', 'n_clicks'), State('configs', 'data'), + State('sim_settings', 'data'), prevent_initial_call=True) -def initiate_playback(_nclick, cfg): +def initiate_playback(_nclick, cfg, sa): """ Enables/disables interval component that elapses the playback time @@ -1251,17 +1267,17 @@ def initiate_playback(_nclick, cfg): btn_text = 'Simulation Launched' btn_disabled = True playback_running = True - start = sa.playback_start_str - end = sa.playback_end_str + start = sa['playback_start_str'] + end = sa['playback_end_str'] style = lc.playback_times_style - options = sa.playback_dropdown_dict + options = sa['playback_dropdown_dict'] if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) + UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if sa['new_radar'] != 'None': + UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) + for _r, radar in enumerate(sa['radar_list']): + UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) return btn_text, btn_disabled, False, playback_running, start, style, end, style, options @@ -1276,44 +1292,55 @@ def initiate_playback(_nclick, cfg): Input('playback_timer', 'n_intervals'), Input('change_time', 'value'), Input('playback_running_store', 'data'), - State('configs', 'data') + State('configs', 'data'), + State('sim_settings', 'data') ], prevent_initial_call=True) -def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg): +def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, sa): """ Test """ interval_disabled = False status = 'Running' - sa.playback_paused = False + sa['playback_paused'] = False playback_btn_text = 'Pause Playback' - if sa.playback_clock.tzinfo is None: - sa.playback_clock = sa.playback_clock.replace(tzinfo=timezone.utc) - readout_time = datetime.strftime(sa.playback_clock, '%Y-%m-%d %H:%M:%S') + + # Variables stored in the sim_settings dict in the dcc.Store object are strings. + playback_clock = datetime.strptime(sa['playback_clock'], '%Y-%m-%dT%H:%M:%S+00:00') + playback_end = datetime.strptime(sa['playback_end'], '%Y-%m-%dT%H:%M:%S+00:00') + print(new_time) + + if playback_clock.tzinfo is None: + sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) + readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') style = lc.feedback_green triggered_id = ctx.triggered_id if triggered_id == 'playback_timer': - if sa.playback_clock.tzinfo is None: - sa.playback_clock = sa.playback_clock.replace(tzinfo=timezone.utc) - sa.playback_clock += timedelta(seconds=15 * sa.playback_speed) - if sa.playback_clock < sa.playback_end: - sa.playback_clock_str = sa.date_time_string(sa.playback_clock) - readout_time = datetime.strftime(sa.playback_clock, '%Y-%m-%d %H:%M:%S') + if playback_clock.tzinfo is None: + sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) + sa['playback_clock'] += timedelta(seconds=15 * sa['playback_speed']) + + if playback_end.tzinfo is None: + playback_end = playback_end.replace(tzinfo=timezone.utc) + + if playback_clock < playback_end: + sa['playback_clock_str'] = date_time_string(playback_clock) + readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) + UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if sa['new_radar'] != 'None': + UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) + for _r, radar in enumerate(sa['radar_list']): + UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) else: pass - if sa.playback_clock >= sa.playback_end: + if playback_clock >= playback_end: interval_disabled = True - sa.playback_paused = True - sa.playback_clock = sa.playback_end - sa.playback_clock_str = sa.date_time_string(sa.playback_clock) + sa['playback_paused'] = True + sa['playback_clock'] = playback_end + sa['playback_clock_str'] = date_time_string(playback_clock) status = 'Simulation Complete' playback_btn_text = 'Restart Simulation' style = lc.feedback_yellow @@ -1321,30 +1348,30 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg): if triggered_id == 'pause_resume_playback_btn': interval_disabled = False status = 'Running' - sa.playback_paused = False + sa['playback_paused'] = False playback_btn_text = 'Pause Playback' style = lc.feedback_green if nclicks % 2 == 1: interval_disabled = True status = 'Paused' - sa.playback_paused = True + sa['playback_paused'] = True playback_btn_text = 'Resume Playback' style = lc.feedback_yellow if triggered_id == 'change_time': - sa.playback_clock = datetime.strptime(new_time, '%Y-%m-%d %H:%M') - if sa.playback_clock.tzinfo is None: - sa.playback_clock = sa.playback_clock.replace(tzinfo=timezone.utc) - sa.playback_clock_str = new_time - readout_time = datetime.strftime(sa.playback_clock, '%Y-%m-%d %H:%M:%S') + playback_clock = datetime.strptime(new_time, '%Y-%m-%d %H:%M') + if playback_clock.tzinfo is None: + sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) + sa['playback_clock_str'] = new_time + readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa.playback_clock_str, cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa.new_radar != 'None': - UpdateDirList(sa.new_radar, sa.playback_clock_str, cfg['POLLING_DIR']) + UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if sa['new_radar'] != 'None': + UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa.radar_list): - UpdateDirList(radar, sa.playback_clock_str, cfg['POLLING_DIR']) + for _r, radar in enumerate(sa['radar_list']): + UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) if triggered_id == 'playback_running_store': pass @@ -1360,17 +1387,19 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg): ################################################################################################ @app.callback( Output('playback_speed_dummy', 'children'), - Input('speed_dropdown', 'value')) -def update_playback_speed(selected_speed): + Input('speed_dropdown', 'value'), + State('sim_settings', 'data'), +) +def update_playback_speed(selected_speed, sa): """ Updates the playback speed in the sa object """ - sa.playback_speed = selected_speed + sa['playback_speed'] = selected_speed try: - sa.playback_speed = float(selected_speed) + sa['playback_speed'] = float(selected_speed) except ValueError: print(f"Error converting {selected_speed} to float") - sa.playback_speed = 1.0 + sa['playback_speed'] = 1.0 return selected_speed @@ -1510,60 +1539,3 @@ def get_duration(duration, sim_settings) -> int: dev_tools_hot_reload=False) else: app.run(debug=True, port=8050, threaded=True, dev_tools_hot_reload=False) - - -''' -# pathname_params = dict() -# if my_settings.hosting_path is not None: -# pathname_params["routes_pathname_prefix"] = "/" -# pathname_params["requests_pathname_prefix"] = "/{}/".format(my_settings.hosting_path) - - -# Monitoring size of data and output directories for progress bar output -def directory_stats(folder): - """Return the size of a directory. If path hasn't been created yet, returns 0.""" - num_files = 0 - total_size = 0 - if os.path.isdir(folder): - total_size = sum( - sum( - os.path.getsize(os.path.join(walk_result[0], element)) - for element in walk_result[2] - ) - for walk_result in os.walk(folder) - ) - - for _, _, files in os.walk(folder): - num_files += len(files) - - return total_size/1024000, num_files -''' -''' -@app.callback( - #Output('tradar', 'value'), - Output('model_dir_size', 'data'), - Output('radar_dir_size', 'data'), - #Output('model_table_df', 'data'), - [Input('directory_monitor', 'n_intervals')], - prevent_initial_call=True) -def monitor(n): - model_dir = directory_stats(f"{sa.data_dir}/model_data") - radar_dir = directory_stats(f"{sa.data_dir}/radar") - print("sa.new_radar", sa.new_radar) - #print(model_dir) - - # Read modeldata.txt file - #filename = f"{sa.data_dir}/model_data/model_list.txt" - #model_table = [] - #if os.path.exists(filename): - # model_listing = [] - # with open(filename, 'r') as f: model_list = f.readlines() - # for line in model_list: - # model_listing.append(line.rsplit('/', 1)[1][:-1]) - # - # df = pd.DataFrame({'Model Data': model_listing}) - # output = df.to_dict('records') - - return model_dir[0], radar_dir[0] - -''' diff --git a/scripts/nse.py b/scripts/nse.py index 87065fd..734c19d 100644 --- a/scripts/nse.py +++ b/scripts/nse.py @@ -7,7 +7,7 @@ class Nse: def __init__(self, sim_start, event_duration, scripts_path, data_path, output_path): - self.sim_start = datetime.strptime(sim_start, '%Y-%m-%d %H:%M:%S+00:00') + self.sim_start = datetime.strptime(sim_start, '%Y-%m-%d %H:%M') self.sim_end = self.sim_start + timedelta(minutes=int(event_duration)) self.start_string = datetime.strftime(self.sim_start,"%Y-%m-%d/%H") self.end_string = datetime.strftime(self.sim_end,"%Y-%m-%d/%H") diff --git a/utils.py b/utils.py index 278ba03..21379ee 100644 --- a/utils.py +++ b/utils.py @@ -7,6 +7,9 @@ import pandas as pd import config import json +import logging +#from app import create_logfile +#create_logfile() def exec_script(script_path, args, session_id): """ @@ -58,7 +61,7 @@ def get_app_processes(): pass return processes -def cancel_all(sa, session_id): +def cancel_all(session_id): """ This function is invoked when the user clicks the Cancel button in the app. See app.cancel_scripts. @@ -84,11 +87,11 @@ def cancel_all(sa, session_id): if process['name'] == 'wgrib2': name = 'wgrib2' if name in config.scripts_list: - sa.log.info(f"Killing process: {name} with pid: {process['pid']}") + logging.info(f"Killing process: {name} with pid: {process['pid']}") os.kill(process['pid'], signal.SIGTERM) if len(process['cmdline']) >= 3 and 'multiprocessing' in process['cmdline'][2]: - sa.log.info(f"Killing spawned multi-process with pid: {process['pid']}") + logging.info(f"Killing spawned multi-process with pid: {process['pid']}") os.kill(process['pid'], signal.SIGTERM) From 1f12d419710c23ff234baadbdb9b14d24c3c2788 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 16:38:59 -0500 Subject: [PATCH 10/23] Getting simulation controls working again. Currently, neither the speed_dropdown nor change_time dropdowns update anything, but the rest of the logic seems to be working. --- app.py | 133 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 80 insertions(+), 53 deletions(-) diff --git a/app.py b/app.py index e02f316..0a8aa06 100644 --- a/app.py +++ b/app.py @@ -701,6 +701,8 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, sim_settings['simulation_running'] = False sim_settings['playback_paused'] = False + sim_settings['script_btn_clicks'] = 0 + # Settings for date dropdowns moved here to avoid specifying different values in # the layout now = datetime.now(pytz.utc) @@ -722,6 +724,9 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, dcc.Store(id='playback_start_store'), dcc.Store(id='playback_end_store'), dcc.Store(id='playback_clock_store'), + + dcc.Store(id='playback_speed_store', data=sim_settings['playback_speed']), + dcc.Store(id='playback_specs', data={}), lc.top_section, lc.top_banner, dbc.Container([ dbc.Container([ @@ -740,7 +745,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, lc.spacer,lc.toggle_placefiles_btn,lc.spacer_mini, lc.full_links_section, lc.spacer, simulation_playback_section, - html.Div(id='playback_speed_dummy', style={'display': 'none'}), + #html.Div(id='playback_speed_dummy', style={'display': 'none'}), lc.radar_id, lc.bottom_section ]) @@ -1081,8 +1086,6 @@ def set_simulation_times(n_clicks, sim_settings): This setter callback will ensure the simulation control settings (playback times) are broadcast to the sim_settings dictionary in dcc.Store. """ - if n_clicks == 0: sim_settings['script_btn_clicks'] = 0 - # User has clicked the run_scripts_btn. B/c of the allow_duplicate=True in output, # this callback fires repeatedly with ctx.triggered_id = run_scripts_button. This is # a workaround to determine when the scripts button has actually been clicked. @@ -1255,15 +1258,28 @@ def toggle_placefiles_section(n) -> dict: Output('end_readout', 'children'), Output('end_readout', 'style'), Output('change_time', 'options'), + Output('playback_specs', 'data', allow_duplicate=True), Input('playback_btn', 'n_clicks'), + State('playback_speed_store', 'data'), State('configs', 'data'), State('sim_settings', 'data'), prevent_initial_call=True) -def initiate_playback(_nclick, cfg, sa): +def initiate_playback(_nclick, playback_speed, cfg, sa): """ - Enables/disables interval component that elapses the playback time - + Enables/disables interval component that elapses the playback time. The user can only + click this button this once. """ + + playback_specs = { + 'playback_paused': False, + 'playback_clock': sa['playback_clock'], + 'playback_clock_str': sa['playback_clock_str'], + 'playback_end': sa['playback_end'], + 'playback_speed': playback_speed, + 'new_radar': sa['new_radar'], + 'radar_list': sa['radar_list'], + } + btn_text = 'Simulation Launched' btn_disabled = True playback_running = True @@ -1279,7 +1295,8 @@ def initiate_playback(_nclick, cfg, sa): for _r, radar in enumerate(sa['radar_list']): UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) - return btn_text, btn_disabled, False, playback_running, start, style, end, style, options + return (btn_text, btn_disabled, False, playback_running, start, style, end, style, options, + playback_specs) @app.callback( Output('playback_timer', 'disabled'), @@ -1288,59 +1305,65 @@ def initiate_playback(_nclick, cfg, sa): Output('pause_resume_playback_btn', 'children'), Output('current_readout', 'children'), Output('current_readout', 'style'), + Output('playback_specs', 'data', allow_duplicate=True), [Input('pause_resume_playback_btn', 'n_clicks'), Input('playback_timer', 'n_intervals'), Input('change_time', 'value'), Input('playback_running_store', 'data'), State('configs', 'data'), - State('sim_settings', 'data') + State('playback_specs', 'data'), ], prevent_initial_call=True) -def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, sa): +def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs): """ Test """ + print(specs) interval_disabled = False status = 'Running' - sa['playback_paused'] = False + specs['playback_paused'] = False playback_btn_text = 'Pause Playback' - # Variables stored in the sim_settings dict in the dcc.Store object are strings. - playback_clock = datetime.strptime(sa['playback_clock'], '%Y-%m-%dT%H:%M:%S+00:00') - playback_end = datetime.strptime(sa['playback_end'], '%Y-%m-%dT%H:%M:%S+00:00') - print(new_time) + # Variables stored dcc.Store object are strings. + specs['playback_clock'] = datetime.strptime(specs['playback_clock'], '%Y-%m-%dT%H:%M:%S+00:00') - if playback_clock.tzinfo is None: - sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) - readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') + # Unsure why these string representations change. + try: + specs['playback_end'] = datetime.strptime(specs['playback_end'], '%Y-%m-%dT%H:%M:%S+00:00') + except ValueError: + specs['playback_end'] = datetime.strptime(specs['playback_end'], '%Y-%m-%dT%H:%M:%S') + + if specs['playback_clock'].tzinfo is None: + specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) + readout_time = datetime.strftime(specs['playback_clock'], '%Y-%m-%d %H:%M:%S') style = lc.feedback_green triggered_id = ctx.triggered_id if triggered_id == 'playback_timer': - if playback_clock.tzinfo is None: - sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) - sa['playback_clock'] += timedelta(seconds=15 * sa['playback_speed']) + if specs['playback_clock'].tzinfo is None: + specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) + specs['playback_clock'] += timedelta(seconds=15 * specs['playback_speed']) - if playback_end.tzinfo is None: - playback_end = playback_end.replace(tzinfo=timezone.utc) + if specs['playback_end'].tzinfo is None: + specs['playback_end'] = specs['playback_end'].replace(tzinfo=timezone.utc) - if playback_clock < playback_end: - sa['playback_clock_str'] = date_time_string(playback_clock) - readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') + if specs['playback_clock'] < specs['playback_end']: + specs['playback_clock_str'] = date_time_string(specs['playback_clock']) + readout_time = datetime.strftime(specs['playback_clock'], '%Y-%m-%d %H:%M:%S') if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa['new_radar'] != 'None': - UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) + UpdateHodoHTML(specs['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if specs['new_radar'] != 'None': + UpdateDirList(specs['new_radar'], specs['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa['radar_list']): - UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) + for _r, radar in enumerate(specs['radar_list']): + UpdateDirList(radar, specs['playback_clock_str'], cfg['POLLING_DIR']) else: pass - if playback_clock >= playback_end: + if specs['playback_clock'] >= specs['playback_end']: interval_disabled = True - sa['playback_paused'] = True - sa['playback_clock'] = playback_end - sa['playback_clock_str'] = date_time_string(playback_clock) + specs['playback_paused'] = True + specs['playback_clock'] = specs['playback_end'] + specs['playback_clock_str'] = date_time_string(specs['playback_clock']) status = 'Simulation Complete' playback_btn_text = 'Restart Simulation' style = lc.feedback_yellow @@ -1348,30 +1371,30 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, sa): if triggered_id == 'pause_resume_playback_btn': interval_disabled = False status = 'Running' - sa['playback_paused'] = False + specs['playback_paused'] = False playback_btn_text = 'Pause Playback' style = lc.feedback_green if nclicks % 2 == 1: interval_disabled = True status = 'Paused' - sa['playback_paused'] = True + specs['playback_paused'] = True playback_btn_text = 'Resume Playback' style = lc.feedback_yellow if triggered_id == 'change_time': - playback_clock = datetime.strptime(new_time, '%Y-%m-%d %H:%M') - if playback_clock.tzinfo is None: - sa['playback_clock'] = playback_clock.replace(tzinfo=timezone.utc) - sa['playback_clock_str'] = new_time - readout_time = datetime.strftime(playback_clock, '%Y-%m-%d %H:%M:%S') + specs['playback_clock'] = datetime.strptime(new_time, '%Y-%m-%d %H:%M') + if specs['playback_clock'].tzinfo is None: + specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) + specs['playback_clock_str'] = new_time + readout_time = datetime.strftime(specs['playback_clock'], '%Y-%m-%d %H:%M:%S') if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa['new_radar'] != 'None': - UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) + UpdateHodoHTML(specs['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if specs['new_radar'] != 'None': + UpdateDirList(specs['new_radar'], specs['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa['radar_list']): - UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) + for _r, radar in enumerate(specs['radar_list']): + UpdateDirList(radar, specs['playback_clock_str'], cfg['POLLING_DIR']) if triggered_id == 'playback_running_store': pass @@ -1380,27 +1403,31 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, sa): # status = 'Paused' # playback_btn_text = 'Resume Simulation' - return interval_disabled, status, style, playback_btn_text, readout_time, style + return interval_disabled, status, style, playback_btn_text, readout_time, style, specs ################################################################################################ # ----------------------------- Playback Speed Callbacks -------------------------------------- ################################################################################################ @app.callback( - Output('playback_speed_dummy', 'children'), + #Output('playback_specs', 'data', allow_duplicate=True), + #Output('sim_settings', 'data', allow_duplicate=True), + Output('playback_speed_store', 'data'), Input('speed_dropdown', 'value'), - State('sim_settings', 'data'), + prevent_initial_call=True ) -def update_playback_speed(selected_speed, sa): +def update_playback_speed(selected_speed): """ Updates the playback speed in the sa object """ - sa['playback_speed'] = selected_speed + #sim_settings['playback_speed'] = selected_speed try: - sa['playback_speed'] = float(selected_speed) + #sim_settings['playback_speed'] = float(selected_speed) + selected_speed = float(selected_speed) except ValueError: print(f"Error converting {selected_speed} to float") - sa['playback_speed'] = 1.0 + selected_speed = 1.0 return selected_speed + #return specs ################################################################################################ From 044fcd7009e5d02fdd06ebea04b87add612d3ff8 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 17:58:28 -0500 Subject: [PATCH 11/23] Getting playback speed selections to work. Disabled dropdown until the launch simulation button clicked, otherwise an error is thrown since one of the dicts hasn't been defined yet. Address playback speed or time selection dropdown restarting a paused simulation --- app.py | 44 +++++++++++++++++++++++++++++++++----------- layout_components.py | 3 ++- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 0a8aa06..3a0d6e9 100644 --- a/app.py +++ b/app.py @@ -1074,7 +1074,7 @@ def run_with_cancel_button(cfg, sim_settings): except Exception as e: print("Error updating hodo html: ", e) logging.exception("Error updating hodo html: ", exc_info=True) - + @app.callback( Output('sim_settings', 'data', allow_duplicate=True), Input('run_scripts_btn', 'n_clicks'), @@ -1258,6 +1258,7 @@ def toggle_placefiles_section(n) -> dict: Output('end_readout', 'children'), Output('end_readout', 'style'), Output('change_time', 'options'), + Output('speed_dropdown', 'disabled'), Output('playback_specs', 'data', allow_duplicate=True), Input('playback_btn', 'n_clicks'), State('playback_speed_store', 'data'), @@ -1278,6 +1279,9 @@ def initiate_playback(_nclick, playback_speed, cfg, sa): 'playback_speed': playback_speed, 'new_radar': sa['new_radar'], 'radar_list': sa['radar_list'], + #'interval_disabled': False, + #'status': 'Running', + #'playback_btn_text': 'playback_btn_text' } btn_text = 'Simulation Launched' @@ -1296,7 +1300,7 @@ def initiate_playback(_nclick, playback_speed, cfg, sa): UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) return (btn_text, btn_disabled, False, playback_running, start, style, end, style, options, - playback_specs) + False, playback_specs) @app.callback( Output('playback_timer', 'disabled'), @@ -1310,17 +1314,21 @@ def initiate_playback(_nclick, playback_speed, cfg, sa): Input('playback_timer', 'n_intervals'), Input('change_time', 'value'), Input('playback_running_store', 'data'), + Input('playback_speed_store', 'data'), State('configs', 'data'), State('playback_specs', 'data'), ], prevent_initial_call=True) -def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs): +def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, playback_speed, + cfg, specs): """ Test """ - print(specs) + triggered_id = ctx.triggered_id + + specs['playback_speed'] = playback_speed interval_disabled = False status = 'Running' - specs['playback_paused'] = False + playback_paused = False playback_btn_text = 'Pause Playback' # Variables stored dcc.Store object are strings. @@ -1336,12 +1344,11 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) readout_time = datetime.strftime(specs['playback_clock'], '%Y-%m-%d %H:%M:%S') style = lc.feedback_green - triggered_id = ctx.triggered_id if triggered_id == 'playback_timer': if specs['playback_clock'].tzinfo is None: specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) - specs['playback_clock'] += timedelta(seconds=15 * specs['playback_speed']) + specs['playback_clock'] += timedelta(seconds=round(15*specs['playback_speed'])) if specs['playback_end'].tzinfo is None: specs['playback_end'] = specs['playback_end'].replace(tzinfo=timezone.utc) @@ -1361,7 +1368,7 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs if specs['playback_clock'] >= specs['playback_end']: interval_disabled = True - specs['playback_paused'] = True + playback_paused = True specs['playback_clock'] = specs['playback_end'] specs['playback_clock_str'] = date_time_string(specs['playback_clock']) status = 'Simulation Complete' @@ -1371,14 +1378,14 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs if triggered_id == 'pause_resume_playback_btn': interval_disabled = False status = 'Running' - specs['playback_paused'] = False + playback_paused = False playback_btn_text = 'Pause Playback' style = lc.feedback_green if nclicks % 2 == 1: interval_disabled = True status = 'Paused' - specs['playback_paused'] = True + playback_paused = True playback_btn_text = 'Resume Playback' style = lc.feedback_yellow @@ -1403,7 +1410,22 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, cfg, specs # status = 'Paused' # playback_btn_text = 'Resume Simulation' - return interval_disabled, status, style, playback_btn_text, readout_time, style, specs + # Without this, a change to either the playback speed or playback time will restart + # a paused simulation + if triggered_id in ['playback_speed_store', 'change_time']: + interval_disabled = specs['interval_disabled'] + status = specs['status'] + playback_btn_text = specs['playback_btn_text'] + playback_paused = specs['playback_paused'] + style = specs['style'] + + specs['interval_disabled'] = interval_disabled + specs['status'] = status + specs['playback_paused'] = playback_paused + specs['playback_btn_text'] = playback_btn_text + specs['style'] = style + return (specs['interval_disabled'], specs['status'], specs['style'], + specs['playback_btn_text'], readout_time, style, specs) ################################################################################################ # ----------------------------- Playback Speed Callbacks -------------------------------------- diff --git a/layout_components.py b/layout_components.py index 619e36e..f48dfe4 100644 --- a/layout_components.py +++ b/layout_components.py @@ -553,7 +553,8 @@ playback_speed_label = html.Div(children="Playback Speed", style=time_headers) playback_speed_dropdown_values = [0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 10.0] playback_speed_options = [{'label': str(i) + 'x', 'value': i} for i in playback_speed_dropdown_values] -playback_speed_dropdown = dcc.Dropdown(options=playback_speed_options, value=1.0, id='speed_dropdown') +playback_speed_dropdown = dcc.Dropdown(options=playback_speed_options, value=1.0, id='speed_dropdown', + disabled=True) playback_speed_col = dbc.Col(html.Div([playback_speed_label, spacer_mini, playback_speed_dropdown])) From e05ed245864ffd6de56ad4d6d75a6540e0df57db Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 18:05:45 -0500 Subject: [PATCH 12/23] Comment out RadarSimulator class and methods to make sure I caught everything. --- app.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/app.py b/app.py index 3a0d6e9..59507a5 100644 --- a/app.py +++ b/app.py @@ -286,6 +286,7 @@ def create_radar_dict(sa) -> dict: 'asos_one': asos_one, 'asos_two': asos_two, 'radar': radar.upper(), 'file_list': []} +''' ################################################################################################ # ----------------------------- Define class RadarSimulator ----------------------------------- ################################################################################################ @@ -623,7 +624,7 @@ def remove_files_and_dirs(self, cfg) -> None: ################################################################################################ #sa = RadarSimulator() - +''' ################################################################################################ # ----------------------------- Build the layout --------------------------------------------- ################################################################################################ @@ -933,20 +934,6 @@ def query_radar_files(cfg, sim_settings): return results -# !!! Not used? Can delete? !!! -#def run_hodo_script(args) -> None: -# """ -# Runs the hodo script with the necessary arguments. -# radar: str - the original radar, tells script where to find raw radar data -# sa.new_radar: str - Either 'None' or the new radar to transpose to -# asos_one: str - the first ASOS station to use for hodographs -# asos_two: str - the second ASOS station to use for hodographs as a backup -# sa.simulation_seconds_shift: str - time shift (seconds) between the event -# start and playback start -# """ -# print(args) -# subprocess.run(["python", config.HODO_SCRIPT_PATH] + args, check=True) - def call_function(func, *args, **kwargs): # For the main script calls @@ -1074,7 +1061,7 @@ def run_with_cancel_button(cfg, sim_settings): except Exception as e: print("Error updating hodo html: ", e) logging.exception("Error updating hodo html: ", exc_info=True) - + @app.callback( Output('sim_settings', 'data', allow_duplicate=True), Input('run_scripts_btn', 'n_clicks'), From a4f4eef6a35f367979420559dfaa9e894a5676aa Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 18:15:13 -0500 Subject: [PATCH 13/23] Small tweaks. --- app.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index 59507a5..9ff2876 100644 --- a/app.py +++ b/app.py @@ -55,11 +55,10 @@ # Regular expressions. First one finds lat/lon pairs, second finds the timestamps. LAT_LON_REGEX = "[0-9]{1,2}.[0-9]{1,100},[ ]{0,1}[|\\s-][0-9]{1,3}.[0-9]{1,100}" TIME_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z" -#TOKEN = 'INSERT YOUR MAPBOX TOKEN HERE' -# Configure logging """ -Idea is to move all of these functions to some other utility script within the main dir +Idea is to move all of these functions to some other utility file within the main dir +to get them out of the app. """ def create_logfile(LOG_DIR): """ @@ -688,8 +687,6 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, sim_settings['new_radar'] = 'None' sim_settings['new_lat'] = None sim_settings['new_lon'] = None - #'] = 'Scripts not started' - #self.base_dir = Path.cwd() sim_settings['playback_initiated'] = False sim_settings['playback_speed'] = 1.0 sim_settings['playback_start'] = 'Not Ready' @@ -732,11 +729,11 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, dbc.Container([ dbc.Container([ html.Div([html.Div([lc.step_select_time_section, lc.spacer, - dbc.Row([ - sim_year_section, sim_month_section, sim_day_selection, - sim_hour_section, sim_minute_section, sim_duration_section, - lc.spacer, lc.step_time_confirm])], style={'padding': '1em'}), - ], style=lc.section_box)]) + dbc.Row([ + sim_year_section, sim_month_section, sim_day_selection, + sim_hour_section, sim_minute_section, sim_duration_section, + lc.spacer, lc.step_time_confirm])], style={'padding': '1em'}), + ], style=lc.section_box)]) ]), lc.spacer, lc.full_radar_select_section, lc.spacer_mini, lc.map_section, @@ -746,7 +743,6 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, lc.spacer,lc.toggle_placefiles_btn,lc.spacer_mini, lc.full_links_section, lc.spacer, simulation_playback_section, - #html.Div(id='playback_speed_dummy', style={'display': 'none'}), lc.radar_id, lc.bottom_section ]) From fe9c273814bc72bb01cd683161a0aa860d2a8b20 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 19:38:09 -0500 Subject: [PATCH 14/23] minor changes --- app.py | 11 +++++------ config.py | 44 +++----------------------------------------- 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/app.py b/app.py index 9ff2876..16ffe56 100644 --- a/app.py +++ b/app.py @@ -1060,16 +1060,17 @@ def run_with_cancel_button(cfg, sim_settings): @app.callback( Output('sim_settings', 'data', allow_duplicate=True), - Input('run_scripts_btn', 'n_clicks'), + Input('confirm_radars_btn', 'n_clicks'), State('sim_settings', 'data'), prevent_initial_call=True ) def set_simulation_times(n_clicks, sim_settings): """ This setter callback will ensure the simulation control settings (playback times) - are broadcast to the sim_settings dictionary in dcc.Store. + are broadcast to the sim_settings dictionary in dcc.Store. Executed when user + finalizes radar selections """ - # User has clicked the run_scripts_btn. B/c of the allow_duplicate=True in output, + # User has clicked confirm_radars_btn. B/c of the allow_duplicate=True in output, # this callback fires repeatedly with ctx.triggered_id = run_scripts_button. This is # a workaround to determine when the scripts button has actually been clicked. if n_clicks > sim_settings['script_btn_clicks']: @@ -1262,9 +1263,7 @@ def initiate_playback(_nclick, playback_speed, cfg, sa): 'playback_speed': playback_speed, 'new_radar': sa['new_radar'], 'radar_list': sa['radar_list'], - #'interval_disabled': False, - #'status': 'Running', - #'playback_btn_text': 'playback_btn_text' + 'playback_start': sa['playback_start'], } btn_text = 'Simulation Launched' diff --git a/config.py b/config.py index b913018..aa1486d 100644 --- a/config.py +++ b/config.py @@ -59,7 +59,7 @@ def init_layout(): return dbc.Container([ # Elements used to store and track the session id dcc.Store(id='session_id', data=session_id, storage_type='session'), - dcc.Interval(id='broadcast_session_id', interval=1, n_intervals=0, max_intervals=1), + dcc.Interval(id='setup', interval=1, n_intervals=0, max_intervals=1), dcc.Store(id='configs', data={}), dcc.Store(id='sim_settings', data={}), @@ -73,10 +73,10 @@ def init_layout(): @app.callback( Output('configs', 'data'), - Input('broadcast_session_id', 'n_intervals'), + Input('setup', 'n_intervals'), State('session_id', 'data') ) -def broadcast_session_id(n_intervals, session_id): +def setup_paths_and_dirs(n_intervals, session_id): """ Callback executed once on page load to query the session id in dcc.Store component. Creates a dictionary of directory paths and stores in a separate dcc.Store component. @@ -127,44 +127,6 @@ def broadcast_session_id(n_intervals, session_id): return dirs -''' -ASSETS_DIR = BASE_DIR / 'assets' -PLACEFILES_DIR = ASSETS_DIR / 'placefiles' - -PLACEFILES_LINKS = f'{LINK_BASE}/placefiles' - -HODOGRAPHS_DIR = ASSETS_DIR / 'hodographs' -HODOGRAPHS_PAGE = ASSETS_DIR / 'hodographs.html' -HODO_HTML_PAGE = HODOGRAPHS_PAGE - -HODO_HTML_LINK = f'{LINK_BASE}/hodographs.html' -HODO_IMAGES = ASSETS_DIR / 'hodographs' -POLLING_DIR = ASSETS_DIR / 'polling' - - -DATA_DIR = BASE_DIR / 'data' -MODEL_DIR = DATA_DIR / 'model_data' -RADAR_DIR = DATA_DIR / 'radar' -LOG_DIR = DATA_DIR / 'logs' -CSV_PATH = BASE_DIR / 'radars.csv' -SCRIPTS_DIR = BASE_DIR / 'scripts' -OBS_SCRIPT_PATH = SCRIPTS_DIR / 'obs_placefile.py' -HODO_SCRIPT_PATH = SCRIPTS_DIR / 'hodo_plot.py' -NEXRAD_SCRIPT_PATH = SCRIPTS_DIR / 'Nexrad.py' -L2MUNGER_FILEPATH = SCRIPTS_DIR / 'l2munger' -MUNGER_SCRIPT_FILEPATH = SCRIPTS_DIR / 'munger.py' -MUNGE_DIR = SCRIPTS_DIR / 'munge' -nse_script_path = SCRIPTS_DIR / 'nse.py' -NSE_SCRIPT_PATH = SCRIPTS_DIR / 'nse.py' -DEBZ_FILEPATH = SCRIPTS_DIR / 'debz.py' -LOG_DIR = DATA_DIR / 'logs' - -os.makedirs(DATA_DIR, exist_ok=True) -os.makedirs(HODO_IMAGES, exist_ok=True) -os.makedirs(PLACEFILES_DIR, exist_ok=True) -''' -DATA_DIR = BASE_DIR / 'data' -LOG_DIR = DATA_DIR / 'logs' # Names (without extensions) of various pre-processing scripts. Needed for script # monitoring and/or cancelling. scripts_list = ["Nexrad", "munger", "obs_placefile", "nse", "wgrib2", From aac86ee254f06c931d69493708cc2d09dd86d303 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 22:57:10 -0500 Subject: [PATCH 15/23] Splitting up main stored dict into several dcc.Store objects to avoid passing large dictionaries back and forth. --- app.py | 810 +++++++++++++-------------------------------------------- 1 file changed, 178 insertions(+), 632 deletions(-) diff --git a/app.py b/app.py index 16ffe56..853ab18 100644 --- a/app.py +++ b/app.py @@ -71,7 +71,7 @@ def create_logfile(LOG_DIR): datefmt="%Y-%m-%d %H:%M:%S" ) -def shift_placefiles(PLACEFILES_DIR, sa) -> None: +def shift_placefiles(PLACEFILES_DIR, sim_times, radar_info) -> None: """ # While the _shifted placefiles should be purged for each run, just ensure we're # only querying the "original" placefiles to shift (exclude any with _shifted.txt) @@ -87,19 +87,20 @@ def shift_placefiles(PLACEFILES_DIR, sa) -> None: for line in data: new_line = line - if sa['simulation_seconds_shift'] is not None and \ + if sim_times['simulation_seconds_shift'] is not None and \ any(x in line for x in ['Valid', 'TimeRange']): - new_line = shift_time(line, sa['simulation_seconds_shift']) + new_line = shift_time(line, sim_times['simulation_seconds_shift']) # Shift this line in space. Only perform if both an original and transpose # radar have been specified. - if sa['new_radar'] != 'None' and sa['radar'] is not None: + if radar_info['new_radar'] != 'None' and radar_info['radar'] is not None: regex = re.findall(LAT_LON_REGEX, line) if len(regex) > 0: idx = regex[0].index(',') plat, plon = float(regex[0][0:idx]), float(regex[0][idx+1:]) - lat_out, lon_out = move_point(plat, plon, sa['lat'], sa['lon'], - sa['new_lat'], sa['new_lon']) + lat_out, lon_out = move_point(plat, plon, radar_info['lat'], + radar_info['lon'], radar_info['new_lat'], + radar_info['new_lon']) new_line = line.replace(regex[0], f"{lat_out}, {lon_out}") outfile.write(new_line) @@ -219,7 +220,7 @@ def date_time_string(dt) -> str: """ return datetime.strftime(dt, "%Y-%m-%d %H:%M") -def make_simulation_times(sa) -> dict: +def make_simulation_times(event_start_time, event_duration) -> dict: """ playback_start_time: datetime object - the time the simulation starts. @@ -238,39 +239,48 @@ def make_simulation_times(sa) -> dict: Variables ending with "_str" are the string representations of the datetime objects """ - sa['playback_start'] = datetime.now(pytz.utc) - timedelta(hours=2) - sa['playback_start'] = sa['playback_start'].replace(second=0, microsecond=0) - if sa['playback_start'].minute < 30: - sa['playback_start'] = sa['playback_start'].replace(minute=0) + playback_start = datetime.now(pytz.utc) - timedelta(hours=2) + playback_start = playback_start.replace(second=0, microsecond=0) + if playback_start.minute < 30: + playback_start = playback_start.replace(minute=0) else: - sa['playback_start'] = sa['playback_start'].replace(minute=30) + playback_start = playback_start.replace(minute=30) + playback_start_str = date_time_string(playback_start) - sa['playback_start_str'] = date_time_string(sa['playback_start']) + playback_end = playback_start + timedelta(minutes=int(event_duration)) + playback_end_str = date_time_string(playback_end) - sa['playback_end'] = sa['playback_start'] + timedelta(minutes=int(sa['event_duration'])) - sa['playback_end_str'] = date_time_string(sa['playback_end']) + playback_clock = playback_start + timedelta(seconds=600) + playback_clock_str = date_time_string(playback_clock) - sa['playback_clock'] = sa['playback_start'] + timedelta(seconds=600) - sa['playback_clock_str'] = date_time_string(sa['playback_clock']) - - sa['event_start_time'] = datetime(sa['event_start_year'], sa['event_start_month'], - sa['event_start_day'], sa['event_start_hour'], - sa['event_start_minute'], second=0, - tzinfo=timezone.utc) # a timedelta object is not JSON serializable, so cannot be included in the output # dictionary stored in the dcc.Store object. All references to simulation_time_shift # will need to use the simulation_seconds_shift reference instead. - simulation_time_shift = sa['playback_start'] - sa['event_start_time'] - sa['simulation_seconds_shift'] = round(simulation_time_shift.total_seconds()) - sa['event_start_str'] = date_time_string(sa['event_start_time']) + simulation_time_shift = playback_start - event_start_time + simulation_seconds_shift = round(simulation_time_shift.total_seconds()) + event_start_str = date_time_string(event_start_time) increment_list = [] - for t in range(0, int(sa['event_duration']/5) + 1 , 1): - new_time = sa['playback_start'] + timedelta(seconds=t*300) + for t in range(0, int(event_duration/5) + 1 , 1): + new_time = playback_start + timedelta(seconds=t*300) new_time_str = date_time_string(new_time) increment_list.append(new_time_str) - sa['playback_dropdown_dict'] = [{'label': increment, 'value': increment} for increment in increment_list] - return sa + playback_dropdown_dict = [{'label': increment, 'value': increment} for increment in increment_list] + + sim_times = { + 'event_start_str': event_start_str, + 'simulation_seconds_shift': simulation_seconds_shift, + 'playback_start_str': playback_start_str, + 'playback_start': playback_start, + 'playback_end_str': playback_end_str, + 'playback_end': playback_end, + 'playback_clock_str': playback_clock_str, + 'playback_clock': playback_clock, + 'playback_dropdown_dict': playback_dropdown_dict, + 'event_duration': event_duration + } + + return sim_times def create_radar_dict(sa) -> dict: """ @@ -285,345 +295,6 @@ def create_radar_dict(sa) -> dict: 'asos_one': asos_one, 'asos_two': asos_two, 'radar': radar.upper(), 'file_list': []} -''' -################################################################################################ -# ----------------------------- Define class RadarSimulator ----------------------------------- -################################################################################################ -class RadarSimulator(Config): - """ - A class to simulate radar operations, inheriting configurations from a base Config class. - - This simulator is designed to mimic the behavior of a radar system over a specified period, - starting from a predefined date and time. It allows for the simulation of radar data generation, - including the handling of time shifts and geographical coordinates. - - Attributes: - """ - - def __init__(self): - super().__init__() - self.event_start_year = 2023 - self.event_start_month = 6 - self.days_in_month = 30 - self.event_start_day = 7 - self.event_start_hour = 21 - self.event_start_minute = 45 - self.event_duration = 30 - self.timestring = None - self.number_of_radars = 1 - self.radar_list = [] - self.playback_dropdown_dict = {} - self.radar_dict = {} - self.radar_files_dict = {} - self.radar = None - self.lat = None - self.lon = None - self.new_radar = 'None' - self.new_lat = None - self.new_lon = None - self.scripts_progress = 'Scripts not started' - self.base_dir = Path.cwd() - self.playback_initiated = False - self.playback_speed = 1.0 - self.playback_start = 'Not Ready' - self.playback_end = 'Not Ready' - self.playback_start_str = 'Not Ready' - self.playback_end_str = 'Not Ready' - self.playback_current_time = 'Not Ready' - self.playback_clock = None - self.playback_clock_str = None - self.simulation_running = False - self.playback_paused = False - #self.make_simulation_times() - # This will generate a logfile. Something we'll want to turn on in the future. - self.log = self.create_logfile() - #UpdateHodoHTML('None', '', '') # set up the hodo page with no images - - def create_logfile(self): - """ - Creates an initial logfile. Stored in the data dir for now. Call is - sa.log.info or sa.log.error or sa.log.warning or sa.log.exception - """ - os.makedirs(config.LOG_DIR, exist_ok=True) - logging.basicConfig(filename=f"{config.LOG_DIR}/logfile.txt", - format='%(levelname)s %(asctime)s :: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S") - log = logging.getLogger() - log.setLevel(logging.DEBUG) - return log - - def create_radar_dict(self) -> None: - """ - Creates dictionary of radar sites and their metadata to be used in the simulation. - """ - for _i, radar in enumerate(self.radar_list): - self.lat = lc.df[lc.df['radar'] == radar]['lat'].values[0] - self.lon = lc.df[lc.df['radar'] == radar]['lon'].values[0] - asos_one = lc.df[lc.df['radar'] == radar]['asos_one'].values[0] - asos_two = lc.df[lc.df['radar'] == radar]['asos_two'].values[0] - self.radar_dict[radar.upper()] = {'lat': self.lat, 'lon': self.lon, - 'asos_one': asos_one, 'asos_two': asos_two, - 'radar': radar.upper(), 'file_list': []} - - def copy_grlevel2_cfg_file(self, cfg) -> None: - """ - Ensures a grlevel2.cfg file is copied into the polling directory. - This file is required for GR2Analyst to poll for radar data. - """ - source = f"{cfg['BASE_DIR']}/grlevel2.cfg" - destination = f"{cfg['POLLING_DIR']}/grlevel2.cfg" - try: - shutil.copyfile(source, destination) - except Exception as e: - print(f"Error copying {source} to {destination}: {e}") - - def date_time_string(self,dt) -> str: - """ - Converts a datetime object to a string. - """ - return datetime.strftime(dt, "%Y-%m-%d %H:%M") - - def date_time_object(self,dt_str) -> datetime: - """ - Converts a string to a timezone aware datetime object. - """ - dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M") - dt.replace(tzinfo=pytz.UTC) - return dt - - def timestamp_from_string(self,dt_str) -> float: - """ - Converts a string to a timestamp. - """ - return datetime.strptime(dt_str, "%Y-%m-%d %H:%M").timestamp() - - def make_simulation_times(self) -> None: - """ - playback_start_time: datetime object - - the time the simulation starts. - - set to (current UTC time rounded to nearest 30 minutes then minus 2hrs) - - This is "recent enough" for GR2Analyst to poll data - playback_timer: datetime object - - the "current" displaced realtime during the playback - event_start_time: datetime object - - the historical time the actual event started. - - based on user inputs of the event start time - simulation_time_shift: timedelta object - the difference between the playback start time and the event start time - simulation_seconds_shift: int - the difference between the playback start time and the event start time in seconds - - Variables ending with "_str" are the string representations of the datetime objects - """ - self.playback_start = datetime.now(pytz.utc) - timedelta(hours=2) - self.playback_start = self.playback_start.replace(second=0, microsecond=0) - if self.playback_start.minute < 30: - self.playback_start = self.playback_start.replace(minute=0) - else: - self.playback_start = self.playback_start.replace(minute=30) - - self.playback_start_str = self.date_time_string(self.playback_start) - - self.playback_end = self.playback_start + \ - timedelta(minutes=int(self.event_duration)) - self.playback_end_str = self.date_time_string(self.playback_end) - - self.playback_clock = self.playback_start + timedelta(seconds=600) - self.playback_clock_str = self.date_time_string(self.playback_clock) - - self.event_start_time = datetime(self.event_start_year, self.event_start_month, - self.event_start_day, self.event_start_hour, - self.event_start_minute, second=0, - tzinfo=timezone.utc) - self.simulation_time_shift = self.playback_start - self.event_start_time - self.simulation_seconds_shift = round( - self.simulation_time_shift.total_seconds()) - self.event_start_str = self.date_time_string(self.event_start_time) - increment_list = [] - for t in range(0, int(self.event_duration/5) + 1 , 1): - new_time = self.playback_start + timedelta(seconds=t*300) - new_time_str = self.date_time_string(new_time) - increment_list.append(new_time_str) - - self.playback_dropdown_dict = [{'label': increment, 'value': increment} for increment in increment_list] - - def change_playback_time(self,dseconds) -> str: - """ - This function is called by the playback_clock interval component. It updates the playback - time and checks if the simulation is complete. If so, it will stop the interval component. - """ - self.playback_clock += timedelta(seconds=dseconds * self.playback_speed) - if self.playback_start < self.playback_clock < self.playback_end: - self.playback_clock_str = self.date_time_string(self.playback_clock) - elif self.playback_clock >= self.playback_end: - self.playback_clock_str = self.playback_end_str - else: - self.playback_clock_str = self.playback_start_str - return self.playback_clock_str - - - #def get_days_in_month(self) -> None: - # """ - # Helper function to determine number of days to display in the dropdown - # """ - # self.days_in_month = calendar.monthrange( - # self.event_start_year, self.event_start_month)[1] - - def get_timestamp(self, file: str) -> float: - """ - - extracts datetime info from the radar filename - - returns a datetime timestamp (epoch seconds) object - """ - file_epoch_time = datetime.strptime( - file[4:19], '%Y%m%d_%H%M%S').timestamp() - return file_epoch_time - - def move_point(self, plat, plon): - # radar1_lat, radar1_lon, radar2_lat, radar2_lon, lat, lon - """ - Shift placefiles to a different radar site. Maintains the original azimuth and range - from a specified RDA and applies it to a new radar location. - - Parameters: - ----------- - plat: float - Original placefile latitude - plon: float - Original palcefile longitude - - self.lat and self.lon is the lat/lon pair for the original radar - self.new_lat and self.new_lon is for the transposed radar. These values are set in - the transpose_radar function after a user makes a selection in the new_radar_selection - dropdown. - - """ - def _clamp(n, minimum, maximum): - """ - Helper function to make sure we're not taking the square root of a negative - number during the calculation of `c` below. Same as numpy.clip(). - """ - return max(min(maximum, n), minimum) - - # Compute the initial distance from the original radar location - phi1, phi2 = math.radians(self.lat), math.radians(plat) - d_phi = math.radians(plat - self.lat) - d_lambda = math.radians(plon - self.lon) - - a = math.sin(d_phi/2)**2 + (math.cos(phi1) * - math.cos(phi2) * math.sin(d_lambda/2)**2) - a = _clamp(a, 0, a) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - d = R * c - - # Compute the bearing - y = math.sin(d_lambda) * math.cos(phi2) - x = (math.cos(phi1) * math.sin(phi2)) - (math.sin(phi1) * math.cos(phi2) * - math.cos(d_lambda)) - theta = math.atan2(y, x) - bearing = (math.degrees(theta) + 360) % 360 - - # Apply this distance and bearing to the new radar location - phi_new, lambda_new = math.radians( - self.new_lat), math.radians(self.new_lon) - phi_out = math.asin((math.sin(phi_new) * math.cos(d/R)) + (math.cos(phi_new) * - math.sin(d/R) * math.cos(math.radians(bearing)))) - lambda_out = lambda_new + math.atan2(math.sin(math.radians(bearing)) * - math.sin(d/R) * math.cos(phi_new), - math.cos(d/R) - math.sin(phi_new) * math.sin(phi_out)) - return math.degrees(phi_out), math.degrees(lambda_out) - - def shift_placefiles(self, PLACEFILES_DIR) -> None: - """ - # While the _shifted placefiles should be purged for each run, just ensure we're - # only querying the "original" placefiles to shift (exclude any with _shifted.txt) - """ - filenames = glob(f"{PLACEFILES_DIR}/*.txt") - filenames = [x for x in filenames if "shifted" not in x] - for file_ in filenames: - with open(file_, 'r', encoding='utf-8') as f: - data = f.readlines() - outfilename = f"{file_[0:file_.index('.txt')]}_shifted.txt" - outfile = open(outfilename, 'w', encoding='utf-8') - - for line in data: - new_line = line - - if self.simulation_time_shift is not None and any(x in line for x in ['Valid', 'TimeRange']): - new_line = self.shift_time(line) - - # Shift this line in space. Only perform if both an original and - # transposing radar have been specified. - if self.new_radar != 'None' and self.radar is not None: - regex = re.findall(LAT_LON_REGEX, line) - if len(regex) > 0: - idx = regex[0].index(',') - lat, lon = float(regex[0][0:idx]), float( - regex[0][idx+1:]) - lat_out, lon_out = self.move_point(lat, lon) - new_line = line.replace( - regex[0], f"{lat_out}, {lon_out}") - - outfile.write(new_line) - outfile.close() - - def shift_time(self, line: str) -> str: - """ - Shifts the time-associated lines in a placefile. - These look for 'Valid' and 'TimeRange'. - """ - new_line = line - if 'Valid:' in line: - idx = line.find('Valid:') - # Leave off \n character - valid_timestring = line[idx+len('Valid:')+1:-1] - dt = datetime.strptime(valid_timestring, '%H:%MZ %a %b %d %Y') - new_validstring = datetime.strftime(dt + self.simulation_time_shift, - '%H:%MZ %a %b %d %Y') - new_line = line.replace(valid_timestring, new_validstring) - - if 'TimeRange' in line: - regex = re.findall(TIME_REGEX, line) - dt = datetime.strptime(regex[0], '%Y-%m-%dT%H:%M:%SZ') - new_datestring_1 = datetime.strftime(dt + self.simulation_time_shift, - '%Y-%m-%dT%H:%M:%SZ') - dt = datetime.strptime(regex[1], '%Y-%m-%dT%H:%M:%SZ') - new_datestring_2 = datetime.strftime(dt + self.simulation_time_shift, - '%Y-%m-%dT%H:%M:%SZ') - new_line = line.replace(f"{regex[0]} {regex[1]}", - f"{new_datestring_1} {new_datestring_2}") - return new_line - - - def datetime_object_from_timestring(self, dt_str: str) -> datetime: - """ - - extracts datetime info from the radar filename - - converts it to a timezone aware datetime object in UTC - """ - file_time = datetime.strptime(dt_str, '%Y%m%d_%H%M%S') - utc_file_time = file_time.replace(tzinfo=pytz.UTC) - return utc_file_time - - def remove_files_and_dirs(self, cfg) -> None: - """ - Cleans up files and directories from the previous simulation so these datasets - are not included in the current simulation. - """ - dirs = [cfg['RADAR_DIR'], cfg['POLLING_DIR'], cfg['HODOGRAPHS_DIR'], cfg['MODEL_DIR'], - cfg['PLACEFILES_DIR']] - for directory in dirs: - for root, dirs, files in os.walk(directory, topdown=False): - for name in files: - if name != 'grlevel2.cfg': - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - -################################################################################################ -# ----------------------------- Initialize the app -------------------------------------------- -################################################################################################ - -#sa = RadarSimulator() -''' ################################################################################################ # ----------------------------- Build the layout --------------------------------------------- ################################################################################################ @@ -668,48 +339,38 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, if not layout_has_initialized['added']: # Initialize variables - sim_settings['event_start_year'] = 2024 - sim_settings['event_start_month'] = 7 - sim_settings['event_start_day'] = 16 - sim_settings['event_start_hour'] = 0 - sim_settings['event_start_minute'] = 30 - sim_settings['event_duration'] = 60 - sim_settings['days_in_month'] = 30 - sim_settings['timestring'] = None - sim_settings['number_of_radars'] = 1 - sim_settings['radar_list'] = [] - sim_settings['playback_dropdown_dict'] = {} - sim_settings['radar_dict'] = {} - sim_settings['radar_files_dict'] = {} - sim_settings['radar'] = None - sim_settings['lat'] = None - sim_settings['lon'] = None - sim_settings['new_radar'] = 'None' - sim_settings['new_lat'] = None - sim_settings['new_lon'] = None + event_start_year = 2024 + event_start_month = 7 + event_start_day = 16 + event_start_hour = 0 + event_start_minute = 30 + event_duration = 60 + + radar_info = { + 'number_of_radars': 1, + 'radar_list': [], + 'radar_dict': {}, + 'radar': None, + 'new_radar': 'None', + 'lat': None, + 'lon': None, + 'new_lat': None, + 'new_lon': None, + 'radar_files_dict': {} + } + sim_settings['playback_initiated'] = False - sim_settings['playback_speed'] = 1.0 - sim_settings['playback_start'] = 'Not Ready' - sim_settings['playback_end'] = 'Not Ready' - sim_settings['playback_start_str'] = 'Not Ready' - sim_settings['playback_end_str'] = 'Not Ready' - sim_settings['playback_current_time'] = 'Not Ready' - sim_settings['playback_clock'] = None - sim_settings['playback_clock_str'] = None - sim_settings['simulation_running'] = False - sim_settings['playback_paused'] = False - - sim_settings['script_btn_clicks'] = 0 + playback_speed = 1.0 # Settings for date dropdowns moved here to avoid specifying different values in # the layout now = datetime.now(pytz.utc) - sim_year_section = dbc.Col(html.Div([lc.step_year, dcc.Dropdown(np.arange(1992, now.year + 1), sim_settings['event_start_year'], id='start_year', clearable=False),])) - sim_month_section = dbc.Col(html.Div([lc.step_month, dcc.Dropdown(np.arange(1, 13), sim_settings['event_start_month'], id='start_month', clearable=False),])) - sim_day_selection = dbc.Col(html.Div([lc.step_day, dcc.Dropdown(np.arange(1, 31), sim_settings['event_start_day'], id='start_day', clearable=False)])) - sim_hour_section = dbc.Col(html.Div([lc.step_hour, dcc.Dropdown(np.arange(0, 24), sim_settings['event_start_hour'], id='start_hour', clearable=False),])) - sim_minute_section = dbc.Col(html.Div([lc.step_minute, dcc.Dropdown([0, 15, 30, 45], sim_settings['event_start_minute'], id='start_minute', clearable=False),])) - sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown(np.arange(0, 240, 15), sim_settings['event_duration'], id='duration', clearable=False),])) + sim_year_section = dbc.Col(html.Div([lc.step_year, dcc.Dropdown(np.arange(1992, now.year + 1), event_start_year, id='start_year', clearable=False),])) + sim_month_section = dbc.Col(html.Div([lc.step_month, dcc.Dropdown(np.arange(1, 13), event_start_month, id='start_month', clearable=False),])) + sim_day_selection = dbc.Col(html.Div([lc.step_day, dcc.Dropdown(np.arange(1, 31), event_start_day, id='start_day', clearable=False)])) + sim_hour_section = dbc.Col(html.Div([lc.step_hour, dcc.Dropdown(np.arange(0, 24), event_start_hour, id='start_hour', clearable=False),])) + sim_minute_section = dbc.Col(html.Div([lc.step_minute, dcc.Dropdown([0, 15, 30, 45], event_start_minute, id='start_minute', clearable=False),])) + sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown(np.arange(0, 240, 15), event_duration, id='duration', clearable=False),])) if children is None: children = [] @@ -723,8 +384,10 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, dcc.Store(id='playback_end_store'), dcc.Store(id='playback_clock_store'), - dcc.Store(id='playback_speed_store', data=sim_settings['playback_speed']), - dcc.Store(id='playback_specs', data={}), + dcc.Store(id='radar_info', data=radar_info), + dcc.Store(id='sim_times'), + dcc.Store(id='playback_speed_store', data=playback_speed), + dcc.Store(id='playback_specs'), lc.top_section, lc.top_banner, dbc.Container([ dbc.Container([ @@ -765,67 +428,47 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, ################################################################################################ # ----------------------------- Radar map section --------------------------------------------- ################################################################################################ - @app.callback( Output('show_radar_selection_feedback', 'children'), Output('confirm_radars_btn', 'children'), Output('confirm_radars_btn', 'disabled'), - Output('sim_settings', 'data', allow_duplicate=True), - Input('radar_quantity', 'value'), + Output('radar_info', 'data'), + [Input('radar_quantity', 'value'), Input('graph', 'clickData'), - State('sim_settings', 'data'), + State('radar_info', 'data')], prevent_initial_call=True ) -def display_click_data(quant_str: str, click_data: dict, sim_settings: dict): +def display_click_data(quant_str: str, click_data: dict, radar_info: dict): """ Any time a radar site is clicked, this function will trigger and update the radar list. - - The allow_duplicate=True addition seems to cause this callback to fire repeatedly - event when no user input has triggered. Necessitated some workarounds. """ # initially have to make radar selections and can't finalize select_action = 'Make' btn_deactivated = True - #triggered_id = ctx.triggered_id - sim_settings['number_of_radars'] = int(quant_str[0:1]) - - # This block was getting triggered repeatedly when adding allow_duplicate=True. - #if triggered_id == 'radar_quantity' and len(sim_settings['radar_list']) != 0: - # sa.number_of_radars = int(quant_str[0:1]) - # sa.radar_list = [] - # sa.radar_dict = {} - - # sim_settings['number_of_radars'] = int(quant_str[0:1]) - # sim_settings['radar_list'] = [] - # sim_settings['radar_dict'] = {} - - # return f'Use map to select {quant_str}', f'{select_action} selections', True, sim_settings - add_to_list = False + triggered_id = ctx.triggered_id + radar_info['number_of_radars'] = int(quant_str[0:1]) + if triggered_id == 'radar_quantity': + radar_info['number_of_radars'] = int(quant_str[0:1]) + radar_info['radar_list'] = [] + radar_info['radar_dict'] = {} + return f'Use map to select {quant_str}', f'{select_action} selections', True, radar_info try: - radar = click_data['points'][0]['customdata'] - if len(sim_settings['radar_list']) > 0: - if radar != sim_settings['radar_list'][-1] and radar not in sim_settings['radar_list']: - add_to_list = True - else: - add_to_list = True + radar_info['radar'] = click_data['points'][0]['customdata'] except (KeyError, IndexError, TypeError): - return 'No radar selected ...', f'{select_action} selections', True, sim_settings - - #if triggered_id != 'radar_quantity': - if add_to_list: - sim_settings['radar_list'].append(radar) - sim_settings['radar'] = radar - if len(sim_settings['radar_list']) > sim_settings['number_of_radars']: - sim_settings['radar_list'] = sim_settings['radar_list'][-sim_settings['number_of_radars']:] - if len(sim_settings['radar_list']) == sim_settings['number_of_radars']: + return 'No radar selected ...', f'{select_action} selections', True, radar_info + + if radar_info['radar'] not in radar_info['radar_list']: + radar_info['radar_list'].append(radar_info['radar']) + if len(radar_info['radar_list']) > radar_info['number_of_radars']: + radar_info['radar_list'] = radar_info['radar_list'][-radar_info['number_of_radars']:] + if len(radar_info['radar_list']) == radar_info['number_of_radars']: select_action = 'Finalize' btn_deactivated = False - #print(f"Radar list: {sim_settings['radar_list']}") - listed_radars = ', '.join(sim_settings['radar_list']) - return listed_radars, f'{select_action} selections', btn_deactivated, sim_settings + listed_radars = ', '.join(radar_info['radar_list']) + return listed_radars, f'{select_action} selections', btn_deactivated, radar_info @app.callback( @@ -850,9 +493,9 @@ def toggle_map_display(map_n, confirm_n) -> dict: Output('run_scripts_btn', 'disabled') ], Input('confirm_radars_btn', 'n_clicks'), Input('radar_quantity', 'value'), - State('sim_settings', 'data'), + State('radar_info', 'data'), prevent_initial_call=True) -def finalize_radar_selections(clicks: int, _quant_str: str, sim_settings: dict) -> dict: +def finalize_radar_selections(clicks: int, _quant_str: str, radar_info: dict) -> dict: """ This will display the transpose section on the page if the user has selected a single radar. """ @@ -862,7 +505,7 @@ def finalize_radar_selections(clicks: int, _quant_str: str, sim_settings: dict) if triggered_id == 'radar_quantity': return disp_none, disp_none, disp_none, True if clicks > 0: - if sim_settings['number_of_radars'] == 1 and len(sim_settings['radar_list']) == 1: + if radar_info['number_of_radars'] == 1 and len(radar_info['radar_list']) == 1: return lc.section_box_pad, disp_none, {'display': 'block'}, False return lc.section_box_pad, {'display': 'block'}, disp_none, False @@ -872,12 +515,12 @@ def finalize_radar_selections(clicks: int, _quant_str: str, sim_settings: dict) @app.callback( Output('tradar', 'data'), - Output('sim_settings', 'data', allow_duplicate=True), + Output('radar_info', 'data', allow_duplicate=True), Input('new_radar_selection', 'value'), - State('sim_settings', 'data'), + State('radar_info', 'data'), prevent_initial_call=True ) -def transpose_radar(value, sim_settings): +def transpose_radar(value, radar_info): """ If a user switches from a selection BACK to "None", without this, the application will not update new_radar to None. Instead, it'll be the previous selection. @@ -887,33 +530,33 @@ def transpose_radar(value, sim_settings): Added tradar as a dcc.Store as this callback didn't seem to execute otherwise. The tradar store value is not used (currently). """ - sim_settings['new_radar'] = 'None' + radar_info['new_radar'] = 'None' if value != 'None': new_radar = value - sim_settings['new_radar'] = new_radar - sim_settings['new_lat'] = lc.df[lc.df['radar'] == new_radar]['lat'].values[0] - sim_settings['new_lon'] = lc.df[lc.df['radar'] == new_radar]['lon'].values[0] - return f'{new_radar}', sim_settings - return 'None', sim_settings + radar_info['new_radar'] = new_radar + radar_info['new_lat'] = lc.df[lc.df['radar'] == new_radar]['lat'].values[0] + radar_info['new_lon'] = lc.df[lc.df['radar'] == new_radar]['lon'].values[0] + return f'{new_radar}', radar_info + return 'None', radar_info ################################################################################################ # ----------------------------- Run Scripts button -------------------------------------------- ################################################################################################ -def query_radar_files(cfg, sim_settings): +def query_radar_files(cfg, radar_info, sim_times): """ Get the radar files from the AWS bucket. This is a preliminary step to build the progess bar. """ # Need to reset the expected files dictionary with each call. Otherwise, if a user # cancels a request, the previously-requested files will still be in the dictionary. # radar_files_dict = {} - sim_settings['radar_files_dict'] = {} - for _r, radar in enumerate(sim_settings['radar_list']): + radar_info['radar_files_dict'] = {} + for _r, radar in enumerate(radar_info['radar_list']): radar = radar.upper() - args = [radar, str(sim_settings['event_start_str']), str(sim_settings['event_duration']), + args = [radar, str(sim_times['event_start_str']), str(sim_times['event_duration']), str(False), cfg['RADAR_DIR']] - logging.info(f"{cfg['SESSION_ID']} :: Passing {args} to Nexrad.py") + #logging.info(f"{cfg['SESSION_ID']} :: Passing {args} to Nexrad.py") results = utils.exec_script(Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if results['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: logging.warning(f"{cfg['SESSION_ID']} :: User cancelled query_radar_files()") @@ -921,19 +564,19 @@ def query_radar_files(cfg, sim_settings): json_data = results['stdout'].decode('utf-8') logging.info(f"{cfg['SESSION_ID']} :: Nexrad.py returned with {json_data}") - sim_settings['radar_files_dict'].update(json.loads(json_data)) + radar_info['radar_files_dict'].update(json.loads(json_data)) # Write radar metadata for this simulation to a text file. More complicated updating the # dcc.Store object with this information since this function isn't a callback. with open(f'{cfg['RADAR_DIR']}/radarinfo.json', 'w') as json_file: - json.dump(sim_settings['radar_files_dict'], json_file) + json.dump(radar_info['radar_files_dict'], json_file) return results def call_function(func, *args, **kwargs): # For the main script calls - if len(args) > 2: + if len(args) > 2 and func.__name__ != 'query_radar_files': logging.info(f"Sending {args[1]} to {args[0]}") result = func(*args, **kwargs) @@ -945,15 +588,22 @@ def call_function(func, *args, **kwargs): return result -def run_with_cancel_button(cfg, sim_settings): +def run_with_cancel_button(cfg, sim_times, radar_info): """ This version of the script-launcher trying to work in cancel button """ + log_string = ( + f"\n" + f"=========================Simulation Settings========================\n" + f"Session ID: {cfg['SESSION_ID']}\n" + f"{sim_times}\n" + f"{radar_info}\n" + f"====================================================================\n" + ) + logging.info(log_string) + UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - #sim_settings['scripts_progress'] = 'Setting up files and times' - # determine actual event time, playback time, diff of these two - #sim_settings = make_simulation_times(sim_settings) # clean out old files and directories try: remove_files_and_dirs(cfg) @@ -962,42 +612,42 @@ def run_with_cancel_button(cfg, sim_settings): # based on list of selected radars, create a dictionary of radar metadata try: - create_radar_dict(sim_settings) + create_radar_dict(radar_info) copy_grlevel2_cfg_file(cfg) except Exception as e: logging.exception("Error creating radar dict or config file: ", exc_info=True) - if len(sim_settings['radar_list']) > 0: + if len(radar_info['radar_list']) > 0: # Create initial dictionary of expected radar files. # TO DO: report back issues with radar downloads (e.g. 0 files found) - res = call_function(query_radar_files, cfg, sim_settings) + res = call_function(query_radar_files, cfg, radar_info, sim_times) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # Radar downloading and mungering steps - for _r, radar in enumerate(sim_settings['radar_list']): + for _r, radar in enumerate(radar_info['radar_list']): radar = radar.upper() try: - if sim_settings['new_radar'] == 'None': + if radar_info['new_radar'] == 'None': new_radar = radar else: - new_radar = sim_settings['new_radar'].upper() + new_radar = radar_info['new_radar'].upper() except Exception as e: logging.exception("Error defining new radar: ", exc_info=True) # Radar download - args = [radar, str(sim_settings['event_start_str']), - str(sim_settings['event_duration']), str(True), cfg['RADAR_DIR']] + args = [radar, str(sim_times['event_start_str']), + str(sim_times['event_duration']), str(True), cfg['RADAR_DIR']] res = call_function(utils.exec_script, Path(cfg['NEXRAD_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # Munger - args = [radar, str(sim_settings['playback_start_str']), - str(sim_settings['event_duration']), - str(sim_settings['simulation_seconds_shift']), cfg['RADAR_DIR'], + args = [radar, str(sim_times['playback_start_str']), + str(sim_times['event_duration']), + str(sim_times['simulation_seconds_shift']), cfg['RADAR_DIR'], cfg['POLLING_DIR'],cfg['L2MUNGER_FILEPATH'], cfg['DEBZ_FILEPATH'], new_radar] res = call_function(utils.exec_script, Path(cfg['MUNGER_SCRIPT_FILEPATH']), @@ -1013,16 +663,16 @@ def run_with_cancel_button(cfg, sim_settings): logging.exception(f"Error with UpdateDirList ", exc_info=True) # Surface observations - args = [str(sim_settings['lat']), str(sim_settings['lon']), - sim_settings['event_start_str'], cfg['PLACEFILES_DIR'], - str(sim_settings['event_duration'])] + args = [str(radar_info['lat']), str(radar_info['lon']), + sim_times['event_start_str'], cfg['PLACEFILES_DIR'], + str(sim_times['event_duration'])] res = call_function(utils.exec_script, Path(cfg['OBS_SCRIPT_PATH']), args, cfg['SESSION_ID']) if res['returncode'] in [signal.SIGTERM, -1*signal.SIGTERM]: return # NSE placefiles - args = [str(sim_settings['event_start_str']), str(sim_settings['event_duration']), + args = [str(sim_times['event_start_str']), str(sim_times['event_duration']), cfg['SCRIPTS_DIR'], cfg['DATA_DIR'], cfg['PLACEFILES_DIR']] res = call_function(utils.exec_script, Path(cfg['NSE_SCRIPT_PATH']), args, cfg['SESSION_ID']) @@ -1033,10 +683,10 @@ def run_with_cancel_button(cfg, sim_settings): # script needs to execute every time, even if a user doesn't select a radar # to transpose to. logging.info(f"Entering function run_transpose_script") - run_transpose_script(cfg['PLACEFILES_DIR'], sim_settings) + run_transpose_script(cfg['PLACEFILES_DIR'], sim_times, radar_info) # Hodographs - for radar, data in sim_settings['radar_dict'].items(): + for radar, data in radar_info['radar_dict'].items(): try: asos_one = data['asos_one'] asos_two = data['asos_two'] @@ -1044,8 +694,8 @@ def run_with_cancel_button(cfg, sim_settings): logging.exception("Error getting radar metadata: ", exc_info=True) # Execute hodograph script - args = [radar, sim_settings['new_radar'], asos_one, asos_two, - str(sim_settings['simulation_seconds_shift']), cfg['RADAR_DIR'], + args = [radar, radar_info['new_radar'], asos_one, asos_two, + str(sim_times['simulation_seconds_shift']), cfg['RADAR_DIR'], cfg['HODOGRAPHS_DIR']] res = call_function(utils.exec_script, Path(cfg['HODO_SCRIPT_PATH']), args, cfg['SESSION_ID']) @@ -1058,32 +708,13 @@ def run_with_cancel_button(cfg, sim_settings): print("Error updating hodo html: ", e) logging.exception("Error updating hodo html: ", exc_info=True) -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('confirm_radars_btn', 'n_clicks'), - State('sim_settings', 'data'), - prevent_initial_call=True -) -def set_simulation_times(n_clicks, sim_settings): - """ - This setter callback will ensure the simulation control settings (playback times) - are broadcast to the sim_settings dictionary in dcc.Store. Executed when user - finalizes radar selections - """ - # User has clicked confirm_radars_btn. B/c of the allow_duplicate=True in output, - # this callback fires repeatedly with ctx.triggered_id = run_scripts_button. This is - # a workaround to determine when the scripts button has actually been clicked. - if n_clicks > sim_settings['script_btn_clicks']: - sim_settings = make_simulation_times(sim_settings) - sim_settings['script_btn_clicks'] = n_clicks - - return sim_settings @app.callback( Output('show_script_progress', 'children', allow_duplicate=True), [Input('run_scripts_btn', 'n_clicks'), State('configs', 'data'), - State('sim_settings', 'data')], + State('sim_times', 'data'), + State('radar_info', 'data')], prevent_initial_call=True, running=[ (Output('start_year', 'disabled'), True, False), @@ -1103,7 +734,7 @@ def set_simulation_times(n_clicks, sim_settings): (Output('change_time', 'disabled'), True, False), # wait to enable change time dropdown (Output('cancel_scripts', 'disabled'), False, True), ]) -def launch_simulation(n_clicks, configs, sim_settings): +def launch_simulation(n_clicks, configs, sim_times, radar_info): """ This function is called when the "Run Scripts" button is clicked. It will execute the necessary scripts to simulate radar operations, create hodographs, and transpose placefiles. @@ -1111,10 +742,8 @@ def launch_simulation(n_clicks, configs, sim_settings): if n_clicks == 0: raise PreventUpdate else: - if config.PLATFORM == 'WINDOWS': - make_simulation_times(sim_settings) - else: - run_with_cancel_button(configs, sim_settings) + if config.PLATFORM != 'WINDOWS': + run_with_cancel_button(configs, sim_times, radar_info) ################################################################################################ # ----------------------------- Monitoring and reporting script status ------------------------ @@ -1203,11 +832,11 @@ def monitor(_n, cfg, sim_settings): # A time shift will always be applied in the case of a simulation. Determination of # whether to also perform a spatial shift occurrs within shift_placefiles where a check for # new_radar != None takes place. -def run_transpose_script(PLACEFILES_DIR, sim_settings) -> None: +def run_transpose_script(PLACEFILES_DIR, sim_times, radar_info) -> None: """ Wrapper function to the shift_placefiles script """ - shift_placefiles(PLACEFILES_DIR, sim_settings) + shift_placefiles(PLACEFILES_DIR, sim_times, radar_info) ################################################################################################ # ----------------------------- Toggle Placefiles Section -------------------------------------- @@ -1244,12 +873,13 @@ def toggle_placefiles_section(n) -> dict: Output('change_time', 'options'), Output('speed_dropdown', 'disabled'), Output('playback_specs', 'data', allow_duplicate=True), - Input('playback_btn', 'n_clicks'), + [Input('playback_btn', 'n_clicks'), State('playback_speed_store', 'data'), State('configs', 'data'), - State('sim_settings', 'data'), + State('sim_times', 'data'), + State('radar_info', 'data')], prevent_initial_call=True) -def initiate_playback(_nclick, playback_speed, cfg, sa): +def initiate_playback(_nclick, playback_speed, cfg, sim_times, radar_info): """ Enables/disables interval component that elapses the playback time. The user can only click this button this once. @@ -1257,29 +887,31 @@ def initiate_playback(_nclick, playback_speed, cfg, sa): playback_specs = { 'playback_paused': False, - 'playback_clock': sa['playback_clock'], - 'playback_clock_str': sa['playback_clock_str'], - 'playback_end': sa['playback_end'], + 'playback_clock': sim_times['playback_clock'], + 'playback_clock_str': sim_times['playback_clock_str'], + 'playback_start': sim_times['playback_start'], + 'playback_start_str': sim_times['playback_start_str'], + 'playback_end': sim_times['playback_end'], + 'playback_end_str': sim_times['playback_end_str'], 'playback_speed': playback_speed, - 'new_radar': sa['new_radar'], - 'radar_list': sa['radar_list'], - 'playback_start': sa['playback_start'], + 'new_radar': radar_info['new_radar'], + 'radar_list': radar_info['radar_list'], } btn_text = 'Simulation Launched' btn_disabled = True playback_running = True - start = sa['playback_start_str'] - end = sa['playback_end_str'] + start = sim_times['playback_start_str'] + end = sim_times['playback_end_str'] style = lc.playback_times_style - options = sa['playback_dropdown_dict'] + options = sim_times['playback_dropdown_dict'] if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sa['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) - if sa['new_radar'] != 'None': - UpdateDirList(sa['new_radar'], sa['playback_clock_str'], cfg['POLLING_DIR']) + UpdateHodoHTML(sim_times['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + if radar_info['new_radar'] != 'None': + UpdateDirList(radar_info['new_radar'], sim_times['playback_clock_str'], cfg['POLLING_DIR']) else: - for _r, radar in enumerate(sa['radar_list']): - UpdateDirList(radar, sa['playback_clock_str'], cfg['POLLING_DIR']) + for _r, radar in enumerate(radar_info['radar_list']): + UpdateDirList(radar, sim_times['playback_clock_str'], cfg['POLLING_DIR']) return (btn_text, btn_disabled, False, playback_running, start, style, end, style, options, False, playback_specs) @@ -1439,40 +1071,24 @@ def update_playback_speed(selected_speed): ################################################################################################ @app.callback( Output('show_time_data', 'children'), - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_year', 'value'), + Output('sim_times', 'data'), + [Input('start_year', 'value'), Input('start_month', 'value'), Input('start_day', 'value'), Input('start_hour', 'value'), Input('start_minute', 'value'), - Input('duration', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' + Input('duration', 'value')] ) -def get_sim(_yr, _mo, _dy, _hr, _mn, _dur, sim_settings) -> str: +def get_sim(yr, mo, dy, hr, mn, dur) -> str: """ Changes to any of the Inputs above will trigger this callback function to update - the time summary displayed on the page. Variables already have been stored in sa - object for use in scripts so don't need to be explicitly returned here. + the time summary displayed on the page, as well as recomputing variables for + the simulation. """ - sim_settings = make_simulation_times(sim_settings) - line1 = f'{sim_settings['event_start_str']}Z ____ {sim_settings['event_duration']} minutes' - return line1, sim_settings - - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_year', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_year(start_year, sim_settings) -> int: - """ - Updates the start year variable - """ - sim_settings['event_start_year'] = start_year - return sim_settings - + dt = datetime(yr, mo, dy, hr, mn, second=0, tzinfo=timezone.utc) + line = f'{dt.strftime("%Y-%m-%d %H:%M")}Z ____ {dur} minutes' + sim_times = make_simulation_times(dt, dur) + return line, sim_times @app.callback( Output('start_day', 'options'), @@ -1486,76 +1102,6 @@ def update_day_dropdown(selected_year, selected_month): for day in range(1, num_days+1)] return day_options - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_month', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_month(start_month, sim_settings) -> int: - """ - Updates the start month variable - """ - sim_settings['event_start_month'] = start_month - return sim_settings - - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_day', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_day(start_day, sim_settings) -> int: - """ - Updates the start day variable - """ - sim_settings['event_start_day'] = start_day - return sim_settings - - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_hour', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_hour(start_hour, sim_settings) -> int: - """ - Updates the start hour variable - """ - sim_settings['event_start_hour'] = start_hour - return sim_settings - - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('start_minute', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_minute(start_minute, sim_settings) -> int: - """ - Updates the start minute variable - """ - sim_settings['event_start_minute'] = start_minute - return sim_settings - - -@app.callback( - Output('sim_settings', 'data', allow_duplicate=True), - Input('duration', 'value'), - State('sim_settings', 'data'), - prevent_initial_call='initial_duplicate' -) -def get_duration(duration, sim_settings) -> int: - """ - Updates the event duration (in minutes) - """ - sim_settings['event_duration'] = duration - return sim_settings - ################################################################################################ # ----------------------------- Start app ----------------------------------------------------- ################################################################################################ From 5537529dcca95a53bc15efe23ee05cb276b6de06 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Sun, 25 Aug 2024 23:47:03 -0500 Subject: [PATCH 16/23] Remove sim_settings dictionary references. Try to solve weird issue of radar selections occasionally briefly showing as valid and then reverting. --- app.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/app.py b/app.py index 853ab18..c362d0e 100644 --- a/app.py +++ b/app.py @@ -323,14 +323,12 @@ def create_radar_dict(sa) -> dict: @app.callback( Output('dynamic_container', 'children'), Output('layout_has_initialized', 'data'), - Output('sim_settings', 'data'), Input('directory_monitor', 'n_intervals'), State('layout_has_initialized', 'data'), State('dynamic_container', 'children'), - State('sim_settings', 'data'), State('configs', 'data') ) -def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, configs): +def generate_layout(n_intervals, layout_has_initialized, children, configs): """ Dynamically generate the layout, which was started in the config file to set up the unique session id. This callback should only be executed once at page load in. @@ -358,8 +356,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, 'new_lon': None, 'radar_files_dict': {} } - - sim_settings['playback_initiated'] = False + playback_speed = 1.0 # Settings for date dropdowns moved here to avoid specifying different values in @@ -415,10 +412,10 @@ def generate_layout(n_intervals, layout_has_initialized, children, sim_settings, layout_has_initialized['added'] = True - return children, layout_has_initialized, sim_settings + return children, layout_has_initialized create_logfile(configs['LOG_DIR']) - return children, layout_has_initialized, sim_settings + return children, layout_has_initialized ################################################################################################ ################################################################################################ @@ -449,23 +446,28 @@ def display_click_data(quant_str: str, click_data: dict, radar_info: dict): triggered_id = ctx.triggered_id radar_info['number_of_radars'] = int(quant_str[0:1]) + if triggered_id == 'radar_quantity': radar_info['number_of_radars'] = int(quant_str[0:1]) radar_info['radar_list'] = [] radar_info['radar_dict'] = {} return f'Use map to select {quant_str}', f'{select_action} selections', True, radar_info - try: - radar_info['radar'] = click_data['points'][0]['customdata'] - except (KeyError, IndexError, TypeError): - return 'No radar selected ...', f'{select_action} selections', True, radar_info - - if radar_info['radar'] not in radar_info['radar_list']: - radar_info['radar_list'].append(radar_info['radar']) - if len(radar_info['radar_list']) > radar_info['number_of_radars']: - radar_info['radar_list'] = radar_info['radar_list'][-radar_info['number_of_radars']:] - if len(radar_info['radar_list']) == radar_info['number_of_radars']: - select_action = 'Finalize' - btn_deactivated = False + + #try: + # radar_info['radar'] = click_data['points'][0]['customdata'] + #except (KeyError, IndexError, TypeError): + # return 'No radar selected ...', f'{select_action} selections', True, radar_info + if triggered_id == 'graph': + radar = click_data['points'][0]['customdata'] + + if radar not in radar_info['radar_list']: + radar_info['radar_list'].append(radar) + if len(radar_info['radar_list']) > radar_info['number_of_radars']: + radar_info['radar_list'] = radar_info['radar_list'][-radar_info['number_of_radars']:] + if len(radar_info['radar_list']) == radar_info['number_of_radars']: + select_action = 'Finalize' + btn_deactivated = False + radar_info['radar'] = radar listed_radars = ', '.join(radar_info['radar_list']) return listed_radars, f'{select_action} selections', btn_deactivated, radar_info @@ -773,11 +775,10 @@ def cancel_scripts(n_clicks, SESSION_ID) -> None: Output('model_status_warning', 'children'), Output('show_script_progress', 'children', allow_duplicate=True), [Input('directory_monitor', 'n_intervals'), - State('configs', 'data'), - State('sim_settings', 'data')], + State('configs', 'data')], prevent_initial_call=True ) -def monitor(_n, cfg, sim_settings): +def monitor(_n, cfg): """ This function is called every second by the directory_monitor interval. It (1) checks the status of the various scripts and reports them to the front-end application and From b1545da76348ef604698ece2cf157dfbc37944c9 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 02:06:15 -0500 Subject: [PATCH 17/23] Addressing latency issues, which seem to be the culprit behind inexplicable behavior when selecting radar sites (some would disappear immediately, others wouldn't even register). Set initial dynamic container which sets up app layout to only execute once with a new dcc.Interval. Only turn on monitor function when download and processing scripts are running. --- app.py | 233 +++++++++++++++++++++++++++++++++--------------------- config.py | 3 +- 2 files changed, 145 insertions(+), 91 deletions(-) diff --git a/app.py b/app.py index c362d0e..5ad9914 100644 --- a/app.py +++ b/app.py @@ -323,7 +323,7 @@ def create_radar_dict(sa) -> dict: @app.callback( Output('dynamic_container', 'children'), Output('layout_has_initialized', 'data'), - Input('directory_monitor', 'n_intervals'), + Input('container_init', 'n_intervals'), State('layout_has_initialized', 'data'), State('dynamic_container', 'children'), State('configs', 'data') @@ -336,38 +336,68 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): """ if not layout_has_initialized['added']: - # Initialize variables + # Initialize configurable variables for load in event_start_year = 2024 event_start_month = 7 event_start_day = 16 event_start_hour = 0 event_start_minute = 30 event_duration = 60 + playback_speed = 1.0 + number_of_radars = 1 + radar_list = [] + radar_dict = {} + radar = None + new_radar = 'None' + lat = None + lon = None + new_lat = None + new_lon = None + radar_files_dict = {} + ################################################# + + monitor_store = {} + monitor_store['radar_dl_completion'] = 0 + monitor_store['hodograph_completion'] = 0 + monitor_store['munger_completion'] = 0 + monitor_store['placefile_status_string'] = "" + monitor_store['model_list'] = [] + monitor_store['model_warning'] = "" radar_info = { - 'number_of_radars': 1, - 'radar_list': [], - 'radar_dict': {}, - 'radar': None, - 'new_radar': 'None', - 'lat': None, - 'lon': None, - 'new_lat': None, - 'new_lon': None, - 'radar_files_dict': {} + 'number_of_radars': number_of_radars, + 'radar_list': radar_list, + 'radar_dict': radar_dict, + 'radar': radar, + 'new_radar': new_radar, + 'lat': lat, + 'lon': lon, + 'new_lat': new_lat, + 'new_lon': new_lon, + 'radar_files_dict': radar_files_dict } - playback_speed = 1.0 - # Settings for date dropdowns moved here to avoid specifying different values in # the layout now = datetime.now(pytz.utc) - sim_year_section = dbc.Col(html.Div([lc.step_year, dcc.Dropdown(np.arange(1992, now.year + 1), event_start_year, id='start_year', clearable=False),])) - sim_month_section = dbc.Col(html.Div([lc.step_month, dcc.Dropdown(np.arange(1, 13), event_start_month, id='start_month', clearable=False),])) - sim_day_selection = dbc.Col(html.Div([lc.step_day, dcc.Dropdown(np.arange(1, 31), event_start_day, id='start_day', clearable=False)])) - sim_hour_section = dbc.Col(html.Div([lc.step_hour, dcc.Dropdown(np.arange(0, 24), event_start_hour, id='start_hour', clearable=False),])) - sim_minute_section = dbc.Col(html.Div([lc.step_minute, dcc.Dropdown([0, 15, 30, 45], event_start_minute, id='start_minute', clearable=False),])) - sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown(np.arange(0, 240, 15), event_duration, id='duration', clearable=False),])) + sim_year_section = dbc.Col(html.Div([lc.step_year, dcc.Dropdown( + np.arange(1992, now.year+1), event_start_year, + id='start_year', clearable=False),])) + sim_month_section = dbc.Col(html.Div([lc.step_month, dcc.Dropdown( + np.arange(1, 13), event_start_month, + id='start_month', clearable=False),])) + sim_day_selection = dbc.Col(html.Div([lc.step_day, dcc.Dropdown( + np.arange(1, 31), event_start_day, + id='start_day', clearable=False)])) + sim_hour_section = dbc.Col(html.Div([lc.step_hour, dcc.Dropdown( + np.arange(0, 24), event_start_hour, + id='start_hour', clearable=False),])) + sim_minute_section = dbc.Col(html.Div([lc.step_minute, dcc.Dropdown( + [0, 15, 30, 45], event_start_minute, + id='start_minute', clearable=False),])) + sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown( + np.arange(0, 240, 15), event_duration, + id='duration', clearable=False),])) if children is None: children = [] @@ -377,14 +407,15 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): dcc.Store(id='tradar'), dcc.Store(id='dummy'), dcc.Store(id='playback_running_store', data=False), - dcc.Store(id='playback_start_store'), - dcc.Store(id='playback_end_store'), - dcc.Store(id='playback_clock_store'), + dcc.Store(id='playback_start_store'), # might be unused + dcc.Store(id='playback_end_store'), # might be unused + dcc.Store(id='playback_clock_store'), # might be unused dcc.Store(id='radar_info', data=radar_info), dcc.Store(id='sim_times'), dcc.Store(id='playback_speed_store', data=playback_speed), dcc.Store(id='playback_specs'), + dcc.Store(id='monitor_store', data=monitor_store), lc.top_section, lc.top_banner, dbc.Container([ dbc.Container([ @@ -453,21 +484,19 @@ def display_click_data(quant_str: str, click_data: dict, radar_info: dict): radar_info['radar_dict'] = {} return f'Use map to select {quant_str}', f'{select_action} selections', True, radar_info - #try: - # radar_info['radar'] = click_data['points'][0]['customdata'] - #except (KeyError, IndexError, TypeError): - # return 'No radar selected ...', f'{select_action} selections', True, radar_info - if triggered_id == 'graph': + try: radar = click_data['points'][0]['customdata'] - - if radar not in radar_info['radar_list']: - radar_info['radar_list'].append(radar) - if len(radar_info['radar_list']) > radar_info['number_of_radars']: - radar_info['radar_list'] = radar_info['radar_list'][-radar_info['number_of_radars']:] - if len(radar_info['radar_list']) == radar_info['number_of_radars']: - select_action = 'Finalize' - btn_deactivated = False - radar_info['radar'] = radar + except (KeyError, IndexError, TypeError): + return 'No radar selected ...', f'{select_action} selections', True, radar_info + + if radar not in radar_info['radar_list']: + radar_info['radar_list'].append(radar) + if len(radar_info['radar_list']) > radar_info['number_of_radars']: + radar_info['radar_list'] = radar_info['radar_list'][-radar_info['number_of_radars']:] + if len(radar_info['radar_list']) == radar_info['number_of_radars']: + select_action = 'Finalize' + btn_deactivated = False + radar_info['radar'] = radar listed_radars = ', '.join(radar_info['radar_list']) return listed_radars, f'{select_action} selections', btn_deactivated, radar_info @@ -729,7 +758,7 @@ def run_with_cancel_button(cfg, sim_times, radar_info): (Output('map_btn', 'disabled'), True, False), (Output('new_radar_selection', 'disabled'), True, False), (Output('run_scripts_btn', 'disabled'), True, False), - (Output('playback_clock_store', 'disabled'), True, False), + #(Output('playback_clock_store', 'disabled'), True, False), (Output('confirm_radars_btn', 'disabled'), True, False), # added radar confirm btn (Output('playback_btn', 'disabled'), True, False), # add start sim btn #(Output('pause_resume_playback_btn', 'disabled'), True, False), # add pause/resume btn @@ -774,58 +803,82 @@ def cancel_scripts(n_clicks, SESSION_ID) -> None: Output('model_table', 'data'), Output('model_status_warning', 'children'), Output('show_script_progress', 'children', allow_duplicate=True), + Output('monitor_store', 'data'), [Input('directory_monitor', 'n_intervals'), - State('configs', 'data')], + State('configs', 'data'), + State('cancel_scripts', 'disabled'), + State('monitor_store', 'data')], prevent_initial_call=True ) -def monitor(_n, cfg): +def monitor(_n, cfg, cancel_scripts_disabled, monitor_store): """ This function is called every second by the directory_monitor interval. It (1) checks the status of the various scripts and reports them to the front-end application and (2) monitors the completion status of the scripts. + + In order to reduce background latency, this funcion only fully executes when the + downloading and pre-processing scripts are running, defined by the cancel button + being enabled. Previous status data is stored in monitor_store. """ - processes = utils.get_app_processes() + radar_dl_completion = monitor_store['radar_dl_completion'] + hodograph_completion = monitor_store['hodograph_completion'] + munger_completion = monitor_store['munger_completion'] + placefile_status_string = monitor_store['placefile_status_string'] + model_list = monitor_store['model_list'] + model_warning = monitor_store['model_warning'] screen_output = "" - seen_scripts = [] - for p in processes: - process_session_id = p['session_id'] - if process_session_id == cfg['SESSION_ID']: - # Returns get_data or process (the two scripts launched by nse.py) - name = p['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] - - # Scripts executed as python modules will be like [python, -m, script.name] - if p['cmdline'][1] == '-m': - # Should return Nexrad, munger, nse, etc. - name = p['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] - if p['name'] == 'wgrib2': - name = 'wgrib2' - - if name in config.scripts_list and name not in seen_scripts: - runtime = time.time() - p['create_time'] - screen_output += f"{name}: running for {round(runtime,1)} s. " - seen_scripts.append(name) - - # Radar file download status - radar_dl_completion, radar_files = utils.radar_monitor(cfg['RADAR_DIR']) - - # Radar mungering/transposing status - munger_completion = utils.munger_monitor(cfg['RADAR_DIR'], cfg['POLLING_DIR']) - - # Surface placefile status - placefile_stats = utils.surface_placefile_monitor(cfg['PLACEFILES_DIR']) - placefile_status_string = f"{placefile_stats[0]}/{placefile_stats[1]} files found" - - # Hodographs. Currently hard-coded to expect 2 files for every radar and radar file. - num_hodograph_images = len(glob(f"{cfg['HODOGRAPHS_DIR']}/*.png")) - hodograph_completion = 0 - if len(radar_files) > 0: - hodograph_completion = 100 * \ - (num_hodograph_images / (2*len(radar_files))) - # NSE placefiles - model_list, model_warning = utils.nse_status_checker(cfg['MODEL_DIR']) + if not cancel_scripts_disabled: + processes = utils.get_app_processes() + seen_scripts = [] + for p in processes: + process_session_id = p['session_id'] + if process_session_id == cfg['SESSION_ID']: + # Returns get_data or process (the two scripts launched by nse.py) + name = p['cmdline'][1].rsplit('/')[-1].rsplit('.')[0] + + # Scripts executed as python modules will be like [python, -m, script.name] + if p['cmdline'][1] == '-m': + # Should return Nexrad, munger, nse, etc. + name = p['cmdline'][2].rsplit('/')[-1].rsplit('.')[-1] + if p['name'] == 'wgrib2': + name = 'wgrib2' + + if name in config.scripts_list and name not in seen_scripts: + runtime = time.time() - p['create_time'] + screen_output += f"{name}: running for {round(runtime,1)} s. " + seen_scripts.append(name) + + # Radar file download status + radar_dl_completion, radar_files = utils.radar_monitor(cfg['RADAR_DIR']) + + # Radar mungering/transposing status + munger_completion = utils.munger_monitor(cfg['RADAR_DIR'], cfg['POLLING_DIR']) + + # Surface placefile status + placefile_stats = utils.surface_placefile_monitor(cfg['PLACEFILES_DIR']) + placefile_status_string = f"{placefile_stats[0]}/{placefile_stats[1]} files found" + + # Hodographs. Currently hard-coded to expect 2 files for every radar and radar file. + num_hodograph_images = len(glob(f"{cfg['HODOGRAPHS_DIR']}/*.png")) + hodograph_completion = 0 + if len(radar_files) > 0: + hodograph_completion = 100 * \ + (num_hodograph_images / (2*len(radar_files))) + + # NSE placefiles + model_list, model_warning = utils.nse_status_checker(cfg['MODEL_DIR']) + + # Capture the latest status information + monitor_store['radar_dl_completion'] = radar_dl_completion + monitor_store['hodograph_completion'] = hodograph_completion + monitor_store['munger_completion'] = munger_completion + monitor_store['placefile_status_string'] = placefile_status_string + monitor_store['model_list'] = model_list + monitor_store['model_warning'] = model_warning + return (radar_dl_completion, hodograph_completion, munger_completion, - placefile_status_string, model_list, model_warning, screen_output) + placefile_status_string, model_list, model_warning, screen_output, monitor_store) ################################################################################################ # ----------------------------- Transpose placefiles in time and space ------------------------ @@ -882,7 +935,7 @@ def toggle_placefiles_section(n) -> dict: prevent_initial_call=True) def initiate_playback(_nclick, playback_speed, cfg, sim_times, radar_info): """ - Enables/disables interval component that elapses the playback time. The user can only + Enables/disables interval component that elapses the playback time. User can only click this button this once. """ @@ -907,9 +960,11 @@ def initiate_playback(_nclick, playback_speed, cfg, sim_times, radar_info): style = lc.playback_times_style options = sim_times['playback_dropdown_dict'] if config.PLATFORM != 'WINDOWS': - UpdateHodoHTML(sim_times['playback_clock_str'], cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) + UpdateHodoHTML(sim_times['playback_clock_str'], cfg['HODOGRAPHS_DIR'], + cfg['HODOGRAPHS_PAGE']) if radar_info['new_radar'] != 'None': - UpdateDirList(radar_info['new_radar'], sim_times['playback_clock_str'], cfg['POLLING_DIR']) + UpdateDirList(radar_info['new_radar'], sim_times['playback_clock_str'], + cfg['POLLING_DIR']) else: for _r, radar in enumerate(radar_info['radar_list']): UpdateDirList(radar, sim_times['playback_clock_str'], cfg['POLLING_DIR']) @@ -947,13 +1002,16 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, playback_s playback_btn_text = 'Pause Playback' # Variables stored dcc.Store object are strings. - specs['playback_clock'] = datetime.strptime(specs['playback_clock'], '%Y-%m-%dT%H:%M:%S+00:00') + specs['playback_clock'] = datetime.strptime(specs['playback_clock'], + '%Y-%m-%dT%H:%M:%S+00:00') # Unsure why these string representations change. try: - specs['playback_end'] = datetime.strptime(specs['playback_end'], '%Y-%m-%dT%H:%M:%S+00:00') + specs['playback_end'] = datetime.strptime(specs['playback_end'], + '%Y-%m-%dT%H:%M:%S+00:00') except ValueError: - specs['playback_end'] = datetime.strptime(specs['playback_end'], '%Y-%m-%dT%H:%M:%S') + specs['playback_end'] = datetime.strptime(specs['playback_end'], + '%Y-%m-%dT%H:%M:%S') if specs['playback_clock'].tzinfo is None: specs['playback_clock'] = specs['playback_clock'].replace(tzinfo=timezone.utc) @@ -1046,8 +1104,6 @@ def manage_clock_(nclicks, _n_intervals, new_time, _playback_running, playback_s # ----------------------------- Playback Speed Callbacks -------------------------------------- ################################################################################################ @app.callback( - #Output('playback_specs', 'data', allow_duplicate=True), - #Output('sim_settings', 'data', allow_duplicate=True), Output('playback_speed_store', 'data'), Input('speed_dropdown', 'value'), prevent_initial_call=True @@ -1056,15 +1112,12 @@ def update_playback_speed(selected_speed): """ Updates the playback speed in the sa object """ - #sim_settings['playback_speed'] = selected_speed try: - #sim_settings['playback_speed'] = float(selected_speed) selected_speed = float(selected_speed) except ValueError: print(f"Error converting {selected_speed} to float") selected_speed = 1.0 return selected_speed - #return specs ################################################################################################ diff --git a/config.py b/config.py index aa1486d..03e5222 100644 --- a/config.py +++ b/config.py @@ -60,11 +60,12 @@ def init_layout(): # Elements used to store and track the session id dcc.Store(id='session_id', data=session_id, storage_type='session'), dcc.Interval(id='setup', interval=1, n_intervals=0, max_intervals=1), + dcc.Interval(id='container_init', interval=1, n_intervals=0, max_intervals=10), dcc.Store(id='configs', data={}), dcc.Store(id='sim_settings', data={}), # Elements needed to set up the layout on page load by app.py - dcc.Interval(id='directory_monitor', interval=1000), + dcc.Interval(id='directory_monitor', interval=500), dcc.Store(id='layout_has_initialized', data={'added': False}), html.Div(id='dynamic_container') ]) From 6c5076bd13885fc93f066c84afaf3c71639c58e8 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 10:34:26 -0500 Subject: [PATCH 18/23] Move placefile and polling link set-up from layout_components into app to dynamically generate links based on session id. --- app.py | 135 ++++++++++++++++++++++++++++++++++++++----- config.py | 14 ++--- layout_components.py | 17 ++++-- 3 files changed, 134 insertions(+), 32 deletions(-) diff --git a/app.py b/app.py index 5ad9914..aa056d1 100644 --- a/app.py +++ b/app.py @@ -335,6 +335,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): Thereafter, layout_has_initialized will be set to True """ if not layout_has_initialized['added']: + if children is None: children = [] # Initialize configurable variables for load in event_start_year = 2024 @@ -398,10 +399,112 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown( np.arange(0, 240, 15), event_duration, id='duration', clearable=False),])) + + #try: + polling_section = dbc.Container(dbc.Container(html.Div( + [ + dbc.Row([ + dbc.Col(dbc.ListGroupItem("Copy this polling address into GR2Analyst -->"), + style=lc.steps_right, width=6), + dbc.Col(dbc.ListGroupItem(f"{configs['LINK_BASE']}/polling", + href=f"{configs['LINK_BASE']}/polling", target="_blank"), + style=lc.polling_link, width=6) + ]) + ]))) + + links_section = dbc.Container(dbc.Container(html.Div( + [polling_section, + lc.spacer_mini, + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("Graphics"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("Hodographs webpage", + href=f"{configs['LINK_BASE']}/hodographs.html", target="_blank"), width=2) + ], + style={"display": "flex", "flexWrap": "wrap"} + ), + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("Sfc obs"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("Regular font", + href=f"{configs['PLACEFILES_LINKS']}/latest_surface_observations_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("Large font", + href=f"{configs['PLACEFILES_LINKS']}/latest_surface_observations_lg_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("Small font", + href=f"{configs['PLACEFILES_LINKS']}/latest_surface_observations_xlg_shifted.txt", target="_blank"), width=2), + ], + style={"display": "flex", "flexWrap": "wrap"} + ), + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("Sfc obs parts"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("Wind", + href=f"{configs['PLACEFILES_LINKS']}/wind_shifted.txt", target="_blank"),width=2), + dbc.Col(dbc.ListGroupItem("Temp", href=f"{configs['PLACEFILES_LINKS']}/temp_shifted.txt", target="_blank"),width=2), + dbc.Col(dbc.ListGroupItem("Dwpt", href=f"{configs['PLACEFILES_LINKS']}/dwpt_shifted.txt", target="_blank"),width=2), + dbc.Col(dbc.ListGroupItem(" "),width=2), + ], + style={"display": "flex", "flexWrap": "wrap"}, + + ), + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("NSE Shear"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("Effective", href=f"{configs['PLACEFILES_LINKS']}/ebwd_shifted.txt", target="_blank"), + width=2), + dbc.Col(dbc.ListGroupItem("0-1 SHR", href=f"{configs['PLACEFILES_LINKS']}/shr1_shifted.txt", target="_blank"), + width=2), + dbc.Col(dbc.ListGroupItem( + "0-3 SHR", href=f"{configs['PLACEFILES_LINKS']}/shr3_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem( + "0-6 SHR", href=f"{configs['PLACEFILES_LINKS']}/shr6_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem( + "0-8 SHR", href=f"{configs['PLACEFILES_LINKS']}/shr8_shifted.txt", target="_blank"), width=2), + ], + style={"display": "flex", "flexWrap": "wrap"}, + + ), + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("NSE SRH"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("Effective", + href=f"{configs['PLACEFILES_LINKS']}/esrh_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("0-500m", href=f"{configs['PLACEFILES_LINKS']}/srh500_shifted.txt", target="_blank"), + width=2), + dbc.Col(dbc.ListGroupItem(" "), width=2), + dbc.Col(dbc.ListGroupItem(" "), width=2), + + ], + style={"display": "flex", "flexWrap": "wrap"}, + + ), + dbc.Row( + [ + dbc.Col(dbc.ListGroupItem("NSE Thermo"), style=lc.group_item_style, width=2), + dbc.Col(dbc.ListGroupItem("MLCAPE", + href=f"{configs['PLACEFILES_LINKS']}/mlcape_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("MLCIN", + href=f"{configs['PLACEFILES_LINKS']}/mlcin_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("0-3 MLCP", + href=f"{configs['PLACEFILES_LINKS']}/cape3km_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("0-3 LR", + href=f"{configs['PLACEFILES_LINKS']}/lr03km_shifted.txt", target="_blank"), width=2), + dbc.Col(dbc.ListGroupItem("MUCAPE", + href=f"{configs['PLACEFILES_LINKS']}/mucape_shifted.txt", target="_blank"), width=2), + ], + style={"display": "flex", "flexWrap": "wrap"}, + + ), + ] + ))) + + full_links_section = dbc.Container( + dbc.Container( + html.Div([ + links_section + ]),id="placefiles_section",style=lc.section_box_pad)) - if children is None: - children = [] - + new_items = dbc.Container([ dcc.Interval(id='playback_timer', disabled=True, interval=15*1000), dcc.Store(id='tradar'), @@ -432,20 +535,20 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): lc.scripts_button, lc.status_section, lc.spacer,lc.toggle_placefiles_btn,lc.spacer_mini, - lc.full_links_section, lc.spacer, + full_links_section, lc.spacer, simulation_playback_section, lc.radar_id, lc.bottom_section ]) + # Append the new component to the current list of children children = list(children) children.append(new_items) layout_has_initialized['added'] = True - + create_logfile(configs['LOG_DIR']) return children, layout_has_initialized - create_logfile(configs['LOG_DIR']) return children, layout_has_initialized ################################################################################################ @@ -623,16 +726,6 @@ def run_with_cancel_button(cfg, sim_times, radar_info): """ This version of the script-launcher trying to work in cancel button """ - log_string = ( - f"\n" - f"=========================Simulation Settings========================\n" - f"Session ID: {cfg['SESSION_ID']}\n" - f"{sim_times}\n" - f"{radar_info}\n" - f"====================================================================\n" - ) - logging.info(log_string) - UpdateHodoHTML('None', cfg['HODOGRAPHS_DIR'], cfg['HODOGRAPHS_PAGE']) # clean out old files and directories @@ -648,6 +741,16 @@ def run_with_cancel_button(cfg, sim_times, radar_info): except Exception as e: logging.exception("Error creating radar dict or config file: ", exc_info=True) + log_string = ( + f"\n" + f"=========================Simulation Settings========================\n" + f"Session ID: {cfg['SESSION_ID']}\n" + f"{sim_times}\n" + f"{radar_info}\n" + f"====================================================================\n" + ) + logging.info(log_string) + if len(radar_info['radar_list']) > 0: # Create initial dictionary of expected radar files. diff --git a/config.py b/config.py index 03e5222..8a380c7 100644 --- a/config.py +++ b/config.py @@ -34,13 +34,6 @@ CLOUD = False PLATFORM = 'WINDOWS' -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# This and LINK_BASE will need to be moved into the dcc.Store object. All references to -# these locations in layout_components will also need to be moved into app.py after -# directory paths are read in from -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -PLACEFILES_LINKS = f'{LINK_BASE}/placefiles' - ######################################################################################## # Initialize the application layout. A unique session ID will be generated on each page # load. @@ -55,12 +48,13 @@ def init_layout(): Initialize the layout with a unique session id. The 'dynamic container' is used within the app to build out the rest of the application layout on page load """ - session_id = f'{time.time_ns()//1000}_{uuid.uuid4().hex}' + #session_id = f'{time.time_ns()//1000}_{uuid.uuid4().hex}' + session_id = f'{time.time_ns()//1000000}' return dbc.Container([ # Elements used to store and track the session id dcc.Store(id='session_id', data=session_id, storage_type='session'), dcc.Interval(id='setup', interval=1, n_intervals=0, max_intervals=1), - dcc.Interval(id='container_init', interval=1, n_intervals=0, max_intervals=10), + dcc.Interval(id='container_init', interval=1, n_intervals=0, max_intervals=5), dcc.Store(id='configs', data={}), dcc.Store(id='sim_settings', data={}), @@ -103,7 +97,7 @@ def setup_paths_and_dirs(n_intervals, session_id): dirs['LOG_DIR'] = f"{dirs['BASE_DIR']}/data/logs" # Need to be updated - dirs['LINK_BASE'] = f"https://rssic.nws.noaa.gov/assets/{session_id}" + dirs['LINK_BASE'] = f"{LINK_BASE}/{session_id}" dirs['PLACEFILES_LINKS'] = f"{dirs['LINK_BASE']}/placefiles" dirs['HODO_HTML_LINK'] = f"{dirs['LINK_BASE']}/hodographs.html" diff --git a/layout_components.py b/layout_components.py index f48dfe4..30d90c9 100644 --- a/layout_components.py +++ b/layout_components.py @@ -10,7 +10,7 @@ import dash_bootstrap_components as dbc from dash import html, dcc, dash_table from dotenv import load_dotenv -from config import LINK_BASE, PLACEFILES_LINKS +#from config import LINK_BASE, PLACEFILES_LINKS load_dotenv() MAP_TOKEN = os.getenv("MAPBOX_TOKEN") @@ -388,6 +388,11 @@ group_item_style_left = {'font-weight': 'bold', 'color': '#cccccc', 'font-size': '1.2em', 'text-align': 'left'} +################################################################################################ +# Below items moved to application so session-specific placefile and polling directories can be +# built dynamically +################################################################################################ +''' polling_section = dbc.Container(dbc.Container(html.Div( [ dbc.Row([ @@ -404,7 +409,6 @@ # ----------------------------- Placefiles section -------------------------------------------- ################################################################################################ - links_section = dbc.Container(dbc.Container(html.Div( [polling_section, spacer_mini, @@ -490,15 +494,16 @@ ]#,id="placefiles_section", style={'display': 'block'} ))) -toggle_placefiles_btn = dbc.Container(dbc.Col(html.Div([dbc.Button( - 'Hide Links Section', size="lg", id='toggle_placefiles_section_btn', - n_clicks=0)],className="d-grid gap-2 col-12 mx-auto"))) - full_links_section = dbc.Container( dbc.Container( html.Div([ links_section ]),id="placefiles_section",style=section_box_pad)) +''' + +toggle_placefiles_btn = dbc.Container(dbc.Col(html.Div([dbc.Button( + 'Hide Links Section', size="lg", id='toggle_placefiles_section_btn', + n_clicks=0)],className="d-grid gap-2 col-12 mx-auto"))) ################################################################################################ # ----------------------------- Clock components ---------------------------------------------- From d4b9dcae23f3da34db9e91b4d4b92d5195b58886 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 11:35:23 -0500 Subject: [PATCH 19/23] Address issues: 1) where user could initially select 1 radar and transpose to a site. If they switched to 2 or 3 radars, the previous "shifted" radar would still be applied. 2) status updates wouldn't complete on hodos as scripts ended before a final pass of monitor() could be made. 3) configs dict from dcc.Store could be None on initial page load. --- app.py | 46 +++++++++++++++++++++++++++++++--------------- config.py | 8 ++++---- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/app.py b/app.py index aa056d1..6f4d444 100644 --- a/app.py +++ b/app.py @@ -323,18 +323,18 @@ def create_radar_dict(sa) -> dict: @app.callback( Output('dynamic_container', 'children'), Output('layout_has_initialized', 'data'), - Input('container_init', 'n_intervals'), + #Input('container_init', 'n_intervals'), State('layout_has_initialized', 'data'), State('dynamic_container', 'children'), - State('configs', 'data') + Input('configs', 'data') ) -def generate_layout(n_intervals, layout_has_initialized, children, configs): +def generate_layout(layout_has_initialized, children, configs): """ Dynamically generate the layout, which was started in the config file to set up the unique session id. This callback should only be executed once at page load in. Thereafter, layout_has_initialized will be set to True """ - if not layout_has_initialized['added']: + if not layout_has_initialized['added'] and configs is not None: if children is None: children = [] # Initialize configurable variables for load in @@ -364,6 +364,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): monitor_store['placefile_status_string'] = "" monitor_store['model_list'] = [] monitor_store['model_warning'] = "" + monitor_store['scripts_previously_running'] = False radar_info = { 'number_of_radars': number_of_radars, @@ -399,8 +400,7 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown( np.arange(0, 240, 15), event_duration, id='duration', clearable=False),])) - - #try: + print(configs) polling_section = dbc.Container(dbc.Container(html.Div( [ dbc.Row([ @@ -518,7 +518,11 @@ def generate_layout(n_intervals, layout_has_initialized, children, configs): dcc.Store(id='sim_times'), dcc.Store(id='playback_speed_store', data=playback_speed), dcc.Store(id='playback_specs'), + + # For app/script monitoring + dcc.Interval(id='directory_monitor', interval=2000), dcc.Store(id='monitor_store', data=monitor_store), + lc.top_section, lc.top_banner, dbc.Container([ dbc.Container([ @@ -650,11 +654,12 @@ def finalize_radar_selections(clicks: int, _quant_str: str, radar_info: dict) -> @app.callback( Output('tradar', 'data'), Output('radar_info', 'data', allow_duplicate=True), - Input('new_radar_selection', 'value'), - State('radar_info', 'data'), + [Input('new_radar_selection', 'value'), + Input('radar_quantity', 'value'), + State('radar_info', 'data')], prevent_initial_call=True ) -def transpose_radar(value, radar_info): +def transpose_radar(value, radar_quantity, radar_info): """ If a user switches from a selection BACK to "None", without this, the application will not update new_radar to None. Instead, it'll be the previous selection. @@ -665,8 +670,9 @@ def transpose_radar(value, radar_info): tradar store value is not used (currently). """ radar_info['new_radar'] = 'None' - - if value != 'None': + radar_info['new_lat'] = None + radar_info['new_lon'] = None + if value != 'None' and int(radar_quantity[0:1]) == 1: new_radar = value radar_info['new_radar'] = new_radar radar_info['new_lat'] = lc.df[lc.df['radar'] == new_radar]['lat'].values[0] @@ -795,7 +801,7 @@ def run_with_cancel_button(cfg, sim_times, radar_info): except Exception as e: print(f"Error with UpdateDirList ", e) logging.exception(f"Error with UpdateDirList ", exc_info=True) - + # Surface observations args = [str(radar_info['lat']), str(radar_info['lon']), sim_times['event_start_str'], cfg['PLACEFILES_DIR'], @@ -818,7 +824,7 @@ def run_with_cancel_button(cfg, sim_times, radar_info): # to transpose to. logging.info(f"Entering function run_transpose_script") run_transpose_script(cfg['PLACEFILES_DIR'], sim_times, radar_info) - + # Hodographs for radar, data in radar_info['radar_dict'].items(): try: @@ -913,7 +919,7 @@ def cancel_scripts(n_clicks, SESSION_ID) -> None: State('monitor_store', 'data')], prevent_initial_call=True ) -def monitor(_n, cfg, cancel_scripts_disabled, monitor_store): +def monitor(_n, cfg, cancel_btn_disabled, monitor_store): """ This function is called every second by the directory_monitor interval. It (1) checks the status of the various scripts and reports them to the front-end application and @@ -931,7 +937,8 @@ def monitor(_n, cfg, cancel_scripts_disabled, monitor_store): model_warning = monitor_store['model_warning'] screen_output = "" - if not cancel_scripts_disabled: + # Scripts are running or they just recently ended. + if not cancel_btn_disabled or monitor_store['scripts_previously_running']: processes = utils.get_app_processes() seen_scripts = [] for p in processes: @@ -979,6 +986,15 @@ def monitor(_n, cfg, cancel_scripts_disabled, monitor_store): monitor_store['placefile_status_string'] = placefile_status_string monitor_store['model_list'] = model_list monitor_store['model_warning'] = model_warning + monitor_store['scripts_previously_running'] = True + + return (radar_dl_completion, hodograph_completion, munger_completion, + placefile_status_string, model_list, model_warning, screen_output, + monitor_store) + + # Scripts have completed/stopped, but were running the previous pass through. + if cancel_btn_disabled and monitor_store['scripts_previously_running']: + monitor_store['scripts_previously_running'] = False return (radar_dl_completion, hodograph_completion, munger_completion, placefile_status_string, model_list, model_warning, screen_output, monitor_store) diff --git a/config.py b/config.py index 8a380c7..7b7fd29 100644 --- a/config.py +++ b/config.py @@ -51,16 +51,16 @@ def init_layout(): #session_id = f'{time.time_ns()//1000}_{uuid.uuid4().hex}' session_id = f'{time.time_ns()//1000000}' return dbc.Container([ - # Elements used to store and track the session id + # Elements used to store and track the session id and initialize the layout dcc.Store(id='session_id', data=session_id, storage_type='session'), dcc.Interval(id='setup', interval=1, n_intervals=0, max_intervals=1), - dcc.Interval(id='container_init', interval=1, n_intervals=0, max_intervals=5), + dcc.Store(id='configs', data={}), - dcc.Store(id='sim_settings', data={}), + #dcc.Store(id='sim_settings', data={}), # Elements needed to set up the layout on page load by app.py - dcc.Interval(id='directory_monitor', interval=500), dcc.Store(id='layout_has_initialized', data={'added': False}), + #dcc.Interval(id='container_init', interval=100, n_intervals=0, max_intervals=1), html.Div(id='dynamic_container') ]) From 61e85626c48c7fa36dcd423918a89689120308e6 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 12:02:55 -0500 Subject: [PATCH 20/23] App on AWS instance still allowing a new_radar when number_of_radars > 1. --- app.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 6f4d444..6ac9df5 100644 --- a/app.py +++ b/app.py @@ -400,7 +400,7 @@ def generate_layout(layout_has_initialized, children, configs): sim_duration_section = dbc.Col(html.Div([lc.step_duration, dcc.Dropdown( np.arange(0, 240, 15), event_duration, id='duration', clearable=False),])) - print(configs) + polling_section = dbc.Container(dbc.Container(html.Div( [ dbc.Row([ @@ -507,7 +507,7 @@ def generate_layout(layout_has_initialized, children, configs): new_items = dbc.Container([ dcc.Interval(id='playback_timer', disabled=True, interval=15*1000), - dcc.Store(id='tradar'), + #dcc.Store(id='tradar'), dcc.Store(id='dummy'), dcc.Store(id='playback_running_store', data=False), dcc.Store(id='playback_start_store'), # might be unused @@ -652,7 +652,7 @@ def finalize_radar_selections(clicks: int, _quant_str: str, radar_info: dict) -> ################################################################################################ @app.callback( - Output('tradar', 'data'), + #Output('tradar', 'data'), Output('radar_info', 'data', allow_duplicate=True), [Input('new_radar_selection', 'value'), Input('radar_quantity', 'value'), @@ -665,20 +665,17 @@ def transpose_radar(value, radar_quantity, radar_info): will not update new_radar to None. Instead, it'll be the previous selection. Since we always evaluate "value" after every user selection, always set new_radar initially to None. - - Added tradar as a dcc.Store as this callback didn't seem to execute otherwise. The - tradar store value is not used (currently). """ radar_info['new_radar'] = 'None' radar_info['new_lat'] = None radar_info['new_lon'] = None - if value != 'None' and int(radar_quantity[0:1]) == 1: + radar_info['number_of_radars'] = int(radar_quantity[0:1]) + if value != 'None' and radar_info['number_of_radars'] == 1: new_radar = value radar_info['new_radar'] = new_radar radar_info['new_lat'] = lc.df[lc.df['radar'] == new_radar]['lat'].values[0] radar_info['new_lon'] = lc.df[lc.df['radar'] == new_radar]['lon'].values[0] - return f'{new_radar}', radar_info - return 'None', radar_info + return radar_info ################################################################################################ # ----------------------------- Run Scripts button -------------------------------------------- From dfdd8f9043e3772780123f7cc5003f54ce9d7c04 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 14:30:51 -0500 Subject: [PATCH 21/23] Moving a few items to config file. --- config.py | 6 ++++++ utils.py | 6 +----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index 7b7fd29..ba66e2e 100644 --- a/config.py +++ b/config.py @@ -126,3 +126,9 @@ def setup_paths_and_dirs(n_intervals, session_id): # monitoring and/or cancelling. scripts_list = ["Nexrad", "munger", "obs_placefile", "nse", "wgrib2", "get_data", "process", "hodo_plot"] + +# Names of surface placefiles for monitoring script +surface_placefiles = [ + 'wind.txt', 'temp.txt', 'latest_surface_observations.txt', + 'latest_surface_observations_lg.txt', 'latest_surface_observations_xlg.txt' +] \ No newline at end of file diff --git a/utils.py b/utils.py index 21379ee..d3711b9 100644 --- a/utils.py +++ b/utils.py @@ -132,11 +132,7 @@ def munger_monitor(RADAR_DIR, POLLING_DIR): return percent_complete def surface_placefile_monitor(PLACEFILES_DIR): - filenames = [ - 'wind.txt', 'temp.txt', 'latest_surface_observations.txt', - 'latest_surface_observations_lg.txt', 'latest_surface_observations_xlg.txt' - ] - expected_files = [f"{PLACEFILES_DIR}/{i}" for i in filenames] + expected_files = [f"{PLACEFILES_DIR}/{i}" for i in config.surface_placefiles] files_on_system = [x for x in expected_files if os.path.exists(x)] #percent_complete = calc_completion_percentage(expected_files, files_on_system) From efa9b6a399209d1ff72918f2b9f121ad92059945 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 14:52:18 -0500 Subject: [PATCH 22/23] Reset basemap to mapbox style. --- layout_components.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/layout_components.py b/layout_components.py index 30d90c9..5153590 100644 --- a/layout_components.py +++ b/layout_components.py @@ -232,8 +232,8 @@ fig.update_layout( mapbox={'accesstoken': MAP_TOKEN, - 'style': "carto-darkmatter", - #'style': "mapbox://styles/mapbox/dark-v10", + #'style': "carto-darkmatter", + 'style': "mapbox://styles/mapbox/dark-v10", 'center': {'lon': -94.4, 'lat': 38.2}, 'zoom': 3.8}) From 4b36d58977597c39ebfaf342d4f6dc46a753e273 Mon Sep 17 00:00:00 2001 From: lcarlaw Date: Mon, 26 Aug 2024 22:09:34 -0500 Subject: [PATCH 23/23] Don't need flask server? Keeps this similar to original. --- config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index ba66e2e..3aae40a 100644 --- a/config.py +++ b/config.py @@ -5,9 +5,9 @@ import os import sys import time -import uuid +#import uuid -import flask +#import flask from dash import Dash, dcc, html, State, Input, Output import dash_bootstrap_components as dbc @@ -38,9 +38,11 @@ # Initialize the application layout. A unique session ID will be generated on each page # load. ######################################################################################## -server = flask.Flask(__name__) app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG], - suppress_callback_exceptions=True, update_title=None, server=server) + suppress_callback_exceptions=True, update_title=None) +#server = flask.Flask(__name__) +#app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG], +# suppress_callback_exceptions=True, update_title=None, server=server) app.title = "Radar Simulator" def init_layout():