Merge pull request #44 from lcarlaw/multiple-sessions-2

Multiple sessions 2

config.py

@@ -4,8 +4,16 @@
 from pathlib import Path
 import os
 import sys
+import time 
+#import uuid
 
-#BASE_DIR = Path.cwd()
+#import flask
+from dash import Dash, dcc, html, State, Input, Output
+import dash_bootstrap_components as dbc
+
+########################################################################################
+# Define the initial base directory 
+########################################################################################
 BASE_DIR = Path('/data/cloud-radar-server')
 LINK_BASE = "https://rssic.nws.noaa.gov/assets"
 CLOUD = True
@@ -26,42 +34,103 @@
     CLOUD = False
     PLATFORM = 'WINDOWS'
 
+########################################################################################
+# Initialize the application layout. A unique session ID will be generated on each page
+# load. 
+########################################################################################
+app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG],
+           suppress_callback_exceptions=True, update_title=None)
+#server = flask.Flask(__name__)
+#app = Dash(__name__, external_stylesheets=[dbc.themes.CYBORG],
+#           suppress_callback_exceptions=True, update_title=None, server=server)
+app.title = "Radar Simulator"
+
+def init_layout():
+    """
+    Initialize the layout with a unique session id.  The 'dynamic container' is used 
+    within the app to build out the rest of the application layout on page load
+    """
+    #session_id = f'{time.time_ns()//1000}_{uuid.uuid4().hex}'
+    session_id = f'{time.time_ns()//1000000}'
+    return dbc.Container([
+        # Elements used to store and track the session id and initialize the layout
+        dcc.Store(id='session_id', data=session_id, storage_type='session'),
+        dcc.Interval(id='setup', interval=1, n_intervals=0, max_intervals=1),
+
+        dcc.Store(id='configs', data={}),
+        #dcc.Store(id='sim_settings', data={}),
+
+        # Elements needed to set up the layout on page load by app.py
+        dcc.Store(id='layout_has_initialized', data={'added': False}),
+        #dcc.Interval(id='container_init', interval=100, n_intervals=0, max_intervals=1),
+        html.Div(id='dynamic_container')
+    ])
 
-ASSETS_DIR = BASE_DIR / 'assets'
-PLACEFILES_DIR = ASSETS_DIR / 'placefiles'
+app.layout = init_layout
 
-PLACEFILES_LINKS = f'{LINK_BASE}/placefiles'
+@app.callback(
+    Output('configs', 'data'),
+    Input('setup', 'n_intervals'),
+    State('session_id', 'data')
+)
+def setup_paths_and_dirs(n_intervals, session_id):
+    """
+    Callback executed once on page load to query the session id in dcc.Store component.
+    Creates a dictionary of directory paths and stores in a separate dcc.Store component.  
 
-HODOGRAPHS_DIR = ASSETS_DIR / 'hodographs'
-HODOGRAPHS_PAGE = ASSETS_DIR / 'hodographs.html'
-HODO_HTML_PAGE = HODOGRAPHS_PAGE
+    We cannot pass pathlib.Path objects in this dictionary since they're not 
+    JSON-serializable.
+    """
+    if n_intervals > 0:
+        dirs = {
+            'BASE_DIR': f'{BASE_DIR}',
+            'ASSETS_DIR': f'{BASE_DIR}/assets/{session_id}',
+            'DATA_DIR': f'{BASE_DIR}/data/{session_id}'
+        }
 
-HODO_HTML_LINK = f'{LINK_BASE}/hodographs.html'
-HODO_IMAGES = ASSETS_DIR / 'hodographs'
-POLLING_DIR = ASSETS_DIR / 'polling'
+        dirs['SESSION_ID'] = session_id
+        dirs['PLACEFILES_DIR'] = f"{dirs['ASSETS_DIR']}/placefiles"
+        dirs['HODOGRAPHS_DIR'] = f"{dirs['ASSETS_DIR']}/hodographs"
+        dirs['HODOGRAPHS_PAGE'] = f"{dirs['ASSETS_DIR']}/hodographs.html"
+        dirs['HODO_HTML_PAGE'] = dirs['HODOGRAPHS_PAGE']
+        dirs['POLLING_DIR'] = f"{dirs['ASSETS_DIR']}/polling"
+        dirs['MODEL_DIR'] = f"{dirs['DATA_DIR']}/model_data"
+        dirs['RADAR_DIR'] = f"{dirs['DATA_DIR']}/radar"
+        dirs['LOG_DIR'] = f"{dirs['BASE_DIR']}/data/logs"
 
+        # Need to be updated
+        dirs['LINK_BASE'] = f"{LINK_BASE}/{session_id}"
+        dirs['PLACEFILES_LINKS'] = f"{dirs['LINK_BASE']}/placefiles"
+        dirs['HODO_HTML_LINK'] = f"{dirs['LINK_BASE']}/hodographs.html"
 
-DATA_DIR = BASE_DIR / 'data'
-MODEL_DIR = DATA_DIR / 'model_data'
-RADAR_DIR = DATA_DIR / 'radar'
-LOG_DIR = DATA_DIR / 'logs'
-CSV_PATH = BASE_DIR / 'radars.csv'
-SCRIPTS_DIR = BASE_DIR / 'scripts'
-OBS_SCRIPT_PATH = SCRIPTS_DIR / 'obs_placefile.py'
-HODO_SCRIPT_PATH = SCRIPTS_DIR / 'hodo_plot.py'
-NEXRAD_SCRIPT_PATH = SCRIPTS_DIR / 'Nexrad.py'
-L2MUNGER_FILEPATH = SCRIPTS_DIR / 'l2munger'
-MUNGER_SCRIPT_FILEPATH = SCRIPTS_DIR / 'munger.py'
-MUNGE_DIR = SCRIPTS_DIR / 'munge'
-nse_script_path = SCRIPTS_DIR / 'nse.py'
-NSE_SCRIPT_PATH = SCRIPTS_DIR / 'nse.py'
-DEBZ_FILEPATH = SCRIPTS_DIR / 'debz.py'
-LOG_DIR = DATA_DIR / 'logs'
+        # Static directories (not dependent on session id)
+        dirs['SCRIPTS_DIR'] = f'{BASE_DIR}/scripts'
+        dirs['OBS_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/obs_placefile.py'
+        dirs['HODO_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/hodo_plot.py'
+        dirs['NEXRAD_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/Nexrad.py'
+        dirs['L2MUNGER_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/l2munger'
+        dirs['MUNGER_SCRIPT_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/munger.py'
+        dirs['MUNGE_DIR'] = f'{dirs['SCRIPTS_DIR']}/munge'
+        dirs['NSE_SCRIPT_PATH'] = f'{dirs['SCRIPTS_DIR']}/nse.py'
+        dirs['DEBZ_FILEPATH'] = f'{dirs['SCRIPTS_DIR']}/debz.py'
+        dirs['CSV_PATH'] = f'{BASE_DIR}/radars.csv'
 
-os.makedirs(DATA_DIR, exist_ok=True)
-os.makedirs(HODO_IMAGES, exist_ok=True)
-os.makedirs(PLACEFILES_DIR, exist_ok=True)
+        os.makedirs(dirs['MODEL_DIR'], exist_ok=True)
+        os.makedirs(dirs['RADAR_DIR'], exist_ok=True)
+        os.makedirs(dirs['HODOGRAPHS_DIR'], exist_ok=True)
+        os.makedirs(dirs['PLACEFILES_DIR'], exist_ok=True)
+        os.makedirs(dirs['POLLING_DIR'], exist_ok=True)
+        os.makedirs(dirs['LOG_DIR'], exist_ok=True)
 
-# 
+        return dirs
+
+# Names (without extensions) of various pre-processing scripts. Needed for script 
+# monitoring and/or cancelling. 
 scripts_list = ["Nexrad", "munger", "obs_placefile", "nse", "wgrib2", 
                 "get_data", "process", "hodo_plot"]
+
+# Names of surface placefiles for monitoring script
+surface_placefiles = [
+    'wind.txt', 'temp.txt', 'latest_surface_observations.txt',
+    'latest_surface_observations_lg.txt', 'latest_surface_observations_xlg.txt'
+]

layout_components.py

@@ -10,11 +10,11 @@
 import dash_bootstrap_components as dbc
 from dash import html, dcc, dash_table
 from dotenv import load_dotenv
-from config import LINK_BASE, PLACEFILES_LINKS
+#from config import LINK_BASE, PLACEFILES_LINKS
 
 load_dotenv()
 MAP_TOKEN = os.getenv("MAPBOX_TOKEN")
-now = datetime.now(pytz.utc)
+#now = datetime.now(pytz.utc)
 
 df = pd.read_csv('radars.csv', dtype={'lat': float, 'lon': float})
 # df = pd.read_csv('radars_no_tdwr.csv', dtype={'lat': float, 'lon': float})
@@ -143,6 +143,10 @@
 step_minute = html.Div(children="Minute", style=time_headers)
 step_duration = html.Div(children="Duration", style=time_headers)
 
+# Date settings (sim_year_section, sim_month_section, etc.) moved to app.py
+'''
+# Moved these into the app in order to control defaults more easily and remove the 
+# potential to specify different values in two locations
 sim_year_section = dbc.Col(html.Div([step_year,
                                      dcc.Dropdown(np.arange(1992, now.year + 1), now.year,
                                                   id='start_year', clearable=False),]))
@@ -158,6 +162,7 @@
 
 sim_duration_section = dbc.Col(html.Div([
     step_duration, dcc.Dropdown(np.arange(0, 240, 15), 60, id='duration', clearable=False),]))
+'''
 
 CONFIRM_TIMES_TEXT = "Confirm start time and duration -->"
 confirm_times_section = dbc.Col(
@@ -383,6 +388,11 @@
 group_item_style_left = {'font-weight': 'bold', 'color': '#cccccc',
                     'font-size': '1.2em', 'text-align': 'left'}
 
+################################################################################################
+# Below items moved to application so session-specific placefile and polling directories can be 
+# built dynamically
+################################################################################################
+'''
 polling_section = dbc.Container(dbc.Container(html.Div(
     [
         dbc.Row([
@@ -399,7 +409,6 @@
 # ----------------------------- Placefiles section  --------------------------------------------
 ################################################################################################
 
-
 links_section = dbc.Container(dbc.Container(html.Div(
     [polling_section,
         spacer_mini,
@@ -485,15 +494,16 @@
     ]#,id="placefiles_section", style={'display': 'block'}
 )))
 
-toggle_placefiles_btn = dbc.Container(dbc.Col(html.Div([dbc.Button(
-    'Hide Links Section', size="lg", id='toggle_placefiles_section_btn',
-    n_clicks=0)],className="d-grid gap-2 col-12 mx-auto")))
-
 full_links_section = dbc.Container(
     dbc.Container(
     html.Div([
             links_section
               ]),id="placefiles_section",style=section_box_pad))
+'''
+
+toggle_placefiles_btn = dbc.Container(dbc.Col(html.Div([dbc.Button(
+    'Hide Links Section', size="lg", id='toggle_placefiles_section_btn',
+    n_clicks=0)],className="d-grid gap-2 col-12 mx-auto")))
 
 ################################################################################################
 # ----------------------------- Clock components  ----------------------------------------------
@@ -548,7 +558,8 @@
 playback_speed_label = html.Div(children="Playback Speed", style=time_headers)
 playback_speed_dropdown_values = [0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 10.0]
 playback_speed_options = [{'label': str(i) + 'x', 'value': i} for i in playback_speed_dropdown_values]
-playback_speed_dropdown = dcc.Dropdown(options=playback_speed_options, value=1.0, id='speed_dropdown')
+playback_speed_dropdown = dcc.Dropdown(options=playback_speed_options, value=1.0, id='speed_dropdown',
+                                       disabled=True)
 playback_speed_col = dbc.Col(html.Div([playback_speed_label, spacer_mini, playback_speed_dropdown]))
 
 

scripts/Nexrad.py

@@ -15,7 +15,7 @@
 import botocore
 from botocore.client import Config
 
-from config import RADAR_DIR
+#from config import RADAR_DIR
 
 class NexradDownloader:
     """
@@ -32,7 +32,7 @@ class NexradDownloader:
     """
 
 
-    def __init__(self, radar_id, start_tstr, duration, download):
+    def __init__(self, radar_id, start_tstr, duration, download, RADAR_DIR):
         super().__init__()
         self.radar_id = radar_id
         self.start_tstr = start_tstr
@@ -45,7 +45,7 @@ def __init__(self, radar_id, start_tstr, duration, download):
                                         user_agent_extra='Resource')).Bucket('noaa-nexrad-level2')
 
         self.prefix_day_one, self.prefix_day_two = self.make_prefix()
-        self.download_directory = RADAR_DIR / self.radar_id / 'downloads'
+        self.download_directory = Path(f"{RADAR_DIR}/{self.radar_id}/downloads")
         os.makedirs(self.download_directory, exist_ok=True)
         self.process_files()
         sys.stdout.write(json.dumps(self.radar_files_dict))
@@ -124,4 +124,4 @@ def process_files(self) -> None:
     download_flag = sys.argv[4]
     if type(download_flag) == str:
         download_flag = ast.literal_eval(download_flag)
-    NexradDownloader(sys.argv[1], sys.argv[2], sys.argv[3], download_flag)
+    NexradDownloader(sys.argv[1], sys.argv[2], sys.argv[3], download_flag, sys.argv[5])

scripts/hodo_plot.py

@@ -18,7 +18,7 @@
 from multiprocessing import Pool, freeze_support
 from pathlib import Path
 
-from config import RADAR_DIR, HODO_IMAGES
+#from config import RADAR_DIR, HODO_IMAGES
 import scripts.hodo_resources as hr
 
 from dotenv import load_dotenv
@@ -42,10 +42,9 @@
     asos_two = None
 
 timeshift_seconds = int(sys.argv[5])
-#BASE_DIR = Path('/data/cloud-radar-server')
-#RADAR_DIR = BASE_DIR / 'data' / 'radar'
-#HODO_IMAGES = BASE_DIR / 'assets'/ 'hodographs'
 
+RADAR_DIR = Path(sys.argv[6]) 
+HODO_IMAGES = Path(sys.argv[7])
 THIS_RADAR = RADAR_DIR / radar_id
 os.makedirs(THIS_RADAR, exist_ok=True)
 DOWNLOADS = THIS_RADAR / 'downloads'

scripts/munger.py

@@ -19,8 +19,9 @@
 from datetime import datetime, timedelta, timezone
 import time
 import pytz
+from pathlib import Path
 
-from config import RADAR_DIR, POLLING_DIR, L2MUNGER_FILEPATH, DEBZ_FILEPATH
+#from config import RADAR_DIR, POLLING_DIR, L2MUNGER_FILEPATH, DEBZ_FILEPATH
 
 class Munger():
     """
@@ -44,19 +45,24 @@ class Munger():
          If None, will use original radar location 
     """
 
-    def __init__(self, original_rda, playback_start, duration, timeshift, new_rda='None'):
+    def __init__(self, original_rda, playback_start, duration, timeshift, RADAR_DIR, POLLING_DIR,
+                 L2MUNGER_FILEPATH, DEBZ_FILEPATH, new_rda='None'):
         self.original_rda = original_rda.upper()
         print(self.original_rda)
-        self.source_directory = RADAR_DIR / self.original_rda / 'downloads'
+        self.source_directory = Path(f"{RADAR_DIR}/{self.original_rda}/downloads")
         os.makedirs(self.source_directory, exist_ok=True)
         self.playback_start = datetime.strptime(playback_start,"%Y-%m-%d %H:%M").replace(tzinfo=pytz.UTC)
         self.duration = duration
         self.seconds_shift = int(timeshift)    # Needed for data passed in via command line.
         self.new_rda = new_rda
-        self.this_radar_polling_dir = POLLING_DIR / self.original_rda
+        self.polling_dir = Path(POLLING_DIR)
+        self.assets_dir = self.polling_dir.parent
+        self.this_radar_polling_dir = Path(f"{POLLING_DIR}/{self.original_rda}")
         if self.new_rda != 'None':
-            self.this_radar_polling_dir = POLLING_DIR / self.new_rda.upper()
+            self.this_radar_polling_dir = Path(f"{POLLING_DIR}/{self.new_rda.upper()}")
 
+        self.l2munger_filepath = Path(L2MUNGER_FILEPATH)
+        self.debz_filepath = Path(DEBZ_FILEPATH)
         os.makedirs(self.this_radar_polling_dir, exist_ok=True)
 
         self.copy_l2munger_executable()
@@ -70,9 +76,9 @@ def copy_l2munger_executable(self) -> None:
         The compiled l2munger executable needs to be in the source
         radar files directory to work properly
         """
-        chmod_cmd = f'chmod 775 {L2MUNGER_FILEPATH}'
+        chmod_cmd = f'chmod 775 {self.l2munger_filepath}'
         os.system(chmod_cmd)
-        cp_cmd = f'cp {L2MUNGER_FILEPATH} {self.source_directory}'
+        cp_cmd = f'cp {self.l2munger_filepath} {self.source_directory}'
         os.system(cp_cmd)
 
 
@@ -102,7 +108,7 @@ def uncompress_files(self) -> None:
 
             if 'V0' in filename_str:
                 # Keep existing logic for .V06 and .V08 files
-                command_str = f'python {DEBZ_FILEPATH} {filename_str} {filename_str}.uncompressed'
+                command_str = f'python {self.debz_filepath} {filename_str} {filename_str}.uncompressed'
                 os.system(command_str)
             else:
                 print(f'File type not recognized: {filename_str}')
@@ -163,7 +169,7 @@ def munge_files(self) -> None:
         munges radar files to start at the reference time
         also changes RDA location
         """
-        fout = open('/data/cloud-radar-server/assets/file_times.txt', 'w', encoding='utf-8')
+        fout = open(f'{self.assets_dir}/file_times.txt', 'w', encoding='utf-8')
         os.chdir(self.source_directory)
         self.source_files = list(self.source_directory.glob('*uncompressed'))
         for uncompressed_file in self.uncompressed_files:
@@ -198,5 +204,6 @@ def munge_files(self) -> None:
         playback_end_time = playback_start_time + timedelta(minutes=int(DURATION))
         Munger(ORIG_RDA, playback_start_str, DURATION, seconds_shift, NEW_RDA)
     else:
-        Munger(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
+        Munger(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], 
+               sys.argv[6], sys.argv[7], sys.argv[8], sys.argv[9])
         #original_rda, playback_start, duration, timeshift, new_rda, playback_speed=1.5

scripts/nse.py

@@ -7,7 +7,7 @@
 
 class Nse:
     def __init__(self, sim_start, event_duration, scripts_path, data_path, output_path):
-        self.sim_start = datetime.strptime(sim_start, '%Y-%m-%d %H:%M:%S+00:00')
+        self.sim_start = datetime.strptime(sim_start, '%Y-%m-%d %H:%M')
         self.sim_end = self.sim_start + timedelta(minutes=int(event_duration))
         self.start_string = datetime.strftime(self.sim_start,"%Y-%m-%d/%H")
         self.end_string = datetime.strftime(self.sim_end,"%Y-%m-%d/%H")