simplify Jupyter API further, use runid generated from portal in Jupy…

…ter information

Signed-off-by: Lance-Drane <[email protected]>

examples-proposed/004-time-loop/mymodule/components.py

@@ -43,13 +43,6 @@ def step(self, timestamp=0.0):
             # TODO - perhaps monitor timestep does not need to be called every step, but only every 20 steps?
             self.services.call(monitor, 'step', t)
 
-        # With this second "example" notebook, we only create it once and only write to it once.
-        self.services.initialize_jupyter_notebook(
-            dest_notebook_name=NOTEBOOK_2_NAME,  # path is relative to JupyterHub directory
-            source_notebook_path=NOTEBOOK_2_TEMPLATE,  # path is relative to input directory
-            initial_data_files=self.services.get_staged_jupyterhub_files(),
-        )
-
         self.services.call(worker, 'finalize', 0)
 
 
@@ -102,9 +95,9 @@ def step(self, timestamp=0.0, **keywords):
             data = f.read()
 
         # stage the state file in the JupyterHub directory
-        data_file = self.services.jupyterhub_make_state(state_file, timestamp)
-        print('ADD DATA FILE', data_file)
-        self.services.add_data_file_to_notebook(NOTEBOOK_1_NAME, data_file)
+        self.services.add_data_file_to_notebook(state_file, timestamp, NOTEBOOK_1_NAME)
 
         print('SEND PORTAL DATA', timestamp, data, file=stderr)
         self.services.send_portal_data(timestamp, data)
+
+        # TODO add a basic sleep to this example for demonstration purposes

examples-proposed/004-time-loop/sim.conf

@@ -13,8 +13,8 @@ INPUT_DIR = $SIM_ROOT/input_dir/
 USER_W3_DIR = $PWD/www
 USER_W3_BASEURL =
 
-PORTAL_URL = http://localhost:5000
-#PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org
+#PORTAL_URL = http://localhost:5000
+PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org
 
 # OPTIONAL
 # The BASE DIRECTORY of your machine's JupyterHub web server directory. This is used strictly for moving files around on the machine itself.

examples-proposed/004-time-loop/sim/input_dir/base-notebook-iterative.ipynb

@@ -8,12 +8,12 @@
    "outputs": [],
    "source": [
     "# Notebook template, the IPS Framework will add a cell before this one\n",
-    "# defining FILES as a list of state file paths.\n",
+    "# defining IPS_STATE_FILES as a list of state file paths.\n",
     "\n",
     "# In this example, this notebook is generated during the time loop.\n",
     "\n",
     "mapping = {}\n",
-    "for file in FILES:\n",
+    "for file in IPS_STATE_FILES:\n",
     "    with open(file, 'rb') as f:\n",
     "        mapping[file] = f.read()\n",
     "print(mapping)\n"

examples-proposed/004-time-loop/sim/input_dir/base-notebook-one-pass.ipynb

@@ -8,12 +8,12 @@
    "outputs": [],
    "source": [
     "# Notebook template, the IPS Framework will add a cell before this one\n",
-    "# defining FILES as a list of state file paths.\n",
+    "# defining IPS_STATE_FILES as a list of state file paths.\n",
     "\n",
     "# In this example, this notebook is only generated at the end of the run.\n",
     "\n",
     "mapping = {}\n",
-    "for file in FILES:\n",
+    "for file in IPS_STATE_FILES:\n",
     "    with open(file, 'rb') as f:\n",
     "        mapping[file] = f.read()\n",
     "print(mapping)\n"

ipsframework/configurationManager.py

@@ -246,6 +246,8 @@ def initialize(self, data_mgr, resource_mgr, task_mgr):
                 # Override platform value for PORTAL_URL if in simulation
                 if 'PORTAL_URL' in conf:
                     self.platform_conf['PORTAL_URL'] = conf['PORTAL_URL']
+                    if '_IPS_PORTAL_URL_HOST' in conf:
+                        self.platform_conf['_IPS_PORTAL_URL_HOST'] = conf['_IPS_PORTAL_URL_HOST']
 
             except (IOError, SyntaxError):
                 self.fwk.exception('Error opening config file %s: ', conf_file)
@@ -383,6 +385,17 @@ def _initialize_fwk_components(self):
 
             portal_conf['PORTAL_URL'] = self.get_platform_parameter('PORTAL_URL', silent=True)
 
+            if portal_conf['PORTAL_URL']:
+                from urllib.parse import urlparse
+
+                parsed_url = urlparse(portal_conf['PORTAL_URL'])
+                if parsed_url.port:  # get the host but be sure to strip out the username/password, so don't use netloc
+                    # don't use colons in filepaths
+                    portal_url_host = f'{parsed_url.hostname}_{parsed_url.port}'
+                else:
+                    portal_url_host = parsed_url.hostname
+                portal_conf['_IPS_PORTAL_URL_HOST'] = portal_url_host
+
             component_id = self._create_component(portal_conf, self.sim_map[self.fwk_sim_name])
             self.fwk_components.append(component_id)
 

ipsframework/jupyter.py

@@ -10,7 +10,9 @@
 ...in a shell on Jupyter NERSC.
 """
 
-from typing import List, Optional
+from os.path import sep
+from pathlib import Path
+from typing import Optional
 
 import nbformat as nbf
 
@@ -27,45 +29,39 @@ def replace_last(source_string: str, old: str, new: str) -> str:
     return f'{head}{new}{tail}'
 
 
-def _initial_jupyter_file_notebook_cell(variable: str, initial_data_files: Optional[List[str]] = None) -> str:
-    if not initial_data_files:
-        initial = ''
-    else:
-        itemsep = '\n'
-        initial = '\n' + itemsep.join([f"'{file}'," for file in initial_data_files])
+def _initial_jupyter_file_notebook_cell(dest: str, variable: str) -> str:
     return f"""{HOOK}
 
 import os
 
 # NOTE: directory should be sim_name plus the run id from the Portal
-# NOTE: add absolute path as a comment to the notebook cell
-# Uncomment below line to use any state files saved
+IPS_DATA_DIR = '{str(Path(dest).parent / 'data') + sep}'
+# Uncomment below line to implicitly use any state files saved in the data directory, note that the IPS framework explicitly lists out each file used
 #{variable} = os.listdir('data')
 # files created during the run
-{variable} = [{initial}
+{variable} = [
 ]
 """
 
 
-def initialize_jupyter_notebook(dest: str, src: str, variable_name: str, index: int, initial_data_files: Optional[List[str]] = None):
+def initialize_jupyter_notebook(dest: str, src: str, variable_name: str, index: int):
     """Create a new notebook from an old notebook, copying the result from 'src' to 'dest'.
 
     Params:
-      - dest - location of notebook to create on filesystem
+      - dest - location of notebook to create on filesystem (absolute file path)
       - src - location of source notebook on filesystem (is not overwritten unless src == dest)
       - variable_name: what to call the variable
       - index: insert new cells at position before this value (will not remove preexisting cells)
       - initial_data_files: optional list of files to initialize the notebook with
 
     """
     # to avoid conversion, use as_version=nbf.NO_CONVERT
-    #
     nb: nbf.NotebookNode = nbf.read(src, as_version=4)
 
     header = '# Next cell generated by IPS Framework'
     nb['cells'] = (
         nb['cells'][:index]
-        + [nbf.v4.new_markdown_cell(header), nbf.v4.new_code_cell(_initial_jupyter_file_notebook_cell(variable_name, initial_data_files))]
+        + [nbf.v4.new_markdown_cell(header), nbf.v4.new_code_cell(_initial_jupyter_file_notebook_cell(dest, variable_name))]
         + nb['cells'][index:]
     )
 
@@ -90,7 +86,7 @@ def add_data_file_to_notebook(dest: str, data_file: str, index: Optional[int] =
     ips_cell = nb['cells'][index]['source']
 
     # search from right of string for the ']' character, should work assuming user does not modify the cell past the variable definition
-    result = replace_last(ips_cell, ']', f"'{data_file}',\n]")
+    result = replace_last(ips_cell, ']', f"f'{{IPS_DATA_DIR}}{data_file}',\n]")
     nb['cells'][index]['source'] = result
 
     with open(dest, 'w') as f:

ipsframework/portalBridge.py

@@ -362,8 +362,9 @@ def check_send_post_responses(self):
 
             try:
                 data = json.loads(msg)
-                if 'runid' in data:
+                if 'runid' in data and 'simname' in data:
                     self.services.info('Run Portal URL = %s/%s', self.portal_url, data.get('runid'))
+                    self.services.set_config_param('_IPS_PORTAL_RUNID', str(data.get('runid')), target_sim_name=data.get('simname'))
 
                 msg = json.dumps(data)
             except (TypeError, json.decoder.JSONDecodeError):
@@ -404,14 +405,6 @@ def check_data_send_post_responses(self):
             except (EOFError, OSError):
                 break
 
-            try:
-                data = json.loads(msg)
-                if 'runid' in data:
-                    self.services.info('Run Portal URL = %s/%s', self.portal_url, data.get('runid'))
-
-                msg = json.dumps(data)
-            except (TypeError, json.decoder.JSONDecodeError):
-                pass
             if code == -1:
                 # disable portal, stop trying to send more data
                 self.portal_url = None
@@ -446,15 +439,6 @@ def send_notebook_url(self, sim_data, event_data):
                 except (EOFError, OSError):
                     break
 
-                print('PUT RESPONSE', code, msg)
-                try:
-                    data = json.loads(msg)
-                    if 'runid' in data:
-                        self.services.info('Run Portal URL = %s/%s', self.portal_url, data.get('runid'))
-
-                    msg = json.dumps(data)
-                except (TypeError, json.decoder.JSONDecodeError):
-                    pass
                 if code == -1:
                     # disable portal, stop trying to send more data
                     self.portal_url = None
@@ -616,6 +600,7 @@ def init_simulation(self, sim_name, sim_root):
         sim_data = self.SimulationData()
         sim_data.sim_name = sim_name
         sim_data.sim_root = sim_root
+        self.services.set_config_param('_IPS_PORTAL_URL_HOST', self._IPS_PORTAL_URL_HOST, target_sim_name=sim_name)
 
         d = datetime.datetime.now()
         date_str = '%s.%03d' % (d.strftime('%Y-%m-%dT%H:%M:%S'), int(d.microsecond / 1000))