Papermill implementation for BM4 and BM8

_data/python-pfhub/pfhub/func.py

@@ -154,6 +154,7 @@ def read_csv(sep_, path):
         FileNotFoundError,
         http.client.IncompleteRead,
         requests.exceptions.ConnectionError,
+        pandas.errors.ParserError,
     ) as error:
         logging.debug("%s for %s", error, path)
         return None

_data/python-pfhub/pfhub/main.py

@@ -575,7 +575,6 @@ def read_vega_data(keys, data):
     read_url = sequence(
         get("url"),
         read_csv(sep(data.get("format"))),
-        maybe(lambda x: x[list(data["format"]["parse"].keys())]),
     )
 
     read_values = sequence(get("values"), pandas.DataFrame)
@@ -584,7 +583,7 @@ def read_vega_data(keys, data):
         data,
         read_url if "url" in data else read_values,
         apply_transforms(data),
-        maybe(lambda x: get(keys, x)),
+        maybe(lambda x: get(keys, x, None)),
     )
 
 
@@ -632,11 +631,11 @@ def line_plot(
             x=columns[0],
             y=columns[1],
             color="sim_name",
-            labels=dict(
-                x=get("x_label", layout, default="x"),
-                y=get("y_label", layout, default="y"),
-                sim_name="Simulation Result",
-            ),
+            labels={
+                columns[0]: get("x_label", layout, default="x"),
+                columns[1]: get("y_label", layout, default="y"),
+                "sim_name": "Simulation Result",
+            },
             title=get("title", layout, default=""),
             log_x=get("log_x", layout, default=False),
             log_y=get("log_y", layout, default=False),

_data/simulations/Linear_Elasticity_4a/meta.yaml

@@ -36,42 +36,35 @@ data:
       - unit: KB
         value: '8388608'
   - name: contour
-    url: 'https://drive.google.com/open?id=1QCbHstjM024FaGELPsU2KYIN9Owddr62'
+    url: 'https://gist.githubusercontent.com/wd15/c8422453c1237b4c7d82411a18a997db/raw/21b4e0fa8d7d853ee55466ea037941bd2a307ffc/eq_data_format.csv'
     format:
       type: csv
       parse:
         x: number
-        'y': number
+        y: number
     description: >-
       The equilibrium data contains the coordinate of the contour plot at
       equilibrium. We plot the contour plot at the value of order parameter is
       equal to 0.5.
     type: line
-    transform:
-      - type: formula
-        expr: datum.x
-        as: x
-      - type: formula
-        expr: datum.y
-        as: 'y'
   - name: all_data
-    url: 'https://drive.google.com/open?id=1GDX3YSUwq7-UrMDcy8e-fyTYZthYlmoX'
+    url: 'https://gist.githubusercontent.com/wd15/c8422453c1237b4c7d82411a18a997db/raw/6b93cf84faf1ca9d55a319dadd89413b1af9168f/EnergyN_dt_p001.csv'
     format:
       type: csv
       parse:
-        Time: number
-        "Total Energy": number
+        time: number
+        total_free_energy: number
     description: >-
       Here, the data file includes the time, elastic, bulk, gradient and total
       free energy of the system.
     type: line
     transform:
       - type: formula
-        expr: datum.Time
+        expr: datum.time
         as: x
       - type: formula
-        expr: "datum['Total Energy']"
-        as: 'y'
+        expr: datum.total_free_energy
+        as: y
   - name: Energy_plot1
     url: 'https://drive.google.com/open?id=1AEuagAOSrmO3Zi1SjYTbavTodGCITRLx'
     type: image

_data/simulations/Linear_Elasticity_4c/meta.yaml

@@ -37,41 +37,34 @@ data:
       - unit: KB
         value: '8388608'
   - name: contour
-    url: 'https://drive.google.com/open?id=1mEbio1E7t7-wqRFSlI8f4gbzgY4JXyxR'
+    url: 'https://gist.githubusercontent.com/wd15/c8422453c1237b4c7d82411a18a997db/raw/050c22f38c42e1332860e07bc7fa035f9bd708a3/eq_data_format_c.csv'
     format:
       type: csv
       parse:
         x: number
-        'y': number
+        y: number
     description: >-
       The data file contains the coordinate of the contour plot at equilibrium
       considering the the order parameter value=0.5
     type: line
-    transform:
-      - type: formula
-        expr: datum.x
-        as: x
-      - type: formula
-        expr: datum.y
-        as: 'y'
   - name: all_data
-    url: 'https://drive.google.com/open?id=1wypfG6lOSeLHiAJzIggPpwqWa5rVAsV8'
+    url: 'https://gist.githubusercontent.com/wd15/c8422453c1237b4c7d82411a18a997db/raw/f2b198a5edf02afbf3be5fa5a6beef699e8d1f2f/EnergyN_dt_p001_c.csv'
     format:
       type: csv
       parse:
-        Time: number
-        "Total Energy": number
+        time: number
+        total_free_energy: number
     description: >-
       Here, the time, elastic, bulk, gradient, total free energy of the system
       are included in the data file in csv format.
     type: line
     transform:
       - type: formula
-        expr: datum.Time
+        expr: datum.time
         as: x
       - type: formula
-        expr: "datum['Total Energy']"
-        as: 'y'
+        expr: datum.total_free_energy
+        as: y
   - name: Energy_plot1
     url: 'https://drive.google.com/open?id=1xkeSuwNOdC_W8KC1XMkpsoKLtBUx-49S'
     type: image

_data/simulations/Linear_Elasticity_4e/meta.yaml

@@ -42,37 +42,30 @@ data:
       type: csv
       parse:
         x: number
-        'y': number
+        y: number
     description: >-
       Coordinate of the equilibrium contour plot of the order
       parameter (considering order parameter = 0.5) has been included
       in this data file.
     type: line
-    transform:
-      - type: formula
-        expr: datum.x
-        as: x
-      - type: formula
-        expr: datum.y
-        as: 'y'
   - name: all_data
     url: 'https://drive.google.com/open?id=1SBa_xeKx7dgKoPdKwo7zCezRUVRiFYAi'
     format:
       type: csv
       parse:
-        Time: number
-        "Total Energy": number
+        time: number
+        total_free_energy: number
     description: >-
       The Energy_data file includes time, elastic, bulk, gradient, total free
       energy of the system.
     type: line
-    transform:
-      - type: formula
-        expr: datum.Time
-        as: x
-      - type: formula
-        expr: "datum['Total Energy']"
-        as: 'y'
+    # transform:
+    #   - type: formula
+    #     expr: datum.Time
+    #     as: x
+    #   - type: formula
+    #     expr: "datum['Total Energy']"
+    #     as: 'y'
   - name: Energy_plot1
     url: 'https://drive.google.com/open?id=1Zo4Uret1Z-7-YI-v0Dcf_r87QzESc8Js'
     type: image

_data/simulations/Linear_Elasticity_4g/meta.yaml

@@ -42,36 +42,29 @@ data:
       type: csv
       parse:
         x: number
-        'y': number
+        y: number
     description: >-
       The data file contains the coordinate of the contour plot at equilibrium
       considering the the order parameter value=0.5
     type: line
-    transform:
-      - type: formula
-        expr: datum.x
-        as: x
-      - type: formula
-        expr: datum.y
-        as: 'y'
   - name: all_data
     url: 'https://drive.google.com/open?id=1MT00aGXdD38Ii_A7ASaEQm4y2Uv3AbRd'
     format:
       type: csv
       parse:
         Time: number
-        "Total Energy": number
+        total_free_energy: number
     description: >-
       Here, the time, elastic, bulk, gradient, total free energy of the system
       are included in the data file in csv format.
     type: line
-    transform:
-      - type: formula
-        expr: datum.Time
-        as: x
-      - type: formula
-        expr: "datum['Total Energy']"
-        as: 'y'
+    # transform:
+    #   - type: formula
+    #     expr: datum.time
+    #     as: x
+    #   - type: formula
+    #     expr: datum.total_free_energy
+    #     as: y
   - name: Energy_plot1
     url: 'https://drive.google.com/open?id=1M6nbZcgPTNfj9cl8gY3O2Y0IvrC44ZBL'
     type: image

_data/simulations/sfepy_4a_wd15/meta.yaml

@@ -15,7 +15,7 @@ data:
 - description: All data
   format:
     parse:
-      free_energy: number
+      total_free_energy: number
       time: number
     type: csv
   name: all_data

results/benchmark1a.1.md

@@ -13,17 +13,18 @@ jupyter:
     name: python3
 ---
 
-```python papermill={"duration": 0.015463, "end_time": "2023-03-07T17:20:02.384302", "exception": false, "start_time": "2023-03-07T17:20:02.368839", "status": "completed"} tags=["parameters"]
+```python papermill={"duration": 0.016968, "end_time": "2023-03-15T16:02:59.587869", "exception": false, "start_time": "2023-03-15T16:02:59.570901", "status": "completed"} tags=["parameters"]
 benchmark_id = '3a.1'
 line_plots = [
     dict(name='free_energy', layout=dict(log_y=True, x_label=r'<i>t</i>', y_label=r'&#8497;', range_y=[1.8e6, 2.4e6], title="Free Energy v Time")),
     dict(name='solid_fraction', layout=dict(log_y=True, x_label=r'<i>t</i>')),
     dict(name='tip_position', layout=dict(log_y=True, x_label=r'<i>t</i>')),
     dict(name='phase_field_1500', layout=dict(aspect_ratio=1.0))
 ]
+contour_plots = []
 ```
 
-```python papermill={"duration": 0.008466, "end_time": "2023-03-07T17:20:02.398145", "exception": false, "start_time": "2023-03-07T17:20:02.389679", "status": "completed"} tags=["injected-parameters"]
+```python papermill={"duration": 0.009966, "end_time": "2023-03-15T16:02:59.603056", "exception": false, "start_time": "2023-03-15T16:02:59.593090", "status": "completed"} tags=["injected-parameters"]
 # Parameters
 benchmark_id = "1a.1"
 line_plots = [
@@ -41,7 +42,7 @@ line_plots = [
 
 ```
 
-```python papermill={"duration": 0.013492, "end_time": "2023-03-07T17:20:02.416671", "exception": false, "start_time": "2023-03-07T17:20:02.403179", "status": "completed"} tags=[]
+```python papermill={"duration": 0.015563, "end_time": "2023-03-15T16:02:59.624039", "exception": false, "start_time": "2023-03-15T16:02:59.608476", "status": "completed"} tags=[]
 from IPython.display import display_markdown
 
 display_markdown(f'''
@@ -51,14 +52,14 @@ All results for the [{ benchmark_id } benchmark specification](../../benchmarks/
 ''', raw=True)
 ```
 
-```python papermill={"duration": 0.010644, "end_time": "2023-03-07T17:20:02.432084", "exception": false, "start_time": "2023-03-07T17:20:02.421440", "status": "completed"} tags=[]
+```python papermill={"duration": 0.007556, "end_time": "2023-03-15T16:02:59.634843", "exception": false, "start_time": "2023-03-15T16:02:59.627287", "status": "completed"} tags=[]
 # To generate the comparison notebooks use:
 # 
 # papermill template.ipynb benchmark{version}.ipynb -f bm{version}.yaml
 #
 ```
 
-```python papermill={"duration": 0.017121, "end_time": "2023-03-07T17:20:02.454101", "exception": false, "start_time": "2023-03-07T17:20:02.436980", "status": "completed"} tags=[]
+```python papermill={"duration": 0.018406, "end_time": "2023-03-15T16:02:59.655363", "exception": false, "start_time": "2023-03-15T16:02:59.636957", "status": "completed"} tags=[]
 from IPython.display import HTML
 
 HTML('''<script>
@@ -78,7 +79,7 @@ $( document ).ready(code_toggle);
 <form action="javascript:code_toggle()"><input type="submit" value="Code Toggle"></form>''')
 ```
 
-```python papermill={"duration": 0.611951, "end_time": "2023-03-07T17:20:03.071140", "exception": false, "start_time": "2023-03-07T17:20:02.459189", "status": "completed"} tags=[]
+```python papermill={"duration": 0.726823, "end_time": "2023-03-15T16:03:00.388367", "exception": false, "start_time": "2023-03-15T16:02:59.661544", "status": "completed"} tags=[]
 #from IPython.display import HTML, display
 #from time import sleep
 
@@ -108,42 +109,55 @@ from itables import init_notebook_mode
 init_notebook_mode(all_interactive=False)
 ```
 
-```python papermill={"duration": 3.416335, "end_time": "2023-03-07T17:20:06.489836", "exception": false, "start_time": "2023-03-07T17:20:03.073501", "status": "completed"} tags=[]
+```python papermill={"duration": 4.74188, "end_time": "2023-03-15T16:03:05.133406", "exception": false, "start_time": "2023-03-15T16:03:00.391526", "status": "completed"} tags=[]
 for x in line_plots:
     line_plot(
         data_name=x['name'],
         benchmark_id=benchmark_id,
         layout=x['layout'],
+        columns=x.get('columns', ('x', 'y'))
     ).show()
 ```
 
-```python papermill={"duration": 2.012773, "end_time": "2023-03-07T17:20:08.799224", "exception": false, "start_time": "2023-03-07T17:20:06.786451", "status": "completed"} tags=[]
+```python papermill={"duration": 0.545314, "end_time": "2023-03-15T16:03:06.056585", "exception": false, "start_time": "2023-03-15T16:03:05.511271", "status": "completed"} tags=[]
+for x in contour_plots:
+    data = get_result_data([x['name']], [benchmark_id], x['columns'])
+
+    levelset_plot(
+        data,
+        layout=x['layout'],
+        mask_func=lambda df: (x['mask_z'][0] < df.z) & (df.z < x['mask_z'][1]),
+        columns=x['columns']
+    ).show()
+```
+
+```python papermill={"duration": 3.160878, "end_time": "2023-03-15T16:03:09.619113", "exception": false, "start_time": "2023-03-15T16:03:06.458235", "status": "completed"} tags=[]
 efficiency_plot(benchmark_id).show()
 
 display_markdown("<span class='plotly-footnote' >* Wall time divided by the total simulated time.</span>", raw=True)
 
 ```
 
-```python papermill={"duration": 0.302543, "end_time": "2023-03-07T17:20:09.396002", "exception": false, "start_time": "2023-03-07T17:20:09.093459", "status": "completed"} tags=[]
+```python papermill={"duration": 0.482732, "end_time": "2023-03-15T16:03:10.519753", "exception": false, "start_time": "2023-03-15T16:03:10.037021", "status": "completed"} tags=[]
 display_markdown(f'''
 # Table of Results
 
 Table of { benchmark_id } benchmark result uploads.
 ''', raw=True)
 ```
 
-```python papermill={"duration": 0.299373, "end_time": "2023-03-07T17:20:09.994639", "exception": false, "start_time": "2023-03-07T17:20:09.695266", "status": "completed"} tags=[]
+```python papermill={"duration": 0.418729, "end_time": "2023-03-15T16:03:11.463587", "exception": false, "start_time": "2023-03-15T16:03:11.044858", "status": "completed"} tags=[]
 
 ```
 
-```python papermill={"duration": 1.189793, "end_time": "2023-03-07T17:20:11.482449", "exception": false, "start_time": "2023-03-07T17:20:10.292656", "status": "completed"} tags=[]
+```python papermill={"duration": 1.724703, "end_time": "2023-03-15T16:03:13.598615", "exception": false, "start_time": "2023-03-15T16:03:11.873912", "status": "completed"} tags=[]
 ## Currently switching off interactive tables as these are not converted to HTML properly.
 ## This might improve when jupyter-nbcovert is updated to a later version.
 
 init_notebook_mode(all_interactive=False)
 get_table_data_style(benchmark_id, pfhub_path='../..')
 ```
 
-```python papermill={"duration": 0.296122, "end_time": "2023-03-07T17:20:12.074308", "exception": false, "start_time": "2023-03-07T17:20:11.778186", "status": "completed"} tags=[]
+```python papermill={"duration": 0.3685, "end_time": "2023-03-15T16:03:14.337111", "exception": false, "start_time": "2023-03-15T16:03:13.968611", "status": "completed"} tags=[]
 
 ```