Support custom label dropdowns from the dynamic config

viz_scripts/bin/generate_plots.py

@@ -33,14 +33,21 @@
 else:
     dynamic_config = json.loads(r.text)
     print(f"Successfully downloaded config with version {dynamic_config['version']} "\
-        f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} "\
-        f"and data collection URL {dynamic_config['server']['connectUrl']}")
+        f"for {dynamic_config['intro']['translated_text']['en']['deployment_name']} ")
 
 if dynamic_config['intro']['program_or_study'] == 'program':
     mode_studied = dynamic_config['intro']['mode_studied']
 else:
     mode_studied = None
 
+# Check if the dynamic config contains dynamic labels 'label_options'
+# Passing the boolean flag is not enough, bcos we need to trace through the dynamic_config - pass label_options URL.
+if 'label_options' in dynamic_config:
+    has_dynamic_labels = True
+    dynamic_labels_url = dynamic_config['label_options']
+else:
+    has_dynamic_labels = False
+
 if args.date is None:
     start_date = arrow.get(int(dynamic_config['intro']['start_year']),
         int(dynamic_config['intro']['start_month']), 1)
@@ -69,8 +76,9 @@ def compute_for_date(month, year):
         program=args.program,
         study_type=dynamic_config['intro']['program_or_study'],
         mode_of_interest=mode_studied,
-        include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False))
-
+        include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
+        has_dynamic_labels = has_dynamic_labels,
+        dynamic_labels_url = dynamic_labels_url)
     print(f"Running at {arrow.get()} with params {params}")
 
     # Make a notebook object with these definitions

viz_scripts/docker/load_mongodump.sh

@@ -1,9 +1,9 @@
 MONGODUMP_FILE=$1
 
 echo "Copying file to docker container"
-docker cp $MONGODUMP_FILE em-public-dashboard_db_1:/tmp
+docker cp $MONGODUMP_FILE em-public-dashboard-db-1:/tmp
 
 FILE_NAME=`basename $MONGODUMP_FILE`
 
 echo "Restoring the dump from $FILE_NAME"
-docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard_db_1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'
+docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'

viz_scripts/generic_metrics.ipynb

@@ -30,7 +30,9 @@
     "program = \"default\"\n",
     "study_type = \"study\"\n",
     "mode_of_interest = None\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "has_dynamic_labels = True\n",
+    "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\"\n"
    ]
   },
   {
@@ -43,7 +45,7 @@
     "from collections import defaultdict\n",
     "\n",
     "import numpy as np\n",
-    "import pandas as pd\n",
+    "import pandas as pd4\n",
     "\n",
     "from plots import *\n",
     "import scaffolding\n",
@@ -88,6 +90,8 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            has_dynamic_labels,\n",
+    "                                                                            dynamic_labels_url,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)"
@@ -392,7 +396,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.12"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,

viz_scripts/mode_specific_metrics.ipynb

@@ -30,7 +30,9 @@
     "program = \"default\"\n",
     "study_type = \"program\"\n",
     "mode_of_interest = \"e-bike\"\n",
-    "include_test_users = False"
+    "include_test_users = False\n",
+    "has_dynamic_labels = False\n",
+    "dynamic_labels_url = \"https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/label_options/example-program-label-options.json\""
    ]
   },
   {
@@ -102,15 +104,25 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
+    "                                                                            has_dynamic_labels,\n",
+    "                                                                            dynamic_labels_url,\n",
     "                                                                            dic_re,\n",
     "                                                                            dic_pur=dic_pur,\n",
     "                                                                            include_test_users=include_test_users)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "surgical-continuity",
+   "metadata": {},
+   "source": [
+    "## Metrics for Specific Mode"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "77eedae6",
+   "id": "b0ee1b51",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -120,14 +132,6 @@
     "    debug_df.loc[f\"{mode_of_interest}_trips_with_replaced_mode\"] = scaffolding.trip_label_count(\"Replaced_mode\", mode_of_interest_df)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "surgical-continuity",
-   "metadata": {},
-   "source": [
-    "## Metrics for Specific Mode"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -347,6 +351,14 @@
     "    generate_missing_plot(plot_title_no_quality,debug_df,file_name)\n",
     "    alt_text = store_alt_text_missing(debug_df, file_name, plot_title_no_quality)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23789666",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -365,7 +377,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.12"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,

viz_scripts/scaffolding.py

@@ -1,5 +1,7 @@
 import pandas as pd
 import numpy as np
+import requests
+import json
 import sys
 
 import emission.storage.timeseries.abstract_timeseries as esta
@@ -106,7 +108,7 @@ def expand_userinputs(labeled_ct):
 unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0
 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0
 
-def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=None, include_test_users=False):
+def load_viz_notebook_data(year, month, program, study_type, has_dynamic_labels, dynamic_labels_url, dic_re, dic_pur=None, include_test_users=False):
     """ Inputs:
     year/month/program/study_type = parameters from the visualization notebook
     dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose
@@ -126,22 +128,59 @@ def load_viz_notebook_data(year, month, program, study_type, dic_re, dic_pur=Non
         unit_conversions(expanded_ct)
 
     # Mapping new mode labels with dictionaries
-    # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
-    if "mode_confirm" in expanded_ct.columns:
-        expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re)
-    if study_type == 'program':
-        # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
-        if 'replaced_mode' in expanded_ct.columns:
-            expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re)
+
+    # Extract translations_data for dynamic_labels
+    if has_dynamic_labels:
+        print("This json has dynamic label")
+        req = requests.get(dynamic_labels_url)
+        if req.status_code != 200:
+            print("Unable to download dynamic_labels")
         else:
-            print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping")
+            print("dynamic labels download was successful.")
+            dynamic_labels = json.loads(req.text)
+
+        # Extract translations section
+        translations_data = dynamic_labels["translations"]["en"]
+
+    # Mapping mode_confirm & replaced_mode to translations_data for Mode_confirm and Replaced_mode respectively.
+    if has_dynamic_labels:
+        print("This is a study, not expecting any replaced modes.")
+        if "mode_confirm" in expanded_ct.columns:
+            expanded_ct["Mode_confirm"] = expanded_ct["mode_confirm"].map(translations_data)
+        if study_type == 'program':
+            # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+            if 'replaced_mode' in expanded_ct.columns:
+                expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(translations_data)
+            else:
+                print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping")
+        else:
+                print("This is a study, not expecting any replaced modes.")
+    # Mapping mode_confirm & replaced_mode to dic_re for Mode_confirm and Replaced_mode respectively.
     else:
-            print("This is a study, not expecting any replaced modes.")
+        print("This is a study, not expecting any replaced modes.")
+        # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+        if "mode_confirm" in expanded_ct.columns:
+            expanded_ct['Mode_confirm']= expanded_ct['mode_confirm'].map(dic_re)
+        if study_type == 'program':
+            # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+            if 'replaced_mode' in expanded_ct.columns:
+                expanded_ct['Replaced_mode']= expanded_ct['replaced_mode'].map(dic_re)
+            else:
+                print("This is a program, but no replaced modes found. Likely cold start case. Ignoring replaced mode mapping")
+        else:
+                print("This is a study, not expecting any replaced modes.")
 
     # Trip purpose mapping
-    # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
-    if dic_pur is not None and "purpose_confirm" in expanded_ct.columns:
-        expanded_ct['Trip_purpose']= expanded_ct['purpose_confirm'].map(dic_pur)
+    # purpose_confirm mapping to translations_data for Trip_purpose
+    if has_dynamic_labels:
+        # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+        if dic_pur is not None and "purpose_confirm" in expanded_ct.columns:
+            expanded_ct['Trip_purpose']=expanded_ct['purpose_confirm'].map(translations_data)
+    # purpose_confirm mapping to dic_pur for Trip_purpose
+    else:
+        # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+        if dic_pur is not None and "purpose_confirm" in expanded_ct.columns:
+            expanded_ct['Trip_purpose']= expanded_ct['purpose_confirm'].map(dic_pur)
 
     # Document data quality
     file_suffix = get_file_suffix(year, month, program)