Resolved merge conflicts

.env

@@ -1 +1 @@
-SERVER_IMAGE_TAG=2024-10-06--46-23
+SERVER_IMAGE_TAG=2025-01-04--44-41

.env.repoTags

@@ -1 +1 @@
-FRONTEND_IMAGE_TAG=2024-10-05--13-20
+FRONTEND_IMAGE_TAG=2025-01-05--30-32

docker-compose.dev.yml

@@ -31,7 +31,7 @@ services:
       - STUDY_CONFIG=stage-program
     ports:
       # ipynb in numbers
-      - "47962:8888"
+      - "47962:47962"
     networks:
       - emission
     volumes:

docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - STUDY_CONFIG=stage-program
     ports:
       # ipynb in numbers
-      - "47962:8888"
+      - "47962:47962"
     networks:
       - emission
     volumes:

viz_scripts/bin/generate_plots.py

@@ -7,7 +7,8 @@
 import json
 import os
 import sys
-
+import asyncio
+import emcommon.util as emcu
 
 # Configuration settings to use for all generated plots by this instance
 # This could also be specified as a parser argument, if we want to generate plots for all programs from one instance
@@ -43,7 +44,11 @@
 
 # dynamic_labels can  be referenced from 
 # https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json
-dynamic_labels = { }
+labels = { }
+
+async def load_default_label_options():
+    labels = await emcu.read_json_resource("label-options.default.json")
+    return labels
 
 # Check if the dynamic config contains dynamic labels 'label_options'
 # Parse through the dynamic_labels_url:
@@ -54,10 +59,16 @@
     if req.status_code != 200:
         print(f"Unable to download dynamic_labels_url, status code: {req.status_code} for {STUDY_CONFIG}")
     else:
-        dynamic_labels = json.loads(req.text)
+        labels = json.loads(req.text)
         print(f"Dynamic labels download was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}" )
 else:
-    print(f"label_options is unavailable for the dynamic_config in {STUDY_CONFIG}")
+    # load default labels from e-mission-common
+    # https://raw.githubusercontent.com/JGreenlee/e-mission-common/refs/heads/master/src/emcommon/resources/label-options.default.json
+    labels = asyncio.run(load_default_label_options())
+    if not labels:
+        print(f"Unable to load labels for : {STUDY_CONFIG}")
+    else:
+        print(f"Labels loading was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}")
 
 if args.date is None:
     start_date = arrow.get(int(dynamic_config['intro']['start_year']),
@@ -88,7 +99,7 @@ def compute_for_date(month, year):
         study_type=dynamic_config['intro']['program_or_study'],
         mode_of_interest=mode_studied,
         include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
-        dynamic_labels = dynamic_labels,
+        labels = labels,
         use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True),
         sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"),
         bluetooth_only = dynamic_config.get('tracking', {}).get('bluetooth_only', False),

viz_scripts/docker/load_mongodump.sh

@@ -53,6 +53,10 @@ echo "Database Name: $DB_NAME"
 DB_HOST="mongodb://db/$DB_NAME"
 sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE"
 
+# Update the docker-compose configuration file with the actual STUDY_CONFIG
+STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-')
+sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE"
+
 echo "Updated docker-compose file:"
 cat "$CONFIG_FILE"
 

viz_scripts/docker/start_notebook.sh

@@ -30,7 +30,7 @@ cd saved-notebooks
 # tail -f /dev/null
 if [ -z ${CRON_MODE} ] ; then
     echo "Running notebook in docker, change host:port to localhost:47962 in the URL below"
-    PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=0.0.0.0 --allow-root
+    PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=0.0.0.0 --port=47962 --allow-root
 else
     echo "Running crontab without user interaction, setting python path"
     export PYTHONPATH=/usr/src/app

viz_scripts/energy_calculations.ipynb

@@ -32,7 +32,7 @@
     "mode_of_interest = \"e-bike\"\n",
     "include_test_users = False\n",
     "is_debug_mode = False\n",
-    "dynamic_labels = {}\n",
+    "labels = {}\n",
     "use_imperial = True"
    ]
   },
@@ -91,7 +91,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users,\n",
     "                                                                            add_footprint=True)\n",
     "\n",
@@ -224,9 +224,9 @@
     "    if (is_debug_mode == True):\n",
     "        ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n",
     "        if (weight_unit == 'kg'):\n",
-    "            scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n",
+    "            scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, labels)\n",
     "        else:\n",
-    "            scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n",
+    "            scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, labels)\n",
     "\n",
     "try:\n",
     "    if use_imperial:\n",

viz_scripts/generic_metrics.ipynb

@@ -30,7 +30,7 @@
     "program = \"default\"\n",
     "study_type = \"study\"\n",
     "include_test_users = False\n",
-    "dynamic_labels = {}\n",
+    "labels = {}\n",
     "use_imperial = True\n",
     "sensed_algo_prefix = \"cleaned\"\n",
     "survey_info = {}"
@@ -97,8 +97,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n",
-    "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)"
+    "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n",
+    "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = scaffolding.translate_values_to_labels(labels)"
    ]
   },
   {
@@ -120,7 +120,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users,\n",
     "                                                                            add_footprint=True)"
    ]
@@ -143,6 +143,7 @@
     "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users,\n",
     "                                                                            sensed_algo_prefix)"
    ]
@@ -166,7 +167,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users,\n",
     "                                                                            add_footprint=True)"
    ]
@@ -382,8 +383,8 @@
     "    expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n",
     "\n",
     "    sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n",
-    "    labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n",
-    "    inferred_u80_quality_text = f\"{len(expanded_ct_inferred_u80)} trips ({round(len(expanded_ct_inferred_u80)/len(expanded_ct_inferred)*100)}% of all inferred,\\n{round(len(expanded_ct_inferred_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_inferred_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct_inferred.columns else \"0 inferred trips\"\n",
+    "    labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"mode_confirm_w_other\" in expanded_ct.columns else \"0 labeled trips\"\n",
+    "    inferred_u80_quality_text = f\"{len(expanded_ct_inferred_u80)} trips ({round(len(expanded_ct_inferred_u80)/len(expanded_ct_inferred)*100)}% of all inferred,\\n{round(len(expanded_ct_inferred_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_inferred_u80)} {sensed_match.group(3)}\" if \"mode_confirm_w_other\" in expanded_ct_inferred.columns else \"0 inferred trips\"\n",
     "\n",
     "    # Plot entries\n",
     "    fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(15,3*2), sharex=True)\n",

viz_scripts/generic_metrics_sensed.ipynb

@@ -30,6 +30,7 @@
     "program = \"default\"\n",
     "study_type = \"study\"\n",
     "mode_of_interest = None\n",
+    "labels = {}\n",
     "include_test_users = False\n",
     "use_imperial = False\n",
     "sensed_algo_prefix = \"cleaned\"\n",
@@ -77,6 +78,7 @@
     "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users,\n",
     "                                                                            sensed_algo_prefix)"
    ]
@@ -96,7 +98,7 @@
     "        expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n",
     "        unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n",
     "        print(unique_keys)\n",
-    "        colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n",
+    "        colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n",
     "        colors_sensed = colors_ble\n",
     "except ValueError as e:\n",
     "    print(\"Got ValueError \", e)"
@@ -125,7 +127,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "plot_title_no_quality=\" Average Miles for each mode with > 3 entries\\n(inferred by OpenPATH from phone sensors)\"\n",
+    "plot_title_no_quality= \"Average \" + label_units + \" for each mode with > 3 entries\\n(inferred by OpenPATH from phone sensors)\"\n",
     "file_name ='average_miles_sensed_mode%s' % file_suffix\n",
     "\n",
     "try:\n",

viz_scripts/generic_timeseries.ipynb

@@ -28,7 +28,7 @@
     "study_type = \"study\"\n",
     "mode_of_interest = None\n",
     "include_test_users = False\n",
-    "dynamic_labels = {}\n",
+    "labels = {}\n",
     "use_imperial = False\n",
     "sensed_algo_prefix = \"cleaned\""
    ]
@@ -84,14 +84,15 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users,\n",
     "                                                                            add_footprint=True)\n",
     "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct\n",
     "\n",
     "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users,\n",
     "                                                                            sensed_algo_prefix)"
    ]
@@ -143,10 +144,6 @@
     "    expanded_ct_sensed.user_id = pd.Categorical(expanded_ct_sensed.user_id)\n",
     "    expanded_ct_sensed.date_time = pd.Categorical(expanded_ct_sensed.date_time)\n",
     "\n",
-    "if len(dynamic_labels) > 0:\n",
-    "    labels=dynamic_labels\n",
-    "else:\n",
-    "    labels = await emcu.read_json_resource(\"label-options.default.json\")\n",
     "dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n",
     "\n",
     "if \"Mode_confirm\" in expanded_ct.columns:\n",

viz_scripts/mode_specific_metrics.ipynb

@@ -31,7 +31,7 @@
     "study_type = \"program\"\n",
     "mode_of_interest = \"e-bike\"\n",
     "include_test_users = False\n",
-    "dynamic_labels = { }\n",
+    "labels = { }\n",
     "use_imperial = True"
    ]
   },
@@ -95,8 +95,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble  = await scaffolding.mapping_color_labels(dynamic_labels)\n",
-    "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)"
+    "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble  = scaffolding.mapping_color_labels(labels)\n",
+    "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = scaffolding.translate_values_to_labels(labels)"
    ]
   },
   {
@@ -118,7 +118,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users)"
    ]
   },
@@ -141,7 +141,7 @@
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
-    "                                                                            dynamic_labels,\n",
+    "                                                                            labels,\n",
     "                                                                            include_test_users=include_test_users)"
    ]
   },
@@ -265,8 +265,7 @@
     "                                    f\"Labeled `{mode_of_interest}` by user\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_purpose, debug_df, value_to_translations_purpose)\n",
     "    plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"purpose_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n",
     "                                    f\"Inferred `{mode_of_interest}` from prior labels\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_purpose, debug_df_inferred, value_to_translations_purpose)\n",
-    "    plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n",
-    "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
+    "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
     "    plt.clf()\n",
     "    generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n",
@@ -302,8 +301,7 @@
     "                                    \"Labeled by user\\n (Trip distance)\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_replaced, debug_df, value_to_translations_replaced)\n",
     "    plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
     "                                    \"Inferred from prior labels\\n (Trip distance)\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_replaced, debug_df_inferred, value_to_translations_replaced)\n",
-    "    plot_title = plot_title_no_quality\n",
-    "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
+    "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
     "    plt.clf()\n",
     "    generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n",
@@ -339,8 +337,7 @@
     "                                    f\"Labeled `{mode_of_interest}` by user\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_replaced, debug_df, value_to_translations_replaced)\n",
     "    plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
     "                                    f\"Inferred `{mode_of_interest}` from prior labels\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_replaced, debug_df_inferred, value_to_translations_replaced)\n",
-    "    plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n",
-    "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
+    "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
     "    plt.clf()\n",
     "    generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n",