From 02fb7adebfe88d2850f107aeaca4057f0aec33c1 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 26 Sep 2024 09:45:21 -0700 Subject: [PATCH 01/37] Update Mode_confirm to mode_confirm_w_other --- viz_scripts/scaffolding.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 880c8190..f9750709 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -170,8 +170,8 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label "Participants_with_at_least_one_trip": unique_users(participant_ct_df), "Participant_with_at_least_one_labeled_trip": unique_users(labeled_ct), "Trips_with_at_least_one_label": len(labeled_ct), - "Trips_with_mode_confirm_label": trip_label_count("Mode_confirm", expanded_ct), - "Trips_with_trip_purpose_label": trip_label_count("Trip_purpose", expanded_ct) + "Trips_with_mode_confirm_label": trip_label_count("mode_confirm_w_other", expanded_ct), + "Trips_with_trip_purpose_label": trip_label_count("purpose_confirm_w_other", expanded_ct) }, orient='index', columns=["value"]) @@ -254,8 +254,8 @@ async def load_viz_notebook_inferred_data(year, month, program, study_type, dyna "Participants_with_at_least_one_trip": unique_users(participant_ct_df), "Participant_with_at_least_one_inferred_trip": unique_users(inferred_ct), "Trips_with_at_least_one_inferred_label": len(inferred_ct), - "Trips_with_mode_confirm_inferred_label": trip_label_count("Mode_confirm", expanded_it), - "Trips_with_trip_purpose_inferred_label": trip_label_count("Trip_purpose", expanded_it) + "Trips_with_mode_confirm_inferred_label": trip_label_count("mode_confirm_w_other", expanded_it), + "Trips_with_trip_purpose_inferred_label": trip_label_count("purpose_confirm_w_other", expanded_it) }, orient='index', columns=["value"]) From 050c675d5a08be01d2957511b97c552d2d5225cc Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 26 Sep 2024 09:46:12 -0700 Subject: [PATCH 02/37] Update Mode_confirm to mode_confirm_w_other in generic_metrics notebook --- viz_scripts/generic_metrics.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 656601cc..f2db032e 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -384,8 +384,8 @@ " expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n", "\n", " sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n", - " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n", - " inferred_u80_quality_text = f\"{len(expanded_ct_inferred_u80)} trips ({round(len(expanded_ct_inferred_u80)/len(expanded_ct_inferred)*100)}% of all inferred,\\n{round(len(expanded_ct_inferred_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_inferred_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct_inferred.columns else \"0 inferred trips\"\n", + " labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"mode_confirm_w_other\" in expanded_ct.columns else \"0 labeled trips\"\n", + " inferred_u80_quality_text = f\"{len(expanded_ct_inferred_u80)} trips ({round(len(expanded_ct_inferred_u80)/len(expanded_ct_inferred)*100)}% of all inferred,\\n{round(len(expanded_ct_inferred_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_inferred_u80)} {sensed_match.group(3)}\" if \"mode_confirm_w_other\" in expanded_ct_inferred.columns else \"0 inferred trips\"\n", "\n", " # Plot entries\n", " fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(15,3*2), sharex=True)\n", From 683385486e015b3fa10108eccb484eca8bfdc571 Mon Sep 17 00:00:00 2001 From: iantei Date: Thu, 26 Sep 2024 09:51:06 -0700 Subject: [PATCH 03/37] Replace plot_title with plot_title_no_quality in set_title_and_save(). We display the relevant information adjacent to the bar labels. --- viz_scripts/mode_specific_metrics.ipynb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a1d362eb..ca823d8e 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -277,8 +277,7 @@ " f\"Labeled `{mode_of_interest}` by user\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_purpose, debug_df, value_to_translations_purpose)\n", " plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"purpose_confirm_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n", " f\"Labeled and Inferred `{mode_of_interest}` by OpenPATH\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_purpose, debug_df_inferred, value_to_translations_purpose)\n", - " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", - " set_title_and_save(fig, text_results, plot_title, file_name)\n", + " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", " generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n", @@ -314,8 +313,7 @@ " \"Labeled by user\\n (Trip distance)\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n", " \"Labeled and Inferred by OpenPATH\\n (Trip distance)\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_replaced, debug_df_inferred, value_to_translations_replaced)\n", - " plot_title = plot_title_no_quality\n", - " set_title_and_save(fig, text_results, plot_title, file_name)\n", + " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", " generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n", @@ -351,8 +349,7 @@ " f\"Labeled `{mode_of_interest}` by user\\n\"+stacked_bar_quality_text, ax[0], text_results[0], colors_replaced, debug_df, value_to_translations_replaced)\n", " plot_and_text_stacked_bar_chart(data_eb_inferred, lambda df: df.groupby(\"replaced_mode_w_other\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n", " f\"Labeled and Inferred `{mode_of_interest}` by OpenPATH\\n\"+stacked_bar_quality_text_inferred, ax[1], text_results[1], colors_replaced, debug_df_inferred, value_to_translations_replaced)\n", - " plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n", - " set_title_and_save(fig, text_results, plot_title, file_name)\n", + " set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n", "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n", " plt.clf()\n", " generate_missing_plot(plot_title_no_quality, debug_df, file_name)\n", From 0632b22fed227f5918dc7a14dbdf96b0f8b0ce9c Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 30 Sep 2024 09:41:58 -0700 Subject: [PATCH 04/37] Update the plot title for average miles per transport mode selected's bar chart in generic_metrics_sensed notebook. - Use appropriate metric/imperial unit instead of hardcoded Miles. --- viz_scripts/generic_metrics_sensed.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 9d4191fc..e29d4649 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -125,7 +125,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_title_no_quality=\" Average Miles for each mode with > 3 entries\\n(inferred by OpenPATH from phone sensors)\"\n", + "plot_title_no_quality= \"Average \" + label_units + \" for each mode with > 3 entries\\n(inferred by OpenPATH from phone sensors)\"\n", "file_name ='average_miles_sensed_mode%s' % file_suffix\n", "\n", "try:\n", From 2d810751513d5fc156c3cca9e56a3b15d4f46d5b Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 2 Oct 2024 13:08:44 -0700 Subject: [PATCH 05/37] Update the STUDY_CONFIG in docker-compose.dev.yml config file with the program/study name from the database name of the dataset. --- viz_scripts/docker/load_mongodump.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 64fc71a6..42f26b47 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -53,6 +53,9 @@ echo "Database Name: $DB_NAME" DB_HOST="mongodb://db/$DB_NAME" sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE" +STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-') +sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE" + echo "Updated docker-compose file:" cat "$CONFIG_FILE" From 242a7eae155a175e95b1d8534641fb9aaccdd257 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 2 Oct 2024 13:29:18 -0700 Subject: [PATCH 06/37] Update comment for updating docker-compose config file with actual STUDY_CONFIG --- viz_scripts/docker/load_mongodump.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 42f26b47..dc5da1bd 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -53,6 +53,7 @@ echo "Database Name: $DB_NAME" DB_HOST="mongodb://db/$DB_NAME" sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE" +# Update the docker-compose configuration file with the actual STUDY_CONFIG STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-') sed -i.bak "s|STUDY_CONFIG=.*|STUDY_CONFIG=$STUDY_CONFIG|" "$CONFIG_FILE" From 4c05924b781cddbe76c98535b3520b1ee1052379 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 11:37:34 -0700 Subject: [PATCH 07/37] Load read_json_resource() from e-mission-common - Pass labels instead of dynamic_labels to the notebooks - labels can either be: - Custom labels from the program deployment config, from nrel-openpath-deploy-configs - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource --- viz_scripts/bin/generate_plots.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/viz_scripts/bin/generate_plots.py b/viz_scripts/bin/generate_plots.py index 38d66c65..65815b27 100644 --- a/viz_scripts/bin/generate_plots.py +++ b/viz_scripts/bin/generate_plots.py @@ -7,7 +7,8 @@ import json import os import sys - +import asyncio +import emcommon.util as emcu # Configuration settings to use for all generated plots by this instance # This could also be specified as a parser argument, if we want to generate plots for all programs from one instance @@ -43,7 +44,11 @@ # dynamic_labels can be referenced from # https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json -dynamic_labels = { } +labels = { } + +async def load_default_label_options(): + labels = await emcu.read_json_resource("label-options.default.json") + return labels # Check if the dynamic config contains dynamic labels 'label_options' # Parse through the dynamic_labels_url: @@ -54,10 +59,16 @@ if req.status_code != 200: print(f"Unable to download dynamic_labels_url, status code: {req.status_code} for {STUDY_CONFIG}") else: - dynamic_labels = json.loads(req.text) + labels = json.loads(req.text) print(f"Dynamic labels download was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}" ) else: - print(f"label_options is unavailable for the dynamic_config in {STUDY_CONFIG}") + # load default labels from e-mission-common + # https://raw.githubusercontent.com/JGreenlee/e-mission-common/refs/heads/master/src/emcommon/resources/label-options.default.json + labels = asyncio.run(load_default_label_options()) + if not labels: + print(f"Unable to load labels for : {STUDY_CONFIG}") + else: + print(f"Labels loading was successful for nrel-openpath-deploy-configs: {STUDY_CONFIG}") if args.date is None: start_date = arrow.get(int(dynamic_config['intro']['start_year']), @@ -88,7 +99,7 @@ def compute_for_date(month, year): study_type=dynamic_config['intro']['program_or_study'], mode_of_interest=mode_studied, include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False), - dynamic_labels = dynamic_labels, + labels = labels, use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True), sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"), bluetooth_only = dynamic_config.get('tracking', {}).get('bluetooth_only', False), From 3b5b346f28de2b59f9a81b9735ea97531699ef85 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 12:56:24 -0700 Subject: [PATCH 08/37] Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py Replace dynamic_labels with "labels" - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common Remove async and await for mapping_color_labels() and translate_values_to_labels() Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. Update load_viz_notebook_* functions to use labels instead of dynamic_labels --- viz_scripts/scaffolding.py | 56 ++++++++++++-------------------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 3e46ebab..7d194247 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -54,9 +54,8 @@ def get_participant_uuids(program, load_test_users): disp.display(participant_list.user_email) return participant_uuid_str -async def add_base_mode_footprint(trip_list): +async def add_base_mode_footprint(trip_list, labels): #TODO filter ahead of this so only labeled trips get a footprint OR display uncertainties - labels = await emcu.read_json_resource("label-options.default.json") value_to_basemode = {mode["value"]: mode.get("base_mode", mode.get("baseMode", "UNKNOWN")) for mode in labels["MODE"]} for trip in trip_list: @@ -84,11 +83,11 @@ async def add_base_mode_footprint(trip_list): return trip_list -async def load_all_confirmed_trips(tq, add_footprint): +async def load_all_confirmed_trips(tq, labels, add_footprint): agg = esta.TimeSeries.get_aggregate_time_series() result_it = agg.find_entries(["analysis/confirmed_trip"], tq) if add_footprint: - processed_list = await add_base_mode_footprint(list(result_it)) + processed_list = await add_base_mode_footprint(list(result_it), labels) all_ct = agg.to_data_df("analysis/confirmed_trip", processed_list) else: all_ct = agg.to_data_df("analysis/confirmed_trip", result_it) @@ -96,9 +95,9 @@ async def load_all_confirmed_trips(tq, add_footprint): disp.display(all_ct.head()) return all_ct -async def load_all_participant_trips(program, tq, load_test_users, add_footprint=False): +async def load_all_participant_trips(program, tq, load_test_users, labels, add_footprint=False): participant_list = get_participant_uuids(program, load_test_users) - all_ct = await load_all_confirmed_trips(tq, add_footprint) + all_ct = await load_all_confirmed_trips(tq, labels, add_footprint) # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if len(all_ct) == 0: return all_ct @@ -177,7 +176,7 @@ def _select_max_label(row): unique_users = lambda df: len(df.user_id.unique()) if "user_id" in df.columns else 0 trip_label_count = lambda s, df: len(df[s].dropna()) if s in df.columns else 0 -async def load_viz_notebook_data(year, month, program, study_type, dynamic_labels, include_test_users=False, add_footprint=False): +async def load_viz_notebook_data(year, month, program, study_type, labels, include_test_users=False, add_footprint=False): #TODO - see how slow the loading the footprint is compared to just the baseMode, and evaluate if passing param around is needed """ Inputs: year/month/program/study_type = parameters from the visualization notebook @@ -187,11 +186,11 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, add_footprint) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, add_footprint) labeled_ct = filter_labeled_trips(participant_ct_df) expanded_ct = expand_userinputs(labeled_ct) expanded_ct = data_quality_check(expanded_ct) - expanded_ct = await map_trip_data(expanded_ct, study_type, dynamic_labels) + expanded_ct = map_trip_data(expanded_ct, study_type, labels) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -211,19 +210,12 @@ async def load_viz_notebook_data(year, month, program, study_type, dynamic_label return expanded_ct, file_suffix, quality_text, debug_df -async def map_trip_data(expanded_trip_df, study_type, dynamic_labels): +def map_trip_data(expanded_trip_df, study_type, labels): # Change meters to miles # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "distance" in expanded_trip_df.columns: unit_conversions(expanded_trip_df) - # Select the labels from dynamic_labels is available, - # else get it from emcommon/resources/label-options.default.json - if (len(dynamic_labels)): - labels = dynamic_labels - else: - labels = await emcu.read_json_resource("label-options.default.json") - # Map new mode labels with translations dictionary from dynamic_labels # CASE 2 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867 if "mode_confirm" in expanded_trip_df.columns: @@ -254,7 +246,7 @@ async def map_trip_data(expanded_trip_df, study_type, dynamic_labels): return expanded_trip_df -async def load_viz_notebook_inferred_data(year, month, program, study_type, dynamic_labels, include_test_users=False): +async def load_viz_notebook_inferred_data(year, month, program, study_type, labels, include_test_users=False): """ Inputs: year/month/program/study_type = parameters from the visualization notebook dic_* = label mappings; if dic_pur is included it will be used to recode trip purpose @@ -263,10 +255,10 @@ async def load_viz_notebook_inferred_data(year, month, program, study_type, dyna """ # Access database tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels) inferred_ct = filter_inferred_trips(participant_ct_df) expanded_it = expand_inferredlabels(inferred_ct) - expanded_it = await map_trip_data(expanded_it, study_type, dynamic_labels) + expanded_it = map_trip_data(expanded_it, study_type, labels) # Document data quality file_suffix = get_file_suffix(year, month, program) @@ -307,15 +299,9 @@ def translate_labels(labels): # Function: Maps "MODE", "PURPOSE", and "REPLACED_MODE" to colors. # Input: dynamic_labels # Output: Dictionary mapping between color with mode/purpose/sensed -async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): - # Load default options from e-mission-common - labels = await emcu.read_json_resource("label-options.default.json") +def mapping_color_labels(labels = {}, unique_keys = []): sensed_values = ["WALKING", "BICYCLING", "IN_VEHICLE", "AIR_OR_HSR", "UNKNOWN", "OTHER", "INVALID"] - # If dynamic_labels are provided, then we will use the dynamic labels for mapping - if len(dynamic_labels) > 0: - labels = dynamic_labels - # Load base mode values and purpose values mode_values = [mode["value"] for mode in labels["MODE"]] if "MODE" in labels else [] purpose_values = [mode["value"] for mode in labels["PURPOSE"]] if "PURPOSE" in labels else [] @@ -343,13 +329,7 @@ async def mapping_color_labels(dynamic_labels = {}, unique_keys = []): ], adjustment_range=[1,1.8]) return colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble -async def translate_values_to_labels(dynamic_labels, language="en"): - # Load default options from e-mission-common - labels = await emcu.read_json_resource("label-options.default.json") - - # If dynamic_labels are provided, then we will use the dynamic labels for mapping - if len(dynamic_labels) > 0: - labels = dynamic_labels +def translate_values_to_labels(labels, language="en"): # Mapping between values and translations for display on plots (for Mode) values_to_translations_mode = mapping_labels(labels, "MODE") # Mapping between values and translations for display on plots (for Purpose) @@ -373,14 +353,14 @@ def mapping_color_surveys(dic_options): return colors -async def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"): +async def load_viz_notebook_sensor_inference_data(year, month, program, labels, include_test_users=False, sensed_algo_prefix="cleaned"): """ Inputs: year/month/program = parameters from the visualization notebook Pipeline to load and process the data before use in sensor-based visualization notebooks. """ tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, False) expanded_ct = participant_ct_df print(f"Loaded expanded_ct with length {len(expanded_ct)} for {tq}") @@ -427,14 +407,14 @@ async def load_viz_notebook_sensor_inference_data(year, month, program, include_ return expanded_ct, file_suffix, quality_text, debug_df -async def load_viz_notebook_survey_data(year, month, program, include_test_users=False): +async def load_viz_notebook_survey_data(year, month, program, labels, include_test_users=False): """ Inputs: year/month/program/test users = parameters from the visualization notebook Returns: df of all trips taken by participants, df of all trips with user_input """ tq = get_time_query(year, month) - participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, False) + participant_ct_df = await load_all_participant_trips(program, tq, include_test_users, labels, False) labeled_ct = filter_labeled_trips(participant_ct_df) # Document data quality From 413420d41382d0308a9d2273130c8ad6af1bfc6b Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:00:27 -0700 Subject: [PATCH 09/37] Replace dynamic_labels with labels. Remove await from mapping_color_labels() and translate_values_to_labels(). --- viz_scripts/generic_metrics.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index b0992f16..370d6d78 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -30,7 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "sensed_algo_prefix = \"cleaned\"\n", "survey_info = {}" @@ -97,8 +97,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", - "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n", + "values_to_translations, value_to_translations_purpose, values_to_translations_replaced = scaffolding.translate_values_to_labels(labels)" ] }, { @@ -120,7 +120,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -165,7 +165,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, From db6797795b932d7f8202fa06d929ec30847c4d9d Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:02:37 -0700 Subject: [PATCH 10/37] Replace dynamic_labels with labels in energy_calculations notebook. --- viz_scripts/energy_calculations.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/viz_scripts/energy_calculations.ipynb b/viz_scripts/energy_calculations.ipynb index 79eccf19..569e1c5c 100644 --- a/viz_scripts/energy_calculations.ipynb +++ b/viz_scripts/energy_calculations.ipynb @@ -32,7 +32,7 @@ "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", "is_debug_mode = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True" ] }, @@ -91,7 +91,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "\n", @@ -224,9 +224,9 @@ " if (is_debug_mode == True):\n", " ebco2_debug = ebco2_compute(data_eb, weight_unit_debug)\n", " if (weight_unit == 'kg'):\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, dynamic_labels)\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2_debug, ebco2, labels)\n", " else:\n", - " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, dynamic_labels)\n", + " scaffolding.print_CO2_emission_calculations(data_eb, ebco2, ebco2_debug, labels)\n", "\n", "try:\n", " if use_imperial:\n", From ab70832f0abfe3295697fa83036255f636bf0f70 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:06:22 -0700 Subject: [PATCH 11/37] Replace dynamic_labels with labels in generic_timeseries notebook. Remove filter for dynamic_labels, since we pass either dynamic_labels or labels based on the availability of dynamic labels in program/study config. --- viz_scripts/generic_timeseries.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 54ab23aa..2171c9a3 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"" ] @@ -84,7 +84,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users,\n", " add_footprint=True)\n", "expanded_ct = scaffolding.unpack_energy_emissions(expanded_ct) if \"mode_confirm_footprint\" in expanded_ct.columns else expanded_ct\n", @@ -143,10 +143,6 @@ " expanded_ct_sensed.user_id = pd.Categorical(expanded_ct_sensed.user_id)\n", " expanded_ct_sensed.date_time = pd.Categorical(expanded_ct_sensed.date_time)\n", "\n", - "if len(dynamic_labels) > 0:\n", - " labels=dynamic_labels\n", - "else:\n", - " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", "dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", "\n", "if \"Mode_confirm\" in expanded_ct.columns:\n", From a050320740c23c82dd3805d1b3b2702f1e6b74c0 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:11:11 -0700 Subject: [PATCH 12/37] Replace dynamic_labels with labels in mode_specific_metrics notebook. Remove await for mapping_color_labels() and translate_values_to_labels(). --- viz_scripts/mode_specific_metrics.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/viz_scripts/mode_specific_metrics.ipynb b/viz_scripts/mode_specific_metrics.ipynb index a1656301..0753a60d 100644 --- a/viz_scripts/mode_specific_metrics.ipynb +++ b/viz_scripts/mode_specific_metrics.ipynb @@ -31,7 +31,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels = { }\n", + "labels = { }\n", "use_imperial = True" ] }, @@ -95,8 +95,8 @@ "metadata": {}, "outputs": [], "source": [ - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(dynamic_labels)\n", - "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.translate_values_to_labels(dynamic_labels)" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(labels)\n", + "values_to_translations, value_to_translations_purpose, value_to_translations_replaced = scaffolding.translate_values_to_labels(labels)" ] }, { @@ -118,7 +118,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -141,7 +141,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, From 595876dcf7a0802be9071f2fe4e81c1ca34caafa Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:13:17 -0700 Subject: [PATCH 13/37] Replace dynamic_labels with labels. Remove check for dynamic_labels since we pass dynamic labels or labels based on the availabilty in study/program's config file. --- viz_scripts/mode_specific_timeseries.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/viz_scripts/mode_specific_timeseries.ipynb b/viz_scripts/mode_specific_timeseries.ipynb index 2342951d..69af5ea6 100644 --- a/viz_scripts/mode_specific_timeseries.ipynb +++ b/viz_scripts/mode_specific_timeseries.ipynb @@ -28,7 +28,7 @@ "study_type = \"program\"\n", "mode_of_interest = \"e-bike\"\n", "include_test_users = False\n", - "dynamic_labels = { }\n", + "labels = { }\n", "use_imperial = False" ] }, @@ -94,7 +94,7 @@ " month,\n", " program,\n", " study_type,\n", - " dynamic_labels,\n", + " labels,\n", " include_test_users=include_test_users)" ] }, @@ -177,10 +177,6 @@ " mode_distance_interest = mode_distance[mode_distance['mode_confirm']==mode_of_interest].copy()\n", "\n", " # Mapping new mode labels with dictionaries\n", - " if len(dynamic_labels) > 0:\n", - " labels=dynamic_labels\n", - " else:\n", - " labels = await emcu.read_json_resource(\"label-options.default.json\")\n", " dic_mode_mapping = scaffolding.mapping_labels(labels, \"MODE\")\n", " mode_counts['Mode_confirm'] = mode_counts['mode_confirm'].map(dic_mode_mapping)\n", " mode_counts_interest['Mode_confirm'] = mode_counts_interest['mode_confirm'].map(dic_mode_mapping)\n", From 876bdc2b3d8e9b8d75c64a2d6156c33679b0ebec Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:15:13 -0700 Subject: [PATCH 14/37] Replace dynamic_labels with labels. Remove await from mapping_color_labels(). --- viz_scripts/survey_metrics.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index cf724a0a..2ff91b61 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -30,7 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "include_test_users = False\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "sensed_algo_prefix = \"cleaned\"\n", "bluetooth_only = True #current proxy for fleet status\n", @@ -61,7 +61,7 @@ "label_units, short_label, label_units_lower, distance_col, weight_unit = scaffolding.get_units(use_imperial)\n", "\n", "# get color mappings\n", - "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels() #just need sensed" + "colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels() #just need sensed" ] }, { @@ -114,7 +114,7 @@ " expanded_ct_sensed[\"primary_mode\"] = expanded_ct_sensed.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct_sensed.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys)\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys)\n", " colors_sensed = colors_ble\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" From 7815a10068e4971b8371d7924c2278b160e4c535 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:16:11 -0700 Subject: [PATCH 15/37] Replace dynamic_labels with labels. --- viz_scripts/survey_responses.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/survey_responses.ipynb b/viz_scripts/survey_responses.ipynb index 53a633cb..6b045cc7 100644 --- a/viz_scripts/survey_responses.ipynb +++ b/viz_scripts/survey_responses.ipynb @@ -13,7 +13,7 @@ "study_type = \"study\"\n", "mode_of_interest = None\n", "include_test_users = True\n", - "dynamic_labels = {}\n", + "labels = {}\n", "use_imperial = True\n", "survey_info = {}" ] From 2916df22e6906f43876812a80d5516ce36027443 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 13:20:09 -0700 Subject: [PATCH 16/37] Remove await from mapping_color_labels() as this is not an async function anymore. --- viz_scripts/generic_metrics_sensed.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index 9d4191fc..aff60da0 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -96,7 +96,7 @@ " expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n", " unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n", " print(unique_keys)\n", - " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = await scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n", + " colors_mode, colors_replaced, colors_purpose, colors_sensed, colors_ble = scaffolding.mapping_color_labels(unique_keys) #Extract ble color mapping\n", " colors_sensed = colors_ble\n", "except ValueError as e:\n", " print(\"Got ValueError \", e)" From 551c0cd359c42992f0ea107c04c7c48cda7dc4cc Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Thu, 3 Oct 2024 16:23:14 -0700 Subject: [PATCH 17/37] Add labels as the parameter for load_viz_notebook_sensor_inference_data() --- viz_scripts/bin/commit_message.txt | 6 ++++++ viz_scripts/commit_message.txt | 8 ++++++++ viz_scripts/generic_metrics.ipynb | 1 + viz_scripts/generic_metrics_sensed.ipynb | 2 ++ viz_scripts/generic_timeseries.ipynb | 1 + viz_scripts/survey_metrics.ipynb | 1 + 6 files changed, 19 insertions(+) create mode 100644 viz_scripts/bin/commit_message.txt create mode 100644 viz_scripts/commit_message.txt diff --git a/viz_scripts/bin/commit_message.txt b/viz_scripts/bin/commit_message.txt new file mode 100644 index 00000000..d458fabf --- /dev/null +++ b/viz_scripts/bin/commit_message.txt @@ -0,0 +1,6 @@ +Load read_json_resource() from e-mission-common + +- Pass labels instead of dynamic_labels to the notebooks +- labels can either be: + - Custom labels from the program deployment config, from nrel-openpath-deploy-configs + - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource \ No newline at end of file diff --git a/viz_scripts/commit_message.txt b/viz_scripts/commit_message.txt new file mode 100644 index 00000000..2171fd97 --- /dev/null +++ b/viz_scripts/commit_message.txt @@ -0,0 +1,8 @@ +Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py + +Replace dynamic_labels with "labels" + - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file + - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common +Remove async and await for mapping_color_labels() and translate_values_to_labels() + Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. +Update load_viz_notebook_* functions to use labels instead of dynamic_labels \ No newline at end of file diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 370d6d78..2cf6b18e 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -142,6 +142,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index aff60da0..a800c840 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -30,6 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", + "labels = {}\n", "include_test_users = False\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"\n", @@ -77,6 +78,7 @@ "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 2171c9a3..40d49d9e 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -92,6 +92,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 2ff91b61..cfa032d1 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -96,6 +96,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] From 76b4a129a0c92124f2e79439b2711d2f6315b65b Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Mon, 14 Oct 2024 12:27:10 -0700 Subject: [PATCH 18/37] Remove commit_message.txt files from the git commit. Add labels as the parameter for load_viz_notebook_sensor_inference_data(). --- viz_scripts/generic_metrics.ipynb | 1 + viz_scripts/generic_metrics_sensed.ipynb | 2 ++ viz_scripts/generic_timeseries.ipynb | 1 + viz_scripts/survey_metrics.ipynb | 1 + 4 files changed, 5 insertions(+) diff --git a/viz_scripts/generic_metrics.ipynb b/viz_scripts/generic_metrics.ipynb index 370d6d78..2cf6b18e 100644 --- a/viz_scripts/generic_metrics.ipynb +++ b/viz_scripts/generic_metrics.ipynb @@ -142,6 +142,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_metrics_sensed.ipynb b/viz_scripts/generic_metrics_sensed.ipynb index aff60da0..a800c840 100644 --- a/viz_scripts/generic_metrics_sensed.ipynb +++ b/viz_scripts/generic_metrics_sensed.ipynb @@ -30,6 +30,7 @@ "program = \"default\"\n", "study_type = \"study\"\n", "mode_of_interest = None\n", + "labels = {}\n", "include_test_users = False\n", "use_imperial = False\n", "sensed_algo_prefix = \"cleaned\"\n", @@ -77,6 +78,7 @@ "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/generic_timeseries.ipynb b/viz_scripts/generic_timeseries.ipynb index 2171c9a3..40d49d9e 100644 --- a/viz_scripts/generic_timeseries.ipynb +++ b/viz_scripts/generic_timeseries.ipynb @@ -92,6 +92,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] diff --git a/viz_scripts/survey_metrics.ipynb b/viz_scripts/survey_metrics.ipynb index 2ff91b61..cfa032d1 100644 --- a/viz_scripts/survey_metrics.ipynb +++ b/viz_scripts/survey_metrics.ipynb @@ -96,6 +96,7 @@ "expanded_ct_sensed, file_suffix_sensed, quality_text_sensed, debug_df_sensed = await scaffolding.load_viz_notebook_sensor_inference_data(year,\n", " month,\n", " program,\n", + " labels,\n", " include_test_users,\n", " sensed_algo_prefix)" ] From 74708a8949dd84c7ed22e7acc4022be8283d0e77 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Mon, 14 Oct 2024 12:34:25 -0700 Subject: [PATCH 19/37] Remove commit_message.txt files --- viz_scripts/bin/commit_message.txt | 6 ------ viz_scripts/commit_message.txt | 8 -------- 2 files changed, 14 deletions(-) delete mode 100644 viz_scripts/bin/commit_message.txt delete mode 100644 viz_scripts/commit_message.txt diff --git a/viz_scripts/bin/commit_message.txt b/viz_scripts/bin/commit_message.txt deleted file mode 100644 index d458fabf..00000000 --- a/viz_scripts/bin/commit_message.txt +++ /dev/null @@ -1,6 +0,0 @@ -Load read_json_resource() from e-mission-common - -- Pass labels instead of dynamic_labels to the notebooks -- labels can either be: - - Custom labels from the program deployment config, from nrel-openpath-deploy-configs - - In case, we do not have custom labels for the program/study, load default labels from e-mission-common with read_json_resource \ No newline at end of file diff --git a/viz_scripts/commit_message.txt b/viz_scripts/commit_message.txt deleted file mode 100644 index 2171fd97..00000000 --- a/viz_scripts/commit_message.txt +++ /dev/null @@ -1,8 +0,0 @@ -Remove call to read_json_resource("label-options.default.json") from e-mission-common in scaffolding.py - -Replace dynamic_labels with "labels" - - labels could be either default labels or dynamic labels (provided for program/study) depending on availability in config file - - Use labels passed from notebook instead of calling read_json_resource("label-options.default.json") from e-mission-common -Remove async and await for mapping_color_labels() and translate_values_to_labels() - Remove filtering for dynamic_labels availability, as we will only have either labels or dynamic_labels. -Update load_viz_notebook_* functions to use labels instead of dynamic_labels \ No newline at end of file From a4fe197af5e6689828d79860b2abbf1ce2567115 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Tue, 15 Oct 2024 19:50:36 -0400 Subject: [PATCH 20/37] Set 47962 port --- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- viz_scripts/docker/start_notebook.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1906101e..560f6d99 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -31,7 +31,7 @@ services: - STUDY_CONFIG=stage-program ports: # ipynb in numbers - - "47962:8888" + - "47962:47962" networks: - emission volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 38a975ae..c3058476 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - STUDY_CONFIG=stage-program ports: # ipynb in numbers - - "47962:8888" + - "47962:47962" networks: - emission volumes: diff --git a/viz_scripts/docker/start_notebook.sh b/viz_scripts/docker/start_notebook.sh index c6d1ad6d..e0deb7e7 100755 --- a/viz_scripts/docker/start_notebook.sh +++ b/viz_scripts/docker/start_notebook.sh @@ -30,7 +30,7 @@ cd saved-notebooks # tail -f /dev/null if [ -z ${CRON_MODE} ] ; then echo "Running notebook in docker, change host:port to localhost:47962 in the URL below" - PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=0.0.0.0 --allow-root + PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=0.0.0.0 --port=47962 --allow-root else echo "Running crontab without user interaction, setting python path" export PYTHONPATH=/usr/src/app From f3baa420f619213aecaa19feb611c7b1872f4b30 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Wed, 16 Oct 2024 00:37:13 +0000 Subject: [PATCH 21/37] Updated docker image tags in .env files to the latest timestamps --- .env.repoTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.repoTags b/.env.repoTags index 8dae8aa5..6054c295 100644 --- a/.env.repoTags +++ b/.env.repoTags @@ -1 +1 @@ -FRONTEND_IMAGE_TAG=2024-10-05--13-20 +FRONTEND_IMAGE_TAG=2024-10-16--33-46 From e09289c42b404554aecfb16abf453cffd53b34ed Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Fri, 18 Oct 2024 18:34:56 +0000 Subject: [PATCH 22/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index a4704b35..3e8de284 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-10-06--46-23 +SERVER_IMAGE_TAG=2024-10-18--26-35 From d3def638d2452180c14fef0442da71f2cc762dee Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Fri, 18 Oct 2024 19:46:07 +0000 Subject: [PATCH 23/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 3e8de284..4a0b0141 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-10-18--26-35 +SERVER_IMAGE_TAG=2024-10-18--38-04 From c7a3cd1c7595425d9de53de22f5e5e894e4259fb Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Mon, 4 Nov 2024 07:09:11 +0000 Subject: [PATCH 24/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 4a0b0141..8f0cf2f8 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-10-18--38-04 +SERVER_IMAGE_TAG=2024-11-04--01-28 From 4efc537fe087afa96c1e06ca4226b72308f8dc06 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Tue, 5 Nov 2024 23:49:11 +0000 Subject: [PATCH 25/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8f0cf2f8..59c46d79 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-11-04--01-28 +SERVER_IMAGE_TAG=2024-11-05--41-12 From 24f5f043f0cf8e8052b435c8a05cbd98dba49862 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 9 Nov 2024 00:58:18 +0000 Subject: [PATCH 26/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 59c46d79..de4d0177 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-11-05--41-12 +SERVER_IMAGE_TAG=2024-11-09--50-29 From 3a7c514e51011ccb4898ac47341bc7bc38b6c79a Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 21 Dec 2024 17:15:05 +0000 Subject: [PATCH 27/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index de4d0177..e5d34691 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-11-09--50-29 +SERVER_IMAGE_TAG=2024-12-21--07-17 From e1793446d94854b9057de1eca5fb26722e7dd983 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 21 Dec 2024 21:29:02 +0000 Subject: [PATCH 28/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index e5d34691..20340f76 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-12-21--07-17 +SERVER_IMAGE_TAG=2024-12-21--21-04 From 828513bf663360725462194c42891822051249d6 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 21 Dec 2024 21:59:01 +0000 Subject: [PATCH 29/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 20340f76..9eacf2ce 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-12-21--21-04 +SERVER_IMAGE_TAG=2024-12-21--51-11 From e4d7dd9087048b83babf5e09abb6b80a822ae708 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 22 Dec 2024 01:41:14 +0000 Subject: [PATCH 30/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 9eacf2ce..b9ca7347 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-12-21--51-11 +SERVER_IMAGE_TAG=2024-12-22--33-07 From 6cc0c37d40af22a186c7e043c2418f4bd6dce1d4 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 22 Dec 2024 04:14:45 +0000 Subject: [PATCH 31/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b9ca7347..4b1c3dac 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-12-22--33-07 +SERVER_IMAGE_TAG=2024-12-22--06-47 From a189e6ad6bab95067c67bfd17def5eb8d1d791c4 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 4 Jan 2025 07:13:50 +0000 Subject: [PATCH 32/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 4b1c3dac..3bcfdb39 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2024-12-22--06-47 +SERVER_IMAGE_TAG=2025-01-04--06-03 From e199ac89149f938f582f5b80d60806bf04b1a527 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sat, 4 Jan 2025 23:52:31 +0000 Subject: [PATCH 33/37] Updated docker image tags in .env files to the latest timestamps --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 3bcfdb39..ce941426 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -SERVER_IMAGE_TAG=2025-01-04--06-03 +SERVER_IMAGE_TAG=2025-01-04--44-41 From 7f54f31892f3954bc0f630c07ed9081cf09582cf Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 5 Jan 2025 00:01:40 +0000 Subject: [PATCH 34/37] Updated docker image tags in .env files to the latest timestamps --- .env.repoTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.repoTags b/.env.repoTags index 6054c295..e94ea2fb 100644 --- a/.env.repoTags +++ b/.env.repoTags @@ -1 +1 @@ -FRONTEND_IMAGE_TAG=2024-10-16--33-46 +FRONTEND_IMAGE_TAG=2025-01-04--58-25 From f927f36101909a1641da06f8a4bbda424446a5f5 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 5 Jan 2025 00:18:25 +0000 Subject: [PATCH 35/37] Updated docker image tags in .env files to the latest timestamps --- .env.repoTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.repoTags b/.env.repoTags index e94ea2fb..5e42a075 100644 --- a/.env.repoTags +++ b/.env.repoTags @@ -1 +1 @@ -FRONTEND_IMAGE_TAG=2025-01-04--58-25 +FRONTEND_IMAGE_TAG=2025-01-05--15-04 From b3aab2b70449a3603faa02e54f3841b7cfc40f6a Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 5 Jan 2025 00:57:47 +0000 Subject: [PATCH 36/37] Updated docker image tags in .env files to the latest timestamps --- .env.repoTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.repoTags b/.env.repoTags index 5e42a075..787e3156 100644 --- a/.env.repoTags +++ b/.env.repoTags @@ -1 +1 @@ -FRONTEND_IMAGE_TAG=2025-01-05--15-04 +FRONTEND_IMAGE_TAG=2025-01-05--54-27 From 045797183928b2b89f2dd267c90af38f06faff99 Mon Sep 17 00:00:00 2001 From: "Github Actions bot to update .env with latest tags" Date: Sun, 5 Jan 2025 03:33:45 +0000 Subject: [PATCH 37/37] Updated docker image tags in .env files to the latest timestamps --- .env.repoTags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.repoTags b/.env.repoTags index 787e3156..20ad5af7 100644 --- a/.env.repoTags +++ b/.env.repoTags @@ -1 +1 @@ -FRONTEND_IMAGE_TAG=2025-01-05--54-27 +FRONTEND_IMAGE_TAG=2025-01-05--30-32