Adding Survey Responses to Public Dashboard

frontend/index.html

@@ -344,6 +344,12 @@
       const end_year = date.getFullYear();
       var current_month = start_month;
       var current_year = start_year;
+
+      //testing dfc fermata .. doesn't start until April...
+      // if ((current_month >= end_month) && (current_year >= end_year)) {
+      //   current_month = current_month - 2; //dfc has not started yet...
+      // }
+
       dates.push([current_month, current_year]);
       while (!(current_month == end_month && current_year == end_year)) {
         current_month += 1;
@@ -356,6 +362,28 @@
       };
       return dates;
     };
+
+    function getDictionaryList(form_list) {
+      var quest_dict = {};
+        return new Promise(async (resolve) => {
+          for (i in form_list) {
+            response = await fetch(form_list[i]);
+            text = await response.text();
+            const parser = new DOMParser();
+            const doc = parser.parseFromString(text, "text/xml");
+            labels = doc.getElementsByTagName("label");
+              for (i in labels) {
+                try {
+                  if ((labels[i].parentNode.getAttribute("appearance") !== "label" && labels[i].parentNode.nodeName != "input")) //label type questions don't ever have answers
+                  {
+                    quest_dict[labels[i].parentNode.getAttribute("ref").split('/').slice(-1)] = labels[i].firstChild.data;
+                  }
+                } catch (e) { }
+              } 
+            }
+          resolve(quest_dict);
+        })
+    };
   </script>
 
   <script type="text/javascript">
@@ -407,7 +435,42 @@
           mode_studied = data.intro.mode_studied
           // Load list of plots corresponding to study/program
           dynamic_labels = data.label_options
-          if (data.intro.program_or_study == 'program') {
+          surveys = data.survey_info.surveys 
+          console.log(data.survey_info['trip-labels'])
+          if (data.survey_info['trip-labels'] === 'ENKETO') { //CASE: SURVEYS
+            survey_list = Object.keys(surveys)
+            survey_list = survey_list.filter(name => name !== 'UserProfileSurvey')
+
+            sheet_list = []
+            for (name in survey_list) {
+              form_path = data.survey_info.surveys[survey_list[name]].formPath;
+              //hard code the old survey
+              // form_path = 'https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/survey_resources/dfc-fermata/fermata-ev-return-trip-v0.xml';
+              //THIS ASSUMES THE FILENAME IS THE SAME AS THE FORM PATH BUT WITH xml FILE TYPE
+              l_path = form_path.split('.')
+              l_path.splice(l_path.length -1, 1, 'xml');
+              console.log(l_path);
+              sheet_path = l_path.join('.')
+              sheet_list.push(sheet_path)
+            }
+
+            getDictionaryList(sheet_list).then((quest_dict) => {
+              console.log(quest_dict);
+              load_file = "metrics_study_surveys.html"
+              $.get(load_file, function (file) {
+                Object.entries(quest_dict).forEach(([key, value]) => {
+                  var text = '<option ' + 'value="' + key + '" data-sizex="4" data-sizey="4">' + value + '</option>';
+                  file = file.concat('\n', text);
+                });
+                console.log("configuring units");
+                const unitConfigured = file.replaceAll("${data.display_config.use_imperial}", dist_units);
+                $('#metric').append(unitConfigured);
+                addPreconfiguredMetrics(Object.keys(quest_dict).slice(0, 5)); //only adding the first 6 elements
+              });
+            });         
+
+          }
+          else if (data.intro.program_or_study == 'program') { //CASE: PROGRAM
             // Note: We're disabling energy metrics on public dashboard when dynamic labels are available.
             // TODO: Remove the if (data.label_options) in future when energy computation is handled properly.
             if (dynamic_labels) {

frontend/metrics_study_surveys.html

@@ -0,0 +1,13 @@
+<!-- htmnl options should be 1 per chart question -->
+<!-- <option value="question key" data-sizex="4" data-sizey="4">translated question</option> -->
+
+<option value="ntrips_sensed_mode" data-sizex="4" data-sizey="4">Number of trips (sensed)</option>
+<option value="ntrips_under10miles_sensed_mode" data-sizex="4" data-sizey="4">Trip count under 80th Percentile (sensed)</option>
+<option value="miles_sensed_mode" data-sizex="4" data-sizey="4">Trip distance (${data.display_config.use_imperial}) by mode (sensed)</option>
+<option value="miles_sensed_mode_land" data-sizex="4" data-sizey="4">Trip distance by land mode (sensed)</option>
+<option value="average_miles_sensed_mode" data-sizex="6" data-sizey="4">Average trip length (${data.display_config.use_imperial}) (sensed)</option>
+<option value="ntrips_per_day" data-sizex="6" data-sizey="4">Trip frequency</option>
+<option value="ntrips_sensed_per_day" data-sizex="6" data-sizey="4">Trip frequency (sensed)</option>
+<option value="ntrips_per_weekday" data-sizex="6" data-sizey="4">Trip frequency (weekday)</option>
+<option value="ntrips_sensed_per_weekday" data-sizex="6" data-sizey="4">Trip frequency (weekday, sensed)</option>
+<option value="ts_users" data-sizex="8" data-sizey="2">Timeseries of active users</option>

viz_scripts/bin/generate_plots.py

@@ -90,7 +90,8 @@ def compute_for_date(month, year):
         include_test_users=dynamic_config.get('metrics', {}).get('include_test_users', False),
         dynamic_labels = dynamic_labels,
         use_imperial = dynamic_config.get('display_config', {}).get('use_imperial', True),
-        sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"))
+        sensed_algo_prefix=dynamic_config.get('metrics', {}).get('sensed_algo_prefix', "cleaned"),
+        survey_info = dynamic_config.get('survey_info', {}))
 
     print(f"Running at {arrow.get()} with params {params}")
 

viz_scripts/docker/crontab

@@ -5,5 +5,6 @@
 0 8 * * * python bin/generate_plots.py mode_specific_metrics.ipynb default >> /var/log/intake.stdinout 2>&1
 0 8 * * * python bin/generate_plots.py mode_specific_timeseries.ipynb default >> /var/log/intake.stdinout 2>&1
 0 8 * * * python bin/generate_plots.py energy_calculations.ipynb default >> /var/log/intake.stdinout 2>&1
+0 8 * * * python bin/generate_plots.py survey_responses.ipynb default >> /var/log/intake.stdinout 2>&1
 # For testing only
 # */5 * * * * python bin/generate_plots.py mode_purpose_share.ipynb default >> /var/log/intake.stdinout 2>&1

viz_scripts/docker/environment36.dashboard.additions.yml

@@ -4,6 +4,8 @@ channels:
 - defaults
 dependencies:
 - seaborn=0.11.1
+- git
 - pip:
   - nbparameterise==0.6
   - devcron==0.4
+  - git+https://github.com/JGreenlee/[email protected]

viz_scripts/plots.py

@@ -48,13 +48,15 @@ def merge_small_entries(labels, values):
     # We could have already had a non-zero other, and it could be small or large
     if "Other" not in v2l_df.index:
         # zero other will end up with misc_count
-        v2l_df.loc["Other"] = misc_count
+        if misc_count.vals > 0:
+            v2l_df.loc["Other"] = misc_count
     elif "Other" in small_chunk.index:
         # non-zero small other will already be in misc_count
         v2l_df.loc["Other"] = misc_count
     else:
         # non-zero large other, will not already be in misc_count
         v2l_df.loc["Other"] = v2l_df.loc["Other"] + misc_count
+
     disp.display(v2l_df)
 
     return (v2l_df.index.to_list(),v2l_df.vals.to_list(), v2l_df.pct.to_list())

viz_scripts/scaffolding.py

@@ -58,6 +58,13 @@ def load_all_confirmed_trips(tq):
     disp.display(all_ct.head())
     return all_ct
 
+def load_all_composite_trips(tq):
+    agg = esta.TimeSeries.get_aggregate_time_series()
+    all_ct = agg.get_data_df("analysis/composite_trip", tq)
+    print("Loaded all composite trips of length %s" % len(all_ct))
+    disp.display(all_ct.head())
+    return all_ct
+
 def load_all_participant_trips(program, tq, load_test_users):
     participant_list = get_participant_uuids(program, load_test_users)
     all_ct = load_all_confirmed_trips(tq)
@@ -69,6 +76,16 @@ def load_all_participant_trips(program, tq, load_test_users):
     disp.display(participant_ct_df.head())
     return participant_ct_df
 
+def filter_composite_trips(all_comp_trips, program, load_test_users):
+    participant_list = get_participant_uuids(program, load_test_users)
+    # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
+    if len(all_comp_trips) == 0:
+        return all_comp_trips
+    participant_ct_df = all_comp_trips[all_comp_trips.user_id.isin(participant_list)]
+    print("After filtering, found %s participant trips " % len(participant_ct_df))
+    disp.display(participant_ct_df.head())
+    return participant_ct_df
+
 def filter_labeled_trips(mixed_trip_df):
     # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867
     if len(mixed_trip_df) == 0:
@@ -214,6 +231,18 @@ def mapping_color_labels(dynamic_labels, dic_re, dic_pur):
 
     return colors_mode, colors_purpose, colors_sensed
 
+# Function: Maps survey answers to colors.
+# Input: dictionary of raw and translated survey answers
+# Output: Map for color with survey answers
+def mapping_color_surveys(dic_options):
+    dictionary_values = (list(OrderedDict.fromkeys(dic_options.values())))
+
+    colors = {}
+    for i in range(len(dictionary_values)):
+        colors[dictionary_values[i]] = plt.cm.tab10.colors[i%10]
+
+    return colors
+
 def load_viz_notebook_sensor_inference_data(year, month, program, include_test_users=False, sensed_algo_prefix="cleaned"):
     """ Inputs:
     year/month/program = parameters from the visualization notebook