From 359684f05f60c5ce7ec12fe49c12b87f34ee1ea8 Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Fri, 24 Jun 2016 11:48:16 -0700 Subject: [PATCH 1/4] Add support for multiple configs that are stored as RW documents For the pipeline, we just pick the max ts since all the configs are from the same user, and time on the phone is synchronous. So if we have seen a particular config override, we must have seen all prior values. --- emission/analysis/configs/config.py | 14 +++++++++++--- emission/analysis/configs/config_utils.py | 5 +++-- emission/analysis/configs/sync_config.py | 8 ++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 emission/analysis/configs/sync_config.py diff --git a/emission/analysis/configs/config.py b/emission/analysis/configs/config.py index a678d2d99..52480ff78 100644 --- a/emission/analysis/configs/config.py +++ b/emission/analysis/configs/config.py @@ -3,12 +3,20 @@ import emission.net.usercache.abstract_usercache as enua -config_list = ["sensor_config"] +config_list = ["sensor_config", "sync_config"] def save_all_configs(user_id, time_query): uc = enua.UserCache.getUserCache(user_id) - config_name = config_list[0] - return save_config(user_id, uc, time_query, config_name) + # This is tricky to extend to beyond one type of config because it is not + # clear which ts we should return. We cannot return two values because only + # one of them can be stored in our current pipeline states. We should be + # able to return the max of them, since these are all from the same user, + # so if we have received a later config, we know that we have processed all + # prior configs + # Let's test it out and see if it works! + last_processed_ts_list = [save_config(user_id, uc, time_query, config_name) + for config_name in config_list] + return max(last_processed_ts_list) def save_config(user_id, uc, time_query, module_name): config_fn = get_configurator("emission.analysis.configs.%s" % module_name) diff --git a/emission/analysis/configs/config_utils.py b/emission/analysis/configs/config_utils.py index 17d3d586c..8b95410aa 100644 --- a/emission/analysis/configs/config_utils.py +++ b/emission/analysis/configs/config_utils.py @@ -6,8 +6,9 @@ def get_last_entry(user_id, time_query, config_key): user_ts = esta.TimeSeries.get_time_series(user_id) - # get the max write_ts for this stream, which corresponds to the last entry - # We expect this to be small, unless users are continuously overriding values + # get the list of overrides for this time range. This should be non zero + # only if there has been an override since the last run, which needs to be + # saved back into the cache. config_overrides = list(user_ts.find_entries([config_key], time_query)) logging.debug("Found %d user overrides for user %s" % (len(config_overrides), user_id)) if len(config_overrides) == 0: diff --git a/emission/analysis/configs/sync_config.py b/emission/analysis/configs/sync_config.py new file mode 100644 index 000000000..2eec2ac0b --- /dev/null +++ b/emission/analysis/configs/sync_config.py @@ -0,0 +1,8 @@ +import logging +import emission.analysis.configs.config_utils as eacc + +def get_config(user_id, time_query): + # right now, we are not doing any server side overrides, so we pick the + # last user defined configuration for this user + SYNC_CONFIG_KEY = "config/sync_config" + return eacc.get_last_entry(user_id, time_query, SYNC_CONFIG_KEY) From ec4deda0354e63d2bed8cfa01433291e3354fcca Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Fri, 24 Jun 2016 11:57:06 -0700 Subject: [PATCH 2/4] Add a new entry for the syncconfig - new wrapper class - new formatters - new entry in entry Fairly straightforward now that we have a template --- emission/core/wrapper/entry.py | 1 + emission/core/wrapper/syncconfig.py | 13 +++++++++ .../formatters/android/sync_config.py | 29 +++++++++++++++++++ .../usercache/formatters/ios/sync_config.py | 26 +++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 emission/core/wrapper/syncconfig.py create mode 100644 emission/net/usercache/formatters/android/sync_config.py create mode 100644 emission/net/usercache/formatters/ios/sync_config.py diff --git a/emission/core/wrapper/entry.py b/emission/core/wrapper/entry.py index 267d86482..ee391d39d 100644 --- a/emission/core/wrapper/entry.py +++ b/emission/core/wrapper/entry.py @@ -31,6 +31,7 @@ def _getData2Wrapper(): "background/battery": "battery", "statemachine/transition": "transition", "config/sensor_config": "sensorconfig", + "config/sync_config": "syncconfig", "segmentation/raw_trip": "rawtrip", "segmentation/raw_place": "rawplace", "segmentation/raw_section": "section", diff --git a/emission/core/wrapper/syncconfig.py b/emission/core/wrapper/syncconfig.py new file mode 100644 index 000000000..bbf7e52ed --- /dev/null +++ b/emission/core/wrapper/syncconfig.py @@ -0,0 +1,13 @@ +import logging +import emission.core.wrapper.wrapperbase as ecwb + +class Syncconfig(ecwb.WrapperBase): + props = {"sync_interval": ecwb.WrapperBase.Access.RO, # the interval at which data is synced and the battery level is read + "ios_device_token": ecwb.WrapperBase.Access.RO} # device_token for ios, used for registering the device to the appropriate channel + enums = {} + geojson = [] + nullable = [] + local_dates = [] + + def _populateDependencies(self): + pass diff --git a/emission/net/usercache/formatters/android/sync_config.py b/emission/net/usercache/formatters/android/sync_config.py new file mode 100644 index 000000000..3accd8e87 --- /dev/null +++ b/emission/net/usercache/formatters/android/sync_config.py @@ -0,0 +1,29 @@ +import logging +import pytz +import attrdict as ad +import copy + +import emission.core.wrapper.syncconfig as ecws +import emission.net.usercache.formatters.common as fc + +# Currently, we just reflect this back to the user, so not much editing to do +# here. Since we get the timezone from javascript guessing, though, let's just +# verify that it is correct. +def format(entry): + formatted_entry = ad.AttrDict() + + metadata = entry.metadata + try: + valid_tz = pytz.timezone(entry.metadata.time_zone) + except UnknownTimeZoneError, e: + logging.warn("Got error %s while checking format validity" % e) + # Default timezone in for the Bay Area, which is probably a fairly safe + # assumption for now + metadata.time_zone = "America/Los_Angeles" + # adds the python datetime and fmt_time entries. important for future searches! + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + formatted_entry.data = entry.data + + return formatted_entry; diff --git a/emission/net/usercache/formatters/ios/sync_config.py b/emission/net/usercache/formatters/ios/sync_config.py new file mode 100644 index 000000000..6a2fc73c3 --- /dev/null +++ b/emission/net/usercache/formatters/ios/sync_config.py @@ -0,0 +1,26 @@ +import copy + +import emission.core.wrapper.syncconfig as ecws +import emission.net.usercache.formatters.common as fc + +# Currently, we just reflect this back to the user, so not much editing to do +# here. Since we get the timezone from javascript guessing, though, let's just +# verify that it is correct. +def format(entry): + formatted_entry = entry + + metadata = entry.metadata + try: + valid_tz = pytz.timezone(entry.metadata.time_zone) + except UnknownTimeZoneError, e: + logging.warn("Got error %s while checking format validity" % e) + # Default timezone in for the Bay Area, which is probably a fairly safe + # assumption for now + metadata.time_zone = "America/Los_Angeles" + # adds the python datetime and fmt_time entries. important for future searches! + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + formatted_entry.data = entry.data + + return formatted_entry; From b2310dc78a99a1e5c409b647c9ea4aadae71f9b2 Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Fri, 24 Jun 2016 14:28:43 -0700 Subject: [PATCH 3/4] One more place to add the new key --- emission/storage/timeseries/builtin_timeseries.py | 1 + 1 file changed, 1 insertion(+) diff --git a/emission/storage/timeseries/builtin_timeseries.py b/emission/storage/timeseries/builtin_timeseries.py index 5c9a0d996..8ab86c174 100644 --- a/emission/storage/timeseries/builtin_timeseries.py +++ b/emission/storage/timeseries/builtin_timeseries.py @@ -23,6 +23,7 @@ def __init__(self, user_id): "background/battery": self.timeseries_db, "statemachine/transition": self.timeseries_db, "config/sensor_config": self.timeseries_db, + "config/sync_config": self.timeseries_db, "segmentation/raw_trip": self.analysis_timeseries_db, "segmentation/raw_place": self.analysis_timeseries_db, "segmentation/raw_section": self.analysis_timeseries_db, From bd69ad0bf57876cef01cc8f7cdce49a301eb2444 Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Fri, 24 Jun 2016 17:03:33 -0700 Subject: [PATCH 4/4] Make the remote push script take in the interval as an argument We will use the interval as the channel --- bin/remotePush.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/remotePush.py b/bin/remotePush.py index fdc378369..4f9afa10d 100644 --- a/bin/remotePush.py +++ b/bin/remotePush.py @@ -1,11 +1,18 @@ import json,httplib +import sys config_data = json.load(open('conf/net/ext_service/parse.json')) +interval = sys.argv[1] +print "pushing for interval %s" % interval + silent_push_msg = { "where": { "deviceType": "ios" }, + "channels": [ + interval + ], "data": { # "alert": "The Mets scored! The game is now tied 1-1.", "content-available": 1, @@ -27,3 +34,4 @@ result = json.loads(connection.getresponse().read()) print result +