Skip to content

Commit

Permalink
Merge pull request #563 from shankari/ground_truth_matching
Browse files Browse the repository at this point in the history 
Match ground truth to existing trips
  • Loading branch information
@shankari
shankari authored Jan 16, 2018
2 parents 8a64611 + 61019a2 commit d38d4ed
Show file tree
Hide file tree
Showing 6 changed files with 817 additions and 3 deletions.
4 changes: 2 additions & 2 deletions emission/net/ext_service/geocoder/nominatim.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
googlemaps_key_file = open("conf/net/ext_service/googlemaps.json")
GOOGLE_MAPS_KEY = json.load(googlemaps_key_file)["api_key"]
except:
logging.warning("google maps key not configured, falling back to nominatim")
print("google maps key not configured, falling back to nominatim")

try:
nominatim_file = open("conf/net/ext_service/nominatim.json")
NOMINATIM_QUERY_URL = json.load(nominatim_file)["query_url"]
except:
logging.warning("nominatim not configured either, place decoding must happen on the client")
print("nominatim not configured either, place decoding must happen on the client")

class Geocoder(object):

Expand Down
21 changes: 21 additions & 0 deletions emission/storage/decorations/trip_queries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,24 @@ def get_stops_for_trip(key, user_id, trip_id):
"data.enter_ts", pymongo.ASCENDING)
return [ecwe.Entry(doc) for doc in stop_doc_cursor]

def get_user_input_for_trip(trip_key, user_id, trip_id, user_input_key):
ts = esta.TimeSeries.get_time_series(user_id)
trip_obj = ts.get_entry_from_id(trip_key, trip_id)
return get_user_input_for_trip_object(ts, trip_obj, user_input_key)

def get_user_input_for_trip_object(ts, trip_obj, user_input_key):
tq = estt.TimeQuery("data.start_ts", trip_obj.data.start_ts, trip_obj.data.end_ts)
# In general, all candiates will have the same start_ts, so no point in
# sorting by it. Only exception to general rule is when user first provides
# input before the pipeline is run, and then overwrites after pipeline is
# run
potential_candidates = ts.get_data_df(user_input_key, tq)
if len(potential_candidates) == 0:
return None

sorted_pc = potential_candidates.sort_values(by="metadata_write_ts")
most_recent_entry_id = potential_candidates._id.iloc[-1]
logging.debug("most recent entry has id %s" % most_recent_entry_id)
ret_val = ts.get_entry_from_id(user_input_key, most_recent_entry_id)
logging.debug("and is mapped to entry %s" % ret_val)
return ret_val
3 changes: 3 additions & 0 deletions emission/storage/timeseries/abstract_timeseries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ def find_entries(self, key_list=None, time_query=None, geo_query=None,
def get_entry_at_ts(self, key, ts_key, ts):
pass

def get_entry_from_id(self, key, entry_id):
pass

def get_data_df(self, key, time_query = None, geo_query=None, extra_query_list=None):
"""
Returns a dataframe of the specified entries. A single key is required,
Expand Down
277 changes: 277 additions & 0 deletions emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12.mode_confirm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,277 @@
[
{
"_id": {
"$oid": "5a5d7005c442d44402e67a21"
},
"user_id": {
"$uuid": "881137a35f07417a9c12530d2ceb23ea"
},
"metadata": {
"time_zone": "America/Los_Angeles",
"plugin": "none",
"write_ts": 1516072708.88161,
"platform": "ios",
"read_ts": 0,
"key": "manual/mode_confirm",
"type": "message",
"write_local_dt": {
"year": 2018,
"month": 1,
"day": 15,
"hour": 19,
"minute": 18,
"second": 28,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"write_fmt_time": "2018-01-15T19:18:28.881610-08:00"
},
"data": {
"end_ts": 1481563931.135,
"label": "bike",
"start_ts": 1481563177.681,
"start_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 9,
"minute": 19,
"second": 37,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"start_fmt_time": "2016-12-12T09:19:37.681000-08:00",
"end_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 9,
"minute": 32,
"second": 11,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"end_fmt_time": "2016-12-12T09:32:11.135000-08:00"
}
},
{
"_id": {
"$oid": "5a5d7005c442d44402e67a25"
},
"user_id": {
"$uuid": "881137a35f07417a9c12530d2ceb23ea"
},
"metadata": {
"time_zone": "America/Los_Angeles",
"plugin": "none",
"write_ts": 1516072722.571955,
"platform": "ios",
"read_ts": 0,
"key": "manual/mode_confirm",
"type": "message",
"write_local_dt": {
"year": 2018,
"month": 1,
"day": 15,
"hour": 19,
"minute": 18,
"second": 42,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"write_fmt_time": "2018-01-15T19:18:42.571955-08:00"
},
"data": {
"end_ts": 1481565307.256,
"label": "bike",
"start_ts": 1481564056.911,
"start_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 9,
"minute": 34,
"second": 16,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"start_fmt_time": "2016-12-12T09:34:16.911000-08:00",
"end_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 9,
"minute": 55,
"second": 7,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"end_fmt_time": "2016-12-12T09:55:07.256000-08:00"
}
},
{
"_id": {
"$oid": "5a5d7005c442d44402e67a29"
},
"user_id": {
"$uuid": "881137a35f07417a9c12530d2ceb23ea"
},
"metadata": {
"time_zone": "America/Los_Angeles",
"plugin": "none",
"write_ts": 1516072729.998598,
"platform": "ios",
"read_ts": 0,
"key": "manual/mode_confirm",
"type": "message",
"write_local_dt": {
"year": 2018,
"month": 1,
"day": 15,
"hour": 19,
"minute": 18,
"second": 49,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"write_fmt_time": "2018-01-15T19:18:49.998598-08:00"
},
"data": {
"end_ts": 1481593788.738,
"label": "bike",
"start_ts": 1481592524.076,
"start_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 17,
"minute": 28,
"second": 44,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"start_fmt_time": "2016-12-12T17:28:44.076000-08:00",
"end_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 17,
"minute": 49,
"second": 48,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"end_fmt_time": "2016-12-12T17:49:48.738000-08:00"
}
},
{
"_id": {
"$oid": "5a5d7005c442d44402e67a31"
},
"user_id": {
"$uuid": "881137a35f07417a9c12530d2ceb23ea"
},
"metadata": {
"time_zone": "America/Los_Angeles",
"plugin": "none",
"write_ts": 1516072761.494732,
"platform": "ios",
"read_ts": 0,
"key": "manual/mode_confirm",
"type": "message",
"write_local_dt": {
"year": 2018,
"month": 1,
"day": 15,
"hour": 19,
"minute": 19,
"second": 21,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"write_fmt_time": "2018-01-15T19:19:21.494732-08:00"
},
"data": {
"end_ts": 1481597197.113,
"label": "walk",
"start_ts": 1481594091.492,
"start_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 17,
"minute": 54,
"second": 51,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"start_fmt_time": "2016-12-12T17:54:51.492000-08:00",
"end_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 18,
"minute": 46,
"second": 37,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"end_fmt_time": "2016-12-12T18:46:37.113000-08:00"
}
},
{
"_id": {
"$oid": "5a5d7005c442d44402e67a35"
},
"user_id": {
"$uuid": "881137a35f07417a9c12530d2ceb23ea"
},
"metadata": {
"time_zone": "America/Los_Angeles",
"plugin": "none",
"write_ts": 1516072773.816434,
"platform": "ios",
"read_ts": 0,
"key": "manual/mode_confirm",
"type": "message",
"write_local_dt": {
"year": 2018,
"month": 1,
"day": 15,
"hour": 19,
"minute": 19,
"second": 33,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"write_fmt_time": "2018-01-15T19:19:33.816434-08:00"
},
"data": {
"end_ts": 1481601004.01,
"label": "bike",
"start_ts": 1481598124.35,
"start_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 19,
"minute": 2,
"second": 4,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"start_fmt_time": "2016-12-12T19:02:04.350000-08:00",
"end_local_dt": {
"year": 2016,
"month": 12,
"day": 12,
"hour": 19,
"minute": 50,
"second": 4,
"weekday": 0,
"timezone": "America/Los_Angeles"
},
"end_fmt_time": "2016-12-12T19:50:04.010000-08:00"
}
}
]
Loading

0 comments on commit d38d4ed

Please sign in to comment.