diff --git a/materializationengine/blueprints/client/api.py b/materializationengine/blueprints/client/api.py index b18cdf32..42440a35 100644 --- a/materializationengine/blueprints/client/api.py +++ b/materializationengine/blueprints/client/api.py @@ -369,6 +369,27 @@ def post( "tablename":{ "column_name":value } + }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "tablename": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] @@ -437,7 +458,27 @@ def post( "tablename":{ "column_name":value } - } + }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "tablename":{ "column_name":[[min_x,min_y,minz], [max_x_max_y_max_z]] diff --git a/materializationengine/blueprints/client/api2.py b/materializationengine/blueprints/client/api2.py index 3b66a7e0..d488bea3 100644 --- a/materializationengine/blueprints/client/api2.py +++ b/materializationengine/blueprints/client/api2.py @@ -425,6 +425,10 @@ def apply_filters(df, user_data, column_names): filter_in_dict = user_data.get("filter_in_dict", None) filter_out_dict = user_data.get("filter_out_dict", None) filter_equal_dict = user_data.get("filter_equal_dict", None) + filter_greater_dict = user_data.get("filter_greater_dict", None) + filter_less_dict = user_data.get("filter_less_dict", None) + filter_greater_equal_dict = user_data.get("filter_greater_equal_dict", None) + filter_less_equal_dict = user_data.get("filter_less_equal_dict", None) if filter_in_dict: for table, filter in filter_in_dict.items(): @@ -441,6 +445,26 @@ def apply_filters(df, user_data, column_names): for col, val in filter.items(): colname = column_names[table][col] df = df[df[colname] == val] + if filter_greater_dict: + for table, filter in filter_greater_dict.items(): + for col, val in filter.items(): + colname = column_names[table][col] + df = df[df[colname] > val] + if filter_less_dict: + for table, filter in filter_less_dict.items(): + for col, val in filter.items(): + colname = column_names[table][col] + df = df[df[colname] < val] + if filter_greater_equal_dict: + for table, filter in filter_greater_equal_dict.items(): + for col, val in filter.items(): + colname = column_names[table][col] + df = df[df[colname] >= val] + if filter_less_equal_dict: + for table, filter in filter_less_equal_dict.items(): + for col, val in filter.items(): + colname = column_names[table][col] + df = df[df[colname] <= val] return df @@ -922,10 +946,31 @@ def post( "tablename":{ "column_name":value } + }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "tablename": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] - } + }, "filter_regex_dict": { "tablename": { "column_name": "regex" @@ -1388,6 +1433,26 @@ def post( "column_name":value } }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "tablename":{ "column_name":[[min_x,min_y,minz], [max_x_max_y_max_z]] @@ -1602,10 +1667,31 @@ def post(self, datastack_name: str): "table_name":{ "column_name":value } + }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "table_name": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] - } + }, "filter_regex_dict":{ "table_name":{ "column_name": "regex" @@ -1778,6 +1864,10 @@ def assemble_view_dataframe(datastack_name, version, view_name, data, args): qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter) qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter) qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter) + qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter) + qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter) + qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter) + qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter) qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter) qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter) select_columns = data.get("select_columns", None) @@ -1969,10 +2059,31 @@ def post( "tablename":{ "column_name":value } + }, + "filter_greater_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "tablename":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "tablename":{ + "column_name":value + } + }, "filter_spatial_dict": { "tablename": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] - } + }, "filter_regex_dict": { "tablename": { "column_name": "regex" diff --git a/materializationengine/blueprints/client/common.py b/materializationengine/blueprints/client/common.py index d04a8c87..75f13dc8 100644 --- a/materializationengine/blueprints/client/common.py +++ b/materializationengine/blueprints/client/common.py @@ -256,6 +256,10 @@ def generate_simple_query_dataframe( qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter) qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter) qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter) + qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter) + qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter) + qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter) + qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter) qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter) qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter) qm.apply_filter({table_name: {"valid": True}}, qm.apply_equal_filter) @@ -419,6 +423,10 @@ def generate_complex_query_dataframe( qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter) qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter) qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter) + qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter) + qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter) + qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter) + qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter) qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter) qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter) for table_info in data["tables"]: diff --git a/materializationengine/blueprints/client/new_query.py b/materializationengine/blueprints/client/new_query.py index 7a74342c..56764ce9 100644 --- a/materializationengine/blueprints/client/new_query.py +++ b/materializationengine/blueprints/client/new_query.py @@ -110,6 +110,10 @@ def strip_filter(filter): strip_filter("filter_in_dict") strip_filter("filter_out_dict") strip_filter("filter_equal_dict") + strip_filter("filter_greater_dict") + strip_filter("filter_less_dict") + strip_filter("filter_greater_equal_dict") + strip_filter("filter_less_equal_dict") return modified_user_data @@ -123,6 +127,10 @@ def remap_query(user_data, mat_timestamp, cg_client, allow_invalid_root_ids=Fals user_data.get("filter_in_dict", None), user_data.get("filter_out_dict", None), user_data.get("filter_equal_dict", None), + user_data.get("filter_greater_dict", None), + user_data.get("filter_less_dict", None), + user_data.get("filter_greater_equal_dict", None), + user_data.get("filter_less_equal_dict", None), ], query_timestamp, mat_timestamp, @@ -130,7 +138,7 @@ def remap_query(user_data, mat_timestamp, cg_client, allow_invalid_root_ids=Fals allow_invalid_root_ids, ) - new_filter_in_dict, new_filter_out_dict, new_equal_dict = new_filters + new_filter_in_dict, new_filter_out_dict, new_equal_dict, new_greater_dict, new_less_dict, new_greater_equal_dict, new_less_equal_dict = new_filters if new_equal_dict is not None: if new_filter_in_dict is None: new_filter_in_dict = defaultdict(lambda: None) @@ -152,6 +160,10 @@ def remap_query(user_data, mat_timestamp, cg_client, allow_invalid_root_ids=Fals modified_user_data = deepcopy(user_data) modified_user_data["filter_equal_dict"] = new_equal_dict + modified_user_data["filter_greater_dict"] = new_greater_dict + modified_user_data["filter_less_dict"] = new_less_dict + modified_user_data["filter_greater_equal_dict"] = new_greater_equal_dict + modified_user_data["filter_less_equal_dict"] = new_less_equal_dict modified_user_data["filter_in_dict"] = new_filter_in_dict modified_user_data["filter_out_dict"] = new_filter_out_dict @@ -339,6 +351,10 @@ def map_filters( # filter_in_dict = data.get("filter_in_dict", None) # filter_out_dict = data.get("filter_notin_dict", None) # filter_equal_dict = data.get("filter_equal_dict", None) +# filter_greater_dict = data.get("filter_greater_dict", None) +# filter_less_dict = data.get("filter_less_dict", None) +# filter_greater_equal_dict = data.get("filter_greater_equal_dict", None) +# filter_less_equal_dict = data.get("filter_less_equal_dict", None) # db = dynamic_annotation_cache.get_db(aligned_volume_name) # table_metadata = db.database.get_table_metadata(table_name) @@ -361,7 +377,7 @@ def map_filters( # ) # past_filters, future_map = map_filters( -# [filter_in_dict, filter_out_dict, filter_equal_dict], +# [filter_in_dict, filter_out_dict, filter_equal_dict, filter_greater_dict, filter_less_dict, filter_greater_equal_dict, filter_less_equal_dict], # timestamp, # timestamp_start, # cg_client, @@ -412,6 +428,10 @@ def map_filters( # filter_in_dict=past_filter_in_dict, # filter_notin_dict=past_filter_out_dict, # filter_equal_dict=None, +# filter_greater_dict=None, +# filter_less_dict=None, +# filter_greater_equal_dict=None, +# filter_less_equal_dict=None, # filter_spatial=data.get("filter_spatial_dict", None), # select_columns=data.get("select_columns", None), # consolidate_positions=False, @@ -485,6 +505,10 @@ def map_filters( # data.get("filter_notin_dict", None), table_name # ), # filter_equal_dict=_format_filter(filter_equal_dict, table_name), +# filter_greater_dict=_format_filter(filter_greater_dict, table_name), +# filter_less_dict=_format_filter(filter_less_dict, table_name), +# filter_greater_equal_dict=_format_filter(filter_greater_equal_dict, table_name), +# filter_less_equal_dict=_format_filter(filter_less_equal_dict, table_name), # filter_spatial=data.get("filter_spatial_dict", None), # select_columns=data.get("select_columns", None), # consolidate_positions=not args["split_positions"], @@ -543,6 +567,22 @@ def map_filters( # for col, val in filter_equal_dict.items(): # if col.endswith("root_id"): # df = df[df[col] == val] +# if filter_greater_dict is not None: +# for col, val in filter_greater_dict.items(): +# if col.endswith("root_id"): +# df = df[df[col] > val] +# if filter_less_dict is not None: +# for col, val in filter_less_dict.items(): +# if col.endswith("root_id"): +# df = df[df[col] < val] +# if filter_greater_equal_dict is not None: +# for col, val in filter_greater_equal_dict.items(): +# if col.endswith("root_id"): +# df = df[df[col] >= val] +# if filter_less_equal_dict is not None: +# for col, val in filter_less_equal_dict.items(): +# if col.endswith("root_id"): +# df = df[df[col] <= val] # now = time.time() # headers = None @@ -571,7 +611,7 @@ def map_filters( # return after_request(response) # def map_filters( -# [filter_in_dict, filter_out_dict, filter_equal_dict], +# [filter_in_dict, filter_out_dict, filter_equal_dict, filter_greater_dict, filter_less_dict, filter_greater_equal_dict, filter_less_equal_dict], # timestamp, # timestamp_start, # cg_client, @@ -586,6 +626,10 @@ def map_filters( # select_columns=None, # filter_in_dict=None, # filter_equal_dict=None, +# filter_greater_dict=None, +# filter_less_dict=None, +# filter_greater_equal_dict=None, +# filter_less_equal_dict=None, # filter_out_dict=None, # filter_spatial_dict=None, # offset=0, diff --git a/materializationengine/blueprints/client/query.py b/materializationengine/blueprints/client/query.py index 18edc0cf..3cc319d6 100644 --- a/materializationengine/blueprints/client/query.py +++ b/materializationengine/blueprints/client/query.py @@ -250,6 +250,10 @@ def specific_query( filter_in_dict=None, filter_notin_dict=None, filter_equal_dict=None, + filter_greater_dict=None, + filter_less_dict=None, + filter_greater_equal_dict=None, + filter_less_equal_dict=None, filter_spatial=None, select_columns=None, consolidate_positions=True, @@ -277,6 +281,18 @@ def specific_query( filter_equal_dict: dict of dicts outer layer: keys are table names inner layer: keys are column names, values are entries to be equal + filter_greater_dict: dict of dicts + outer layer: keys are table names + inner layer: keys are column names, values are entries to be exclusive upper-bound + filter_less_dict: dict of dicts + outer layer: keys are table names + inner layer: keys are column names, values are entries to be exclusive lower-bound + filter_greater_equal_dict: dict of dicts + outer layer: keys are table names + inner layer: keys are column names, values are entries to be inclusive upper-bound + filter_less_equal_dict: dict of dicts + outer layer: keys are table names + inner layer: keys are column names, values are entries to be inclusive lower-bound filter_spatial: dict of dicts outer layer: keys are table_namess inner layer: keys are column names, values are [min,max] as list of lists @@ -384,6 +400,34 @@ def specific_query( filter_args.append( (model_dict[filter_table].__dict__[column_name] == filter_value,) ) + if filter_greater_dict is not None: + for filter_table, filter_table_dict in filter_greater_dict.items(): + for column_name in filter_table_dict.keys(): + filter_value = filter_table_dict[column_name] + filter_args.append( + (model_dict[filter_table].__dict__[column_name] > filter_value,) + ) + if filter_less_dict is not None: + for filter_table, filter_table_dict in filter_less_dict.items(): + for column_name in filter_table_dict.keys(): + filter_value = filter_table_dict[column_name] + filter_args.append( + (model_dict[filter_table].__dict__[column_name] < filter_value,) + ) + if filter_greater_equal_dict is not None: + for filter_table, filter_table_dict in filter_greater_equal_dict.items(): + for column_name in filter_table_dict.keys(): + filter_value = filter_table_dict[column_name] + filter_args.append( + (model_dict[filter_table].__dict__[column_name] >= filter_value,) + ) + if filter_less_equal_dict is not None: + for filter_table, filter_table_dict in filter_less_equal_dict.items(): + for column_name in filter_table_dict.keys(): + filter_value = filter_table_dict[column_name] + filter_args.append( + (model_dict[filter_table].__dict__[column_name] <= filter_value,) + ) if filter_spatial is not None: for filter_table, filter_table_dict in filter_spatial.items(): diff --git a/materializationengine/blueprints/client/query_manager.py b/materializationengine/blueprints/client/query_manager.py index 8c4aa51d..824d3847 100644 --- a/materializationengine/blueprints/client/query_manager.py +++ b/materializationengine/blueprints/client/query_manager.py @@ -231,6 +231,30 @@ def apply_equal_filter(self, table_name, column_name, value): ) self._filters.append((get_column(model, column_name) == value,)) + def apply_greater_filter(self, table_name, column_name, value): + model = self._find_relevant_model( + table_name=table_name, column_name=column_name + ) + self._filters.append((get_column(model, column_name) > value,)) + + def apply_less_filter(self, table_name, column_name, value): + model = self._find_relevant_model( + table_name=table_name, column_name=column_name + ) + self._filters.append((get_column(model, column_name) < value,)) + + def apply_greater_equal_filter(self, table_name, column_name, value): + model = self._find_relevant_model( + table_name=table_name, column_name=column_name + ) + self._filters.append((get_column(model, column_name) >= value,)) + + def apply_less_equal_filter(self, table_name, column_name, value): + model = self._find_relevant_model( + table_name=table_name, column_name=column_name + ) + self._filters.append((get_column(model, column_name) <= value,)) + def apply_isin_filter(self, table_name, column_name, value): model = self._find_relevant_model( table_name=table_name, column_name=column_name @@ -361,10 +385,31 @@ def configure_query(self, user_data): "table_name":{ "column_name":value } + }, + "filter_greater_dict": { + "table_name":{ + "column_name":value + } + }, + "filter_less_dict": { + "table_name":{ + "column_name":value + } + }, + "filter_greater_equal_dict": { + "table_name":{ + "column_name":value + } + }, + "filter_less_equal_dict": { + "table_name":{ + "column_name":value + } + }, "filter_spatial_dict": { "table_name": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] - } + }, "filter_regex_dict":{ "table_name":{ "column_name":"regex" @@ -407,6 +452,18 @@ def apply_filter(filter_key, filter_func): self.apply_filter( user_data.get("filter_equal_dict", None), self.apply_equal_filter ) + self.apply_filter( + user_data.get("filter_greater_dict", None), self.apply_greater_filter + ) + self.apply_filter( + user_data.get("filter_less_dict", None), self.apply_less_filter + ) + self.apply_filter( + user_data.get("filter_greater_equal_dict", None), self.apply_greater_equal_filter + ) + self.apply_filter( + user_data.get("filter_less_equal_dict", None), self.apply_less_equal_filter + ) self.apply_filter( user_data.get("filter_spatial_dict", None), self.apply_spatial_filter ) diff --git a/materializationengine/blueprints/client/schemas.py b/materializationengine/blueprints/client/schemas.py index 223b252d..c63a9ed0 100644 --- a/materializationengine/blueprints/client/schemas.py +++ b/materializationengine/blueprints/client/schemas.py @@ -50,6 +50,10 @@ class V2QuerySchema(Schema): filter_in_dict = fields.Dict() filter_notin_dict = fields.Dict() filter_equal_dict = fields.Dict() + filter_greater_dict = fields.Dict() + filter_less_dict = fields.Dict() + filter_greater_equal_dict = fields.Dict() + filter_less_equal_dict = fields.Dict() filter_spatial_dict = fields.Dict() filter_regex_dict = fields.Dict(required=False) select_columns = fields.Dict() @@ -65,6 +69,10 @@ class SimpleQuerySchema(Schema): filter_in_dict = fields.Dict() filter_notin_dict = fields.Dict() filter_equal_dict = fields.Dict() + filter_greater_dict = fields.Dict() + filter_less_dict = fields.Dict() + filter_greater_equal_dict = fields.Dict() + filter_less_equal_dict = fields.Dict() filter_spatial_dict = fields.Dict() filter_regex_dict = fields.Dict(required=False) select_columns = fields.List(fields.Str) @@ -82,6 +90,10 @@ class ComplexQuerySchema(Schema): filter_in_dict = fields.Dict() filter_notin_dict = fields.Dict() filter_equal_dict = fields.Dict() + filter_greater_dict = fields.Dict() + filter_less_dict = fields.Dict() + filter_greater_equal_dict = fields.Dict() + filter_less_equal_dict = fields.Dict() filter_spatial_dict = fields.Dict() filter_regex_dict = fields.Dict(required=False) select_columns = fields.List(fields.Str)