Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Addition of four new inequality filters #165

Merged
merged 5 commits into from
Oct 11, 2024
Merged

Addition of four new inequality filters #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bd966a6
Addition of four new filters.
kebwi Oct 8, 2024
93a8d01
Addition of four new filters.
kebwi Oct 8, 2024
84ecd7d
Addition of four new filters.
kebwi Oct 8, 2024
4a7ef79
Addition of four new filters.
kebwi Oct 8, 2024
8f701ca
Addition of four new filters.
kebwi Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion materializationengine/blueprints/client/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,27 @@ def post(
"tablename":{
"column_name":value
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"tablename": {
"column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
Expand Down Expand Up @@ -437,7 +458,27 @@ def post(
"tablename":{
"column_name":value
}
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"tablename":{
"column_name":[[min_x,min_y,minz], [max_x_max_y_max_z]]
Expand Down
117 changes: 114 additions & 3 deletions materializationengine/blueprints/client/api2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,10 @@ def apply_filters(df, user_data, column_names):
filter_in_dict = user_data.get("filter_in_dict", None)
filter_out_dict = user_data.get("filter_out_dict", None)
filter_equal_dict = user_data.get("filter_equal_dict", None)
filter_greater_dict = user_data.get("filter_greater_dict", None)
filter_less_dict = user_data.get("filter_less_dict", None)
filter_greater_equal_dict = user_data.get("filter_greater_equal_dict", None)
filter_less_equal_dict = user_data.get("filter_less_equal_dict", None)

if filter_in_dict:
for table, filter in filter_in_dict.items():
Expand All @@ -441,6 +445,26 @@ def apply_filters(df, user_data, column_names):
for col, val in filter.items():
colname = column_names[table][col]
df = df[df[colname] == val]
if filter_greater_dict:
for table, filter in filter_greater_dict.items():
for col, val in filter.items():
colname = column_names[table][col]
df = df[df[colname] > val]
if filter_less_dict:
for table, filter in filter_less_dict.items():
for col, val in filter.items():
colname = column_names[table][col]
df = df[df[colname] < val]
if filter_greater_equal_dict:
for table, filter in filter_greater_equal_dict.items():
for col, val in filter.items():
colname = column_names[table][col]
df = df[df[colname] >= val]
if filter_less_equal_dict:
for table, filter in filter_less_equal_dict.items():
for col, val in filter.items():
colname = column_names[table][col]
df = df[df[colname] <= val]
return df


Expand Down Expand Up @@ -922,10 +946,31 @@ def post(
"tablename":{
"column_name":value
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"tablename": {
"column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
}
},
"filter_regex_dict": {
"tablename": {
"column_name": "regex"
Expand Down Expand Up @@ -1388,6 +1433,26 @@ def post(
"column_name":value
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"tablename":{
"column_name":[[min_x,min_y,minz], [max_x_max_y_max_z]]
Expand Down Expand Up @@ -1602,10 +1667,31 @@ def post(self, datastack_name: str):
"table_name":{
"column_name":value
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"table_name": {
"column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
}
},
"filter_regex_dict":{
"table_name":{
"column_name": "regex"
Expand Down Expand Up @@ -1778,6 +1864,10 @@ def assemble_view_dataframe(datastack_name, version, view_name, data, args):
qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter)
qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter)
qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter)
qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter)
qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter)
qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
select_columns = data.get("select_columns", None)
Expand Down Expand Up @@ -1969,10 +2059,31 @@ def post(
"tablename":{
"column_name":value
}
},
"filter_greater_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_dict": {
"tablename":{
"column_name":value
}
},
"filter_greater_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_less_equal_dict": {
"tablename":{
"column_name":value
}
},
"filter_spatial_dict": {
"tablename": {
"column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
}
},
"filter_regex_dict": {
"tablename": {
"column_name": "regex"
Expand Down
8 changes: 8 additions & 0 deletions materializationengine/blueprints/client/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,10 @@ def generate_simple_query_dataframe(
qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter)
qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter)
qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter)
qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter)
qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter)
qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
qm.apply_filter({table_name: {"valid": True}}, qm.apply_equal_filter)
Expand Down Expand Up @@ -419,6 +423,10 @@ def generate_complex_query_dataframe(
qm.apply_filter(data.get("filter_in_dict", None), qm.apply_isin_filter)
qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
qm.apply_filter(data.get("filter_greater_dict", None), qm.apply_greater_filter)
qm.apply_filter(data.get("filter_less_dict", None), qm.apply_less_filter)
qm.apply_filter(data.get("filter_greater_equal_dict", None), qm.apply_greater_equal_filter)
qm.apply_filter(data.get("filter_less_equal_dict", None), qm.apply_less_equal_filter)
qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
for table_info in data["tables"]:
Expand Down
50 changes: 47 additions & 3 deletions materializationengine/blueprints/client/new_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ def strip_filter(filter):
strip_filter("filter_in_dict")
strip_filter("filter_out_dict")
strip_filter("filter_equal_dict")
strip_filter("filter_greater_dict")
strip_filter("filter_less_dict")
strip_filter("filter_greater_equal_dict")
strip_filter("filter_less_equal_dict")
return modified_user_data


Expand All @@ -123,14 +127,18 @@ def remap_query(user_data, mat_timestamp, cg_client, allow_invalid_root_ids=Fals
user_data.get("filter_in_dict", None),
user_data.get("filter_out_dict", None),
user_data.get("filter_equal_dict", None),
user_data.get("filter_greater_dict", None),
user_data.get("filter_less_dict", None),
user_data.get("filter_greater_equal_dict", None),
user_data.get("filter_less_equal_dict", None),
],
query_timestamp,
mat_timestamp,
cg_client,
allow_invalid_root_ids,
)

new_filter_in_dict, new_filter_out_dict, new_equal_dict = new_filters
new_filter_in_dict, new_filter_out_dict, new_equal_dict, new_greater_dict, new_less_dict, new_greater_equal_dict, new_less_equal_dict = new_filters
if new_equal_dict is not None:
if new_filter_in_dict is None:
new_filter_in_dict = defaultdict(lambda: None)
Expand All @@ -152,6 +160,10 @@ def remap_query(user_data, mat_timestamp, cg_client, allow_invalid_root_ids=Fals

modified_user_data = deepcopy(user_data)
modified_user_data["filter_equal_dict"] = new_equal_dict
modified_user_data["filter_greater_dict"] = new_greater_dict
modified_user_data["filter_less_dict"] = new_less_dict
modified_user_data["filter_greater_equal_dict"] = new_greater_equal_dict
modified_user_data["filter_less_equal_dict"] = new_less_equal_dict
modified_user_data["filter_in_dict"] = new_filter_in_dict
modified_user_data["filter_out_dict"] = new_filter_out_dict

Expand Down Expand Up @@ -339,6 +351,10 @@ def map_filters(
# filter_in_dict = data.get("filter_in_dict", None)
# filter_out_dict = data.get("filter_notin_dict", None)
# filter_equal_dict = data.get("filter_equal_dict", None)
# filter_greater_dict = data.get("filter_greater_dict", None)
# filter_less_dict = data.get("filter_less_dict", None)
# filter_greater_equal_dict = data.get("filter_greater_equal_dict", None)
# filter_less_equal_dict = data.get("filter_less_equal_dict", None)

# db = dynamic_annotation_cache.get_db(aligned_volume_name)
# table_metadata = db.database.get_table_metadata(table_name)
Expand All @@ -361,7 +377,7 @@ def map_filters(
# )

# past_filters, future_map = map_filters(
# [filter_in_dict, filter_out_dict, filter_equal_dict],
# [filter_in_dict, filter_out_dict, filter_equal_dict, filter_greater_dict, filter_less_dict, filter_greater_equal_dict, filter_less_equal_dict],
# timestamp,
# timestamp_start,
# cg_client,
Expand Down Expand Up @@ -412,6 +428,10 @@ def map_filters(
# filter_in_dict=past_filter_in_dict,
# filter_notin_dict=past_filter_out_dict,
# filter_equal_dict=None,
# filter_greater_dict=None,
# filter_less_dict=None,
# filter_greater_equal_dict=None,
# filter_less_equal_dict=None,
# filter_spatial=data.get("filter_spatial_dict", None),
# select_columns=data.get("select_columns", None),
# consolidate_positions=False,
Expand Down Expand Up @@ -485,6 +505,10 @@ def map_filters(
# data.get("filter_notin_dict", None), table_name
# ),
# filter_equal_dict=_format_filter(filter_equal_dict, table_name),
# filter_greater_dict=_format_filter(filter_greater_dict, table_name),
# filter_less_dict=_format_filter(filter_less_dict, table_name),
# filter_greater_equal_dict=_format_filter(filter_greater_equal_dict, table_name),
# filter_less_equal_dict=_format_filter(filter_less_equal_dict, table_name),
# filter_spatial=data.get("filter_spatial_dict", None),
# select_columns=data.get("select_columns", None),
# consolidate_positions=not args["split_positions"],
Expand Down Expand Up @@ -543,6 +567,22 @@ def map_filters(
# for col, val in filter_equal_dict.items():
# if col.endswith("root_id"):
# df = df[df[col] == val]
# if filter_greater_dict is not None:
# for col, val in filter_greater_dict.items():
# if col.endswith("root_id"):
# df = df[df[col] > val]
# if filter_less_dict is not None:
# for col, val in filter_less_dict.items():
# if col.endswith("root_id"):
# df = df[df[col] < val]
# if filter_greater_equal_dict is not None:
# for col, val in filter_greater_equal_dict.items():
# if col.endswith("root_id"):
# df = df[df[col] >= val]
# if filter_less_equal_dict is not None:
# for col, val in filter_less_equal_dict.items():
# if col.endswith("root_id"):
# df = df[df[col] <= val]

# now = time.time()
# headers = None
Expand Down Expand Up @@ -571,7 +611,7 @@ def map_filters(
# return after_request(response)

# def map_filters(
# [filter_in_dict, filter_out_dict, filter_equal_dict],
# [filter_in_dict, filter_out_dict, filter_equal_dict, filter_greater_dict, filter_less_dict, filter_greater_equal_dict, filter_less_equal_dict],
# timestamp,
# timestamp_start,
# cg_client,
Expand All @@ -586,6 +626,10 @@ def map_filters(
# select_columns=None,
# filter_in_dict=None,
# filter_equal_dict=None,
# filter_greater_dict=None,
# filter_less_dict=None,
# filter_greater_equal_dict=None,
# filter_less_equal_dict=None,
# filter_out_dict=None,
# filter_spatial_dict=None,
# offset=0,
Expand Down
Loading
Loading