adding regex_dict options

CAVEconnectome · Sep 19, 2023 · bf00bf9 · bf00bf9
1 parent 7514163
commit bf00bf9
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 17 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -11,6 +11,8 @@ tag = True
 
 [bumpversion:file:materializationengine/blueprints/materialize/api.py]
 
+[bumpversion:file:materializationengine/blueprints/materialize/api2.py]
+
 [bumpversion:file:docs/conf.py]
 search = release = "{current_version}"
 replace = release = "{new_version}"
diff --git a/materializationengine/blueprints/client/api2.py b/materializationengine/blueprints/client/api2.py
@@ -5,6 +5,15 @@
 from flask import abort, request, current_app, g
 from flask_accepts import accepts
 from flask_restx import Namespace, Resource, inputs, reqparse
+from middle_auth_client import (
+    auth_requires_permission,
+)
+import pandas as pd
+import datetime
+from typing import List
+import werkzeug
+from sqlalchemy.sql.sqltypes import String, Integer, Float, DateTime, Boolean, Numeric
+from geoalchemy2.types import Geometry
 from materializationengine.blueprints.client.datastack import validate_datastack
 from materializationengine.blueprints.client.new_query import (
     remap_query,
@@ -41,16 +50,10 @@
 )
 from materializationengine.info_client import get_aligned_volumes, get_datastack_info
 from materializationengine.schemas import AnalysisTableSchema, AnalysisVersionSchema
-from middle_auth_client import (
-    auth_requires_permission,
-)
 from materializationengine.blueprints.client.utils import update_notice_text_warnings
-import pandas as pd
-import datetime
-from typing import List
-import werkzeug
 
-__version__ = "4.0.20"
+
+__version__ = "4.18.0"
 
 
 authorizations = {
@@ -276,7 +279,7 @@ def execute_materialized_query(
         .scalar()
     )
     if random_sample is not None:
-        random_sample = (100.0*random_sample)/mat_row_count
+        random_sample = (100.0 * random_sample) / mat_row_count
     if mat_row_count:
         # setup a query manager
         qm = QueryManager(
@@ -841,7 +844,11 @@ def post(
                 }
             "filter_spatial_dict": {
                 "tablename": {
-                "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
+                    "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
+            }
+            "filter_regex_dict": {
+                "tablename": {
+                    "column_name": "regex"
             }
         }
         Returns:
@@ -911,11 +918,16 @@ def post(
                 "tablename":{
                     "column_name":value
                 }
-            }
+            },
             "filter_spatial_dict": {
                 "tablename":{
                     "column_name":[[min_x,min_y,minz], [max_x_max_y_max_z]]
                 }
+            },
+            "filter_regex_dict": {
+                "tablename":{
+                    "column_name": "regex"
+                }
             }
         }
         Returns:
@@ -989,6 +1001,10 @@ def post(self, datastack_name: str):
                 "table_name": {
                 "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
             }
+            "filter_regex_dict":{
+                "table_name":{
+                    "column_name": "regex"
+            }
         }
         Returns:
             pyarrow.buffer: a series of bytes that can be deserialized using pyarrow.deserialize
@@ -1262,6 +1278,11 @@ def post(
                 "tablename": {
                 "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
             }
+            "filter_regex_dict": {
+                "tablename": {
+                    "column_name": "regex"
+                }
+            }
         }
         Returns:
             pyarrow.buffer: a series of bytes that can be deserialized using pyarrow.deserialize
@@ -1315,7 +1336,7 @@ def post(
         qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
         qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
         qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
-
+        qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
         select_columns = data.get("select_columns", None)
         if select_columns:
             for column in select_columns:
@@ -1346,10 +1367,6 @@ def post(
         )
 
 
-from sqlalchemy.sql.sqltypes import String, Integer, Float, DateTime, Boolean, Numeric
-from geoalchemy2.types import Geometry
-
-
 def get_table_schema(table):
     """
     Get the schema of a table as a jsonschema

diff --git a/materializationengine/blueprints/client/common.py b/materializationengine/blueprints/client/common.py
@@ -254,6 +254,7 @@ def handle_simple_query(
     qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
     qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
     qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
+    qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
     qm.apply_filter({table_name: {"valid": True}}, qm.apply_equal_filter)
     select_columns = data.get("select_columns", None)
     if select_columns:
@@ -389,6 +390,7 @@ def handle_complex_query(
     qm.apply_filter(data.get("filter_out_dict", None), qm.apply_notequal_filter)
     qm.apply_filter(data.get("filter_equal_dict", None), qm.apply_equal_filter)
     qm.apply_filter(data.get("filter_spatial_dict", None), qm.apply_spatial_filter)
+    qm.apply_filter(data.get("filter_regex_dict", None), qm.apply_regex_filter)
     for table_info in data["tables"]:
         table_name = table_info[0]
         qm.apply_filter({table_name: {"valid": True}}, qm.apply_equal_filter)

diff --git a/materializationengine/blueprints/client/query_manager.py b/materializationengine/blueprints/client/query_manager.py
@@ -8,7 +8,7 @@
 )
 import numpy as np
 from geoalchemy2.types import Geometry
-from sqlalchemy.sql.sqltypes import Integer
+from sqlalchemy.sql.sqltypes import Integer, String
 from sqlalchemy import or_, func
 from sqlalchemy.orm import aliased
 from sqlalchemy.sql.selectable import Alias
@@ -265,6 +265,15 @@ def apply_table_crud_filter(
         f2 = get_column(model, deleted_column).between(str(start_time), str(end_time))
         self._filters.append((or_(f1, f2),))
 
+    def apply_regex_filter(self, table_name, column_name, regex):
+        model = self._find_relevant_model(
+            table_name=table_name, column_name=column_name
+        )
+        column = get_column(model, column_name)
+        if not isinstance(column.type, String):
+            raise ValueError("Regex filter is only supported on string columns")
+        self._filters.append((column.op("~")(regex),))
+
     def select_column(self, table_name, column_name):
         # if the column_name is not in the table_name list
         # then we should add it
@@ -356,6 +365,10 @@ def configure_query(self, user_data):
                 "table_name": {
                 "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]]
             }
+            "filter_regex_dict":{
+                "table_name":{
+                    "column_name":"regex"
+            }
         }"""
         self.add_table(user_data["table"])
         if user_data.get("join_tables", None):
@@ -395,6 +408,9 @@ def apply_filter(filter_key, filter_func):
         self.apply_filter(
             user_data.get("filter_spatial_dict", None), self.apply_spatial_filter
         )
+        self.apply_filter(
+            user_data.get("filter_regex_dict", None), self.apply_regex_filter
+        )
 
         if user_data.get("suffixes", None):
             self._suffixes.update(user_data["suffixes"])

diff --git a/materializationengine/blueprints/client/schemas.py b/materializationengine/blueprints/client/schemas.py
@@ -51,6 +51,7 @@ class V2QuerySchema(Schema):
     filter_notin_dict = fields.Dict()
     filter_equal_dict = fields.Dict()
     filter_spatial_dict = fields.Dict()
+    filter_regex_dict = fields.Dict(required=False)
     select_columns = fields.Dict()
     offset = fields.Integer()
     limit = fields.Integer()
@@ -65,6 +66,7 @@ class SimpleQuerySchema(Schema):
     filter_notin_dict = fields.Dict()
     filter_equal_dict = fields.Dict()
     filter_spatial_dict = fields.Dict()
+    filter_regex_dict = fields.Dict(required=False)
     select_columns = fields.List(fields.Str)
     offset = fields.Integer()
     limit = fields.Integer()
@@ -81,6 +83,7 @@ class ComplexQuerySchema(Schema):
     filter_notin_dict = fields.Dict()
     filter_equal_dict = fields.Dict()
     filter_spatial_dict = fields.Dict()
+    filter_regex_dict = fields.Dict(required=False)
     select_columns = fields.List(fields.Str)
     select_column_map = fields.Dict()
     offset = fields.Integer()