exclude columns

tercen · Aug 27, 2024 · 58fc54b · 58fc54b
1 parent 67015cf
commit 58fc54b
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 20 deletions.
diff --git a/runner.py b/runner.py
@@ -41,6 +41,8 @@ def parse_args(argv):
     params["tolerance"] = 0.001
     params["toleranceType"] = "relative"
     params["hidden_columns"] = False
+
+    params["exclude_columns"] = []
 
     params["templateFolder"] = "workflow_tests" #None
 
@@ -98,6 +100,9 @@ def parse_args(argv):
 
         if opt == '--tag':
             params["tag"] = arg
+
+        if opt == '--exclude':
+            params["exclude_columns"] = arg.split(",")
 
         if opt == '--branch':
             params["branch"] = arg
@@ -127,6 +132,8 @@ def parse_args(argv):
 
     params["gitToken"] = gitToken
 
+
+
     return params
 
 
@@ -239,7 +246,7 @@ def run_with_params(params, mode="cli"):
 
                 util.msg("Comparing Results", verbose)
                 resultDict = workflow_compare.diff_workflow(client, workflowRun, gsWkf,  params["tolerance"],
-                                        params["toleranceType"], params["hidden_columns"], verbose)
+                                        params["toleranceType"], params["hidden_columns"], verbose, exclude=params["exclude_columns"])
 
 
                 if resultDict != None and resultDict != []:
@@ -326,16 +333,7 @@ def fixDictTypes(dictObj):
                 o = float(o)
             dictObj[i] = o
 
-
-            # elif(isinstance(v, np.ndarray)):
-            #     print("{} is ndarray".format(k))
-            #     if( v.dtype == np.int32 ):
-            #         print("{}".format(k))
-            #         v = float(v.tolist())
-            #     else:
-            #         v = v.tolist()
-            #     o[k] = v
-
+
     else:
         for k, v in dictObj.items():
             if isinstance(v, dict)  or isinstance(v, list):

diff --git a/workflow_funcs/workflow_compare.py b/workflow_funcs/workflow_compare.py
@@ -89,7 +89,7 @@ def get_simple_relation_id_list(obj):
     return idList
 
 
-def compare_schema(client, tableIdx, schema, refSchema, tol=0, tolType="absolute", hiddenColumns=False, verbose=False):
+def compare_schema(client, tableIdx, schema, refSchema, tol=0, tolType="absolute", hiddenColumns=False, exclude=[], verbose=False):
     tableRes = {}
     hasDiff = False
 
@@ -102,13 +102,15 @@ def compare_schema(client, tableIdx, schema, refSchema, tol=0, tolType="absolute
     refColNames = []
 
     for name in colNamesIn:
-        if not name.startswith(".") or hiddenColumns == True:
-            colNames.append(name)
+        if not name in exclude:
+            if not name.startswith(".") or hiddenColumns == True:
+                colNames.append(name)
 
 
     for name in refColNamesIn:
-        if not name.startswith(".") or hiddenColumns == True:
-            refColNames.append(name)
+        if not name in exclude:
+            if not name.startswith(".") or hiddenColumns == True:
+                refColNames.append(name)
     res = compare_columns_metadata(colNames, refColNames)
 
     if len(res) > 0:
@@ -276,7 +278,7 @@ def compare_export_step(client, tableIdx, stp, refStp,  tol=0, tolType="absolute
 
 
 
-def compare_step(client, tableIdx, stp, refStp,  tol=0, tolType="absolute", hiddenColumns=False, verbose=False):
+def compare_step(client, tableIdx, stp, refStp,  tol=0, tolType="absolute", hiddenColumns=False, exclude=[], verbose=False,):
     stepResult = {}
 
 
@@ -338,7 +340,7 @@ def compare_step(client, tableIdx, stp, refStp,  tol=0, tolType="absolute", hidd
                         schema = client.tableSchemaService.get(idList[k])
                         refSchema = client.tableSchemaService.get(refIdList[k])
 
-                        res = compare_schema(client, w, schema, refSchema,  tol, tolType=tolType, hiddenColumns=hiddenColumns)
+                        res = compare_schema(client, w, schema, refSchema,  tol, tolType=tolType, hiddenColumns=hiddenColumns, exclude=exclude)
                         tableRes = res[0]
                         hasDiff = res[1]
 
@@ -352,7 +354,7 @@ def compare_step(client, tableIdx, stp, refStp,  tol=0, tolType="absolute", hidd
 
     return stepResult
 
-def diff_workflow(client, workflow, refWorkflow,  tol=0, tolType="absolute", hiddenColumns=False, verbose=False):
+def diff_workflow(client, workflow, refWorkflow,  tol=0, tolType="absolute", hiddenColumns=False, verbose=False, exclude=[]):
     resultDict = []
 
     if len(workflow.steps) != len(refWorkflow.steps):
@@ -365,7 +367,7 @@ def diff_workflow(client, workflow, refWorkflow,  tol=0, tolType="absolute", hid
             refStp = refWorkflow.steps[i]
 
             if isinstance(stp.state.taskState, DoneState) and isinstance(refStp.state.taskState, DoneState):
-                stepRes = compare_step(client, i, stp, refStp,  tol, tolType, hiddenColumns)
+                stepRes = compare_step(client, i, stp, refStp,  tol, tolType, hiddenColumns, exclude=exclude)
 
                 if len(stepRes) > 0:
                     resultDict.append(stepRes)