Added metadata validation demo (#208)

* Added metadata validation demo * Do not run hooks-metadata-validation notebook in GitHub action --------- Co-authored-by: Your Name <[email protected]>
treeverse · Jun 28, 2024 · bb4ad0d · bb4ad0d
1 parent 75a08ad
commit bb4ad0d
Show file tree

Hide file tree

Showing 7 changed files with 1,064 additions and 0 deletions.
diff --git a/.github/workflows/notebooks_to_exclude.txt b/.github/workflows/notebooks_to_exclude.txt
@@ -8,6 +8,7 @@
 ./hooks-webhooks-demo.ipynb
 ./hooks-schema-and-pii-validation.ipynb
 ./hooks-schema-validation.ipynb
+./hooks-metadata-validation.ipynb
 
 # This only works against lakeFS cloud
 ./rbac-demo.ipynb

diff --git a/00_notebooks/00_index.ipynb b/00_notebooks/00_index.ipynb
@@ -34,6 +34,7 @@
     "* [**Only allow specific file formats in data lake**](hooks-webhooks-demo.ipynb) (with lakeFS webhooks)\n",
     "* [**Prevent unintended schema change**](hooks-schema-validation.ipynb) (with lakeFS Lua hooks)\n",
     "* [**Avoid leaking PII data**](hooks-schema-and-pii-validation.ipynb) (shows how to use multiple Lua hooks)\n",
+    "* [**Validate metadata**](hooks-metadata-validation.ipynb) (shows how to use multiple Lua hooks)\n",
     "* [**Import into a lakeFS repository from multiple paths**](./import-multiple-buckets.ipynb) \n",
     "* [**ML Experimentation/Reproducibility 01 (Dogs)**](./ml-reproducibility.ipynb)\n",
     "* [**ML Experimentation 02 (Wine Quality)**](./ml-experimentation-wine-quality-prediction.ipynb)</br>_See also the [accompanying blog](https://lakefs.io/blog/building-an-ml-experimentation-platform-for-easy-reproducibility-using-lakefs/)_\n",

diff --git a/00_notebooks/assets/lakefs_demo.py b/00_notebooks/assets/lakefs_demo.py
@@ -22,3 +22,13 @@ def print_commit(log):
     print('Parents:', log.parents)
     print('Metadata:')
     pprint(log.metadata)
+
+def lakefs_ui_endpoint(lakefsEndPoint):
+    if lakefsEndPoint.startswith('http://host.docker.internal'):
+        lakefsUIEndPoint = lakefsEndPoint.replace('host.docker.internal','127.0.0.1')
+    elif lakefsEndPoint.startswith('http://lakefs'):
+        lakefsUIEndPoint = lakefsEndPoint.replace('lakefs','127.0.0.1')
+    else:
+        lakefsUIEndPoint = lakefsEndPoint
+
+    return lakefsUIEndPoint