updated Dockerfile, created actions, and updated the cli

Aquaveo · May 1, 2024 · 7e819b5 · 7e819b5
1 parent c9adc12
commit 7e819b5
Show file tree

Hide file tree

Showing 4 changed files with 219 additions and 28 deletions.
diff --git a/.github/scripts/clean_up_docker_hub.sh b/.github/scripts/clean_up_docker_hub.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Based on kizbitz/dockerhub-v2-api-organization.sh at https://gist.github.com/kizbitz/175be06d0fbbb39bc9bfa6c0cb0d4721
+
+# Example for the Docker Hub V2 API
+# Returns all images and tags associated with a Docker Hub organization account.
+# Requires 'jq': https://stedolan.github.io/jq/
+
+# set username, password, and organization
+UNAME=$1
+UPASS=$2
+ORG=$3
+REPO=$4
+MAX_IMAGE=$5
+# -------
+
+set -e
+
+# get token
+echo "Retrieving token ..."
+TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${UNAME}'", "password": "'${UPASS}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
+
+# get list of repositories
+IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/?page_size=300 | jq -r '.results | sort_by(.last_updated) | reverse[] | .name')
+count=0
+echo ${IMAGE_TAGS}
+for j in ${IMAGE_TAGS}
+do
+  count_all=$((count_all + 1))
+  if [[ ${j} == *"dev_"* ]]; then
+    count=$((count + 1))
+    # Keep the first max_image.
+    if [ ${MAX_IMAGE} -lt ${count} ]; then
+      echo -n "  - ${j} ... "
+      curl -X DELETE -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/${j}/
+      echo "DELETED"
+    fi
+  fi
+done
diff --git a/.github/workflows/build_and_push_dev_image.yml b/.github/workflows/build_and_push_dev_image.yml
@@ -0,0 +1,67 @@
+name: build_push_dev_image
+
+on:
+  push:
+    branches:
+      - '*'
+
+env:
+  DOCKER_HUB_ORG: gioelkin
+  DOCKER_REPO: tethys-ngiab
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # build image
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      # Checks-out the hydrocompute extra submodule
+      - name: checkout Hydrocompute submodule
+        run: |
+          cd apps/tethysapp-hydrocompute/tethysapp/hydrocompute/public/HydroCompute && git submodule update --force --init --recursive --remote
+
+      - name: Set Tag
+        run: |
+          echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV
+          echo "TAG_LATEST=dev_latest" >> $GITHUB_ENV
+
+      - name: Test Tag
+        run: |
+          echo $TAG
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_BUILDER_USERNAME }}
+          password: ${{ secrets.DOCKER_BUILDER_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG }}
+            ${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG_LATEST }}
+          cache-from: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest
+          cache-to: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest,mode=max
+
+  cleanup:
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: only keeps the first 5 image
+        run: |
+          echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV
+          echo $TAG
+          . .github/scripts/clean_up_docker_hub.sh '${{ secrets.DOCKER_BUILDER_USERNAME }}' '${{ secrets.DOCKER_BUILDER_TOKEN }}' '${{ env.DOCKER_HUB_ORG }}' '${{ env.DOCKER_REPO }}' '${{ env.MAX_NUMBER_IMAGE }}'
diff --git a/Dockerfile b/Dockerfile
@@ -35,7 +35,9 @@ ENV PORTAL_SUPERUSER_PASSWORD=pass
 RUN micromamba install --yes -c conda-forge --file requirements.txt \
     && micromamba clean --all --yes \
     && cd ${TETHYS_HOME}/apps/tethysapp-ngiab \
-    && tethys install -d -N
+    && npm install && npm run build \
+    && tethys install -d -N \
+    && mv salt /srv/salt/
 
 #########
 # PORTS #

diff --git a/cli/convert_geom.py b/cli/convert_geom.py
@@ -1,3 +1,94 @@
+# import geopandas as gpd
+# import argparse
+# from geo.Geoserver import Geoserver
+# import shutil
+# import os
+
+
+# def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path):
+#     """
+#     Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326.
+
+#     Parameters:
+#     - gpkg_path: Path to the GeoPackage file.
+#     - layer_name: Name of the layer in the GeoPackage to convert.
+#     - output_path: Path where the output GeoJSON file should be saved.
+#     """
+#     # Load the specified layer from the GeoPackage
+#     gdf = gpd.read_file(gpkg_path, layer=layer_name)
+
+#     # Check and transform the coordinate system to EPSG:4326 if needed
+#     if gdf.crs.to_string() != "EPSG:4326":
+#         gdf = gdf.to_crs(f"EPSG:4326")
+
+#     # Save the GeoDataFrame as a GeoJSON file
+#     gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON")
+
+
+# def publish_gpkg_layer_to_geoserver(
+#     gpkg_path,
+#     layer_name,
+#     shp_path,
+#     geoserver_host,
+#     geoserver_port,
+#     geoserver_username,
+#     geoserver_password,
+# ):
+#     # convert to shapefile to uploaded to geoserver
+#     gdf = gpd.read_file(gpkg_path, layer=layer_name)
+
+#     gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp")
+
+#     # zip it
+#     shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]]
+
+#     # Using shutil to create a zip archive containing all shapefile components
+#     with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf:
+#         for file in shp_files:
+#             zipf.write(file, arcname=os.path.basename(file))
+
+#     # Optional: Remove the shapefile files after zipping, if you don't need them
+#     for file in shp_files:
+#         os.remove(file)
+
+#     geo = Geoserver(
+#         f"{geoserver_host}:{geoserver_port}/geoserver",
+#         username=geoserver_username,
+#         password=geoserver_password,
+#     )
+#     geo.create_workspace(workspace="ngiab")
+
+#     geo.create_shp_datastore(
+#         path=f"{shp_path}.zip",
+#         store_name="hydrofabrics",
+#         workspace="ngiab",
+#     )
+
+
+# def main():
+#     # Setup argument parser
+#     parser = argparse.ArgumentParser(
+#         description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326."
+#     )
+#     parser.add_argument("gpkg_path", help="Path to the GeoPackage file.")
+#     parser.add_argument("layer_name", help="Name of the layer to convert.")
+#     parser.add_argument("output_path", help="Output path for the GeoJSON file.")
+#     parser.add_argument(
+#         "--publish", action="store_true", help="Publish the layer to GeoServer."
+#     )
+#     parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.")
+#     # Parse arguments
+#     args = parser.parse_args()
+#     # Call the function with parsed arguments
+#     convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path)
+#     if args.publish:
+#         publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path)
+
+
+# if __name__ == "__main__":
+#     main()
+
+
 import geopandas as gpd
 import argparse
 from geo.Geoserver import Geoserver
@@ -6,27 +97,12 @@
 
 
 def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path):
-    """
-    Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326.
-
-    Parameters:
-    - gpkg_path: Path to the GeoPackage file.
-    - layer_name: Name of the layer in the GeoPackage to convert.
-    - output_path: Path where the output GeoJSON file should be saved.
-    """
-    # Load the specified layer from the GeoPackage
     gdf = gpd.read_file(gpkg_path, layer=layer_name)
 
-    # Check and transform the coordinate system to EPSG:4326 if needed
     if gdf.crs.to_string() != "EPSG:4326":
         gdf = gdf.to_crs(epsg=4326)
 
-    # Save the GeoDataFrame as a GeoJSON file
-    gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON")
-
-
-def get_remove_unimportant_nexus():
-    pass
+    gdf.to_file(output_path, driver="GeoJSON")
 
 
 def publish_gpkg_layer_to_geoserver(
@@ -38,20 +114,16 @@ def publish_gpkg_layer_to_geoserver(
     geoserver_username,
     geoserver_password,
 ):
-    # convert to shapefile to uploaded to geoserver
     gdf = gpd.read_file(gpkg_path, layer=layer_name)
 
-    gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp")
+    gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp", driver="ESRI Shapefile")
 
-    # zip it
     shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]]
 
-    # Using shutil to create a zip archive containing all shapefile components
     with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf:
         for file in shp_files:
             zipf.write(file, arcname=os.path.basename(file))
 
-    # Optional: Remove the shapefile files after zipping, if you don't need them
     for file in shp_files:
         os.remove(file)
 
@@ -60,17 +132,16 @@ def publish_gpkg_layer_to_geoserver(
         username=geoserver_username,
         password=geoserver_password,
     )
-    geo.create_workspace(workspace="ngiab")
+    geo.create_workspace(workspace="nextgen")
 
     geo.create_shp_datastore(
         path=f"{shp_path}.zip",
         store_name="hydrofabrics",
-        workspace="ngiab",
+        workspace="nextgen",
     )
 
 
 def main():
-    # Setup argument parser
     parser = argparse.ArgumentParser(
         description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326."
     )
@@ -81,12 +152,25 @@ def main():
         "--publish", action="store_true", help="Publish the layer to GeoServer."
     )
     parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.")
-    # Parse arguments
+    parser.add_argument("--geoserver_host", help="GeoServer host.")
+    parser.add_argument("--geoserver_port", help="GeoServer port.")
+    parser.add_argument("--geoserver_username", help="GeoServer username.")
+    parser.add_argument("--geoserver_password", help="GeoServer password.")
+
     args = parser.parse_args()
-    # Call the function with parsed arguments
+
     convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path)
+
     if args.publish:
-        publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path)
+        publish_gpkg_layer_to_geoserver(
+            args.gpkg_path,
+            args.layer_name,
+            args.shp_path,
+            args.geoserver_host,
+            args.geoserver_port,
+            args.geoserver_username,
+            args.geoserver_password,
+        )
 
 
 if __name__ == "__main__":