diff --git a/.github/scripts/clean_up_docker_hub.sh b/.github/scripts/clean_up_docker_hub.sh new file mode 100755 index 0000000..98d74b8 --- /dev/null +++ b/.github/scripts/clean_up_docker_hub.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Based on kizbitz/dockerhub-v2-api-organization.sh at https://gist.github.com/kizbitz/175be06d0fbbb39bc9bfa6c0cb0d4721 + +# Example for the Docker Hub V2 API +# Returns all images and tags associated with a Docker Hub organization account. +# Requires 'jq': https://stedolan.github.io/jq/ + +# set username, password, and organization +UNAME=$1 +UPASS=$2 +ORG=$3 +REPO=$4 +MAX_IMAGE=$5 +# ------- + +set -e + +# get token +echo "Retrieving token ..." +TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${UNAME}'", "password": "'${UPASS}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) + +# get list of repositories +IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/?page_size=300 | jq -r '.results | sort_by(.last_updated) | reverse[] | .name') +count=0 +echo ${IMAGE_TAGS} +for j in ${IMAGE_TAGS} +do + count_all=$((count_all + 1)) + if [[ ${j} == *"dev_"* ]]; then + count=$((count + 1)) + # Keep the first max_image. + if [ ${MAX_IMAGE} -lt ${count} ]; then + echo -n " - ${j} ... " + curl -X DELETE -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/${j}/ + echo "DELETED" + fi + fi +done \ No newline at end of file diff --git a/.github/workflows/build_and_push_dev_image.yml b/.github/workflows/build_and_push_dev_image.yml new file mode 100644 index 0000000..da88a8b --- /dev/null +++ b/.github/workflows/build_and_push_dev_image.yml @@ -0,0 +1,67 @@ +name: build_push_dev_image + +on: + push: + branches: + - '*' + +env: + DOCKER_HUB_ORG: gioelkin + DOCKER_REPO: tethys-ngiab + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # build image + build: + runs-on: ubuntu-latest + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'true' + + # Checks-out the hydrocompute extra submodule + - name: checkout Hydrocompute submodule + run: | + cd apps/tethysapp-hydrocompute/tethysapp/hydrocompute/public/HydroCompute && git submodule update --force --init --recursive --remote + + - name: Set Tag + run: | + echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV + echo "TAG_LATEST=dev_latest" >> $GITHUB_ENV + + - name: Test Tag + run: | + echo $TAG + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_BUILDER_USERNAME }} + password: ${{ secrets.DOCKER_BUILDER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: | + ${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG }} + ${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG_LATEST }} + cache-from: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest + cache-to: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest,mode=max + + cleanup: + needs: [build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: only keeps the first 5 image + run: | + echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV + echo $TAG + . .github/scripts/clean_up_docker_hub.sh '${{ secrets.DOCKER_BUILDER_USERNAME }}' '${{ secrets.DOCKER_BUILDER_TOKEN }}' '${{ env.DOCKER_HUB_ORG }}' '${{ env.DOCKER_REPO }}' '${{ env.MAX_NUMBER_IMAGE }}' diff --git a/Dockerfile b/Dockerfile index 558d6ee..7ac0d66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,9 @@ ENV PORTAL_SUPERUSER_PASSWORD=pass RUN micromamba install --yes -c conda-forge --file requirements.txt \ && micromamba clean --all --yes \ && cd ${TETHYS_HOME}/apps/tethysapp-ngiab \ - && tethys install -d -N + && npm install && npm run build \ + && tethys install -d -N \ + && mv salt /srv/salt/ ######### # PORTS # diff --git a/cli/convert_geom.py b/cli/convert_geom.py index 0fe7779..fae45cd 100644 --- a/cli/convert_geom.py +++ b/cli/convert_geom.py @@ -1,3 +1,94 @@ +# import geopandas as gpd +# import argparse +# from geo.Geoserver import Geoserver +# import shutil +# import os + + +# def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path): +# """ +# Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326. + +# Parameters: +# - gpkg_path: Path to the GeoPackage file. +# - layer_name: Name of the layer in the GeoPackage to convert. +# - output_path: Path where the output GeoJSON file should be saved. +# """ +# # Load the specified layer from the GeoPackage +# gdf = gpd.read_file(gpkg_path, layer=layer_name) + +# # Check and transform the coordinate system to EPSG:4326 if needed +# if gdf.crs.to_string() != "EPSG:4326": +# gdf = gdf.to_crs(f"EPSG:4326") + +# # Save the GeoDataFrame as a GeoJSON file +# gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON") + + +# def publish_gpkg_layer_to_geoserver( +# gpkg_path, +# layer_name, +# shp_path, +# geoserver_host, +# geoserver_port, +# geoserver_username, +# geoserver_password, +# ): +# # convert to shapefile to uploaded to geoserver +# gdf = gpd.read_file(gpkg_path, layer=layer_name) + +# gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp") + +# # zip it +# shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] + +# # Using shutil to create a zip archive containing all shapefile components +# with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf: +# for file in shp_files: +# zipf.write(file, arcname=os.path.basename(file)) + +# # Optional: Remove the shapefile files after zipping, if you don't need them +# for file in shp_files: +# os.remove(file) + +# geo = Geoserver( +# f"{geoserver_host}:{geoserver_port}/geoserver", +# username=geoserver_username, +# password=geoserver_password, +# ) +# geo.create_workspace(workspace="ngiab") + +# geo.create_shp_datastore( +# path=f"{shp_path}.zip", +# store_name="hydrofabrics", +# workspace="ngiab", +# ) + + +# def main(): +# # Setup argument parser +# parser = argparse.ArgumentParser( +# description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326." +# ) +# parser.add_argument("gpkg_path", help="Path to the GeoPackage file.") +# parser.add_argument("layer_name", help="Name of the layer to convert.") +# parser.add_argument("output_path", help="Output path for the GeoJSON file.") +# parser.add_argument( +# "--publish", action="store_true", help="Publish the layer to GeoServer." +# ) +# parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.") +# # Parse arguments +# args = parser.parse_args() +# # Call the function with parsed arguments +# convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path) +# if args.publish: +# publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path) + + +# if __name__ == "__main__": +# main() + + import geopandas as gpd import argparse from geo.Geoserver import Geoserver @@ -6,27 +97,12 @@ def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path): - """ - Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326. - - Parameters: - - gpkg_path: Path to the GeoPackage file. - - layer_name: Name of the layer in the GeoPackage to convert. - - output_path: Path where the output GeoJSON file should be saved. - """ - # Load the specified layer from the GeoPackage gdf = gpd.read_file(gpkg_path, layer=layer_name) - # Check and transform the coordinate system to EPSG:4326 if needed if gdf.crs.to_string() != "EPSG:4326": gdf = gdf.to_crs(epsg=4326) - # Save the GeoDataFrame as a GeoJSON file - gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON") - - -def get_remove_unimportant_nexus(): - pass + gdf.to_file(output_path, driver="GeoJSON") def publish_gpkg_layer_to_geoserver( @@ -38,20 +114,16 @@ def publish_gpkg_layer_to_geoserver( geoserver_username, geoserver_password, ): - # convert to shapefile to uploaded to geoserver gdf = gpd.read_file(gpkg_path, layer=layer_name) - gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp") + gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp", driver="ESRI Shapefile") - # zip it shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] - # Using shutil to create a zip archive containing all shapefile components with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf: for file in shp_files: zipf.write(file, arcname=os.path.basename(file)) - # Optional: Remove the shapefile files after zipping, if you don't need them for file in shp_files: os.remove(file) @@ -60,17 +132,16 @@ def publish_gpkg_layer_to_geoserver( username=geoserver_username, password=geoserver_password, ) - geo.create_workspace(workspace="ngiab") + geo.create_workspace(workspace="nextgen") geo.create_shp_datastore( path=f"{shp_path}.zip", store_name="hydrofabrics", - workspace="ngiab", + workspace="nextgen", ) def main(): - # Setup argument parser parser = argparse.ArgumentParser( description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326." ) @@ -81,12 +152,25 @@ def main(): "--publish", action="store_true", help="Publish the layer to GeoServer." ) parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.") - # Parse arguments + parser.add_argument("--geoserver_host", help="GeoServer host.") + parser.add_argument("--geoserver_port", help="GeoServer port.") + parser.add_argument("--geoserver_username", help="GeoServer username.") + parser.add_argument("--geoserver_password", help="GeoServer password.") + args = parser.parse_args() - # Call the function with parsed arguments + convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path) + if args.publish: - publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path) + publish_gpkg_layer_to_geoserver( + args.gpkg_path, + args.layer_name, + args.shp_path, + args.geoserver_host, + args.geoserver_port, + args.geoserver_username, + args.geoserver_password, + ) if __name__ == "__main__":