Skip to content

Commit

Permalink
updated Dockerfile, created actions, and updated the cli
Browse files Browse the repository at this point in the history
  • Loading branch information
romer8 committed May 1, 2024
1 parent c9adc12 commit 7e819b5
Show file tree
Hide file tree
Showing 4 changed files with 219 additions and 28 deletions.
38 changes: 38 additions & 0 deletions .github/scripts/clean_up_docker_hub.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/bin/bash
# Based on kizbitz/dockerhub-v2-api-organization.sh at https://gist.github.com/kizbitz/175be06d0fbbb39bc9bfa6c0cb0d4721

# Example for the Docker Hub V2 API
# Returns all images and tags associated with a Docker Hub organization account.
# Requires 'jq': https://stedolan.github.io/jq/

# set username, password, and organization
UNAME=$1
UPASS=$2
ORG=$3
REPO=$4
MAX_IMAGE=$5
# -------

set -e

# get token
echo "Retrieving token ..."
TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${UNAME}'", "password": "'${UPASS}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token)

# get list of repositories
IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/?page_size=300 | jq -r '.results | sort_by(.last_updated) | reverse[] | .name')
count=0
echo ${IMAGE_TAGS}
for j in ${IMAGE_TAGS}
do
count_all=$((count_all + 1))
if [[ ${j} == *"dev_"* ]]; then
count=$((count + 1))
# Keep the first max_image.
if [ ${MAX_IMAGE} -lt ${count} ]; then
echo -n " - ${j} ... "
curl -X DELETE -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/${j}/
echo "DELETED"
fi
fi
done
67 changes: 67 additions & 0 deletions .github/workflows/build_and_push_dev_image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: build_push_dev_image

on:
push:
branches:
- '*'

env:
DOCKER_HUB_ORG: gioelkin
DOCKER_REPO: tethys-ngiab

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# build image
build:
runs-on: ubuntu-latest
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: 'true'

# Checks-out the hydrocompute extra submodule
- name: checkout Hydrocompute submodule
run: |
cd apps/tethysapp-hydrocompute/tethysapp/hydrocompute/public/HydroCompute && git submodule update --force --init --recursive --remote
- name: Set Tag
run: |
echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV
echo "TAG_LATEST=dev_latest" >> $GITHUB_ENV
- name: Test Tag
run: |
echo $TAG
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_BUILDER_USERNAME }}
password: ${{ secrets.DOCKER_BUILDER_TOKEN }}

- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: |
${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG }}
${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}:${{ env.TAG_LATEST }}
cache-from: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest
cache-to: type=registry,ref=${{ env.DOCKER_HUB_ORG }}/${{ env.DOCKER_REPO }}-dev-cache:latest,mode=max

cleanup:
needs: [build]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: only keeps the first 5 image
run: |
echo "TAG=dev_${GITHUB_SHA}" >> $GITHUB_ENV
echo $TAG
. .github/scripts/clean_up_docker_hub.sh '${{ secrets.DOCKER_BUILDER_USERNAME }}' '${{ secrets.DOCKER_BUILDER_TOKEN }}' '${{ env.DOCKER_HUB_ORG }}' '${{ env.DOCKER_REPO }}' '${{ env.MAX_NUMBER_IMAGE }}'
4 changes: 3 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ ENV PORTAL_SUPERUSER_PASSWORD=pass
RUN micromamba install --yes -c conda-forge --file requirements.txt \
&& micromamba clean --all --yes \
&& cd ${TETHYS_HOME}/apps/tethysapp-ngiab \
&& tethys install -d -N
&& npm install && npm run build \
&& tethys install -d -N \
&& mv salt /srv/salt/

#########
# PORTS #
Expand Down
138 changes: 111 additions & 27 deletions cli/convert_geom.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,94 @@
# import geopandas as gpd
# import argparse
# from geo.Geoserver import Geoserver
# import shutil
# import os


# def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path):
# """
# Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326.

# Parameters:
# - gpkg_path: Path to the GeoPackage file.
# - layer_name: Name of the layer in the GeoPackage to convert.
# - output_path: Path where the output GeoJSON file should be saved.
# """
# # Load the specified layer from the GeoPackage
# gdf = gpd.read_file(gpkg_path, layer=layer_name)

# # Check and transform the coordinate system to EPSG:4326 if needed
# if gdf.crs.to_string() != "EPSG:4326":
# gdf = gdf.to_crs(f"EPSG:4326")

# # Save the GeoDataFrame as a GeoJSON file
# gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON")


# def publish_gpkg_layer_to_geoserver(
# gpkg_path,
# layer_name,
# shp_path,
# geoserver_host,
# geoserver_port,
# geoserver_username,
# geoserver_password,
# ):
# # convert to shapefile to uploaded to geoserver
# gdf = gpd.read_file(gpkg_path, layer=layer_name)

# gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp")

# # zip it
# shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]]

# # Using shutil to create a zip archive containing all shapefile components
# with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf:
# for file in shp_files:
# zipf.write(file, arcname=os.path.basename(file))

# # Optional: Remove the shapefile files after zipping, if you don't need them
# for file in shp_files:
# os.remove(file)

# geo = Geoserver(
# f"{geoserver_host}:{geoserver_port}/geoserver",
# username=geoserver_username,
# password=geoserver_password,
# )
# geo.create_workspace(workspace="ngiab")

# geo.create_shp_datastore(
# path=f"{shp_path}.zip",
# store_name="hydrofabrics",
# workspace="ngiab",
# )


# def main():
# # Setup argument parser
# parser = argparse.ArgumentParser(
# description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326."
# )
# parser.add_argument("gpkg_path", help="Path to the GeoPackage file.")
# parser.add_argument("layer_name", help="Name of the layer to convert.")
# parser.add_argument("output_path", help="Output path for the GeoJSON file.")
# parser.add_argument(
# "--publish", action="store_true", help="Publish the layer to GeoServer."
# )
# parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.")
# # Parse arguments
# args = parser.parse_args()
# # Call the function with parsed arguments
# convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path)
# if args.publish:
# publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path)


# if __name__ == "__main__":
# main()


import geopandas as gpd
import argparse
from geo.Geoserver import Geoserver
Expand All @@ -6,27 +97,12 @@


def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path):
"""
Convert a layer from a GeoPackage to a GeoJSON file, projected to EPSG:4326.
Parameters:
- gpkg_path: Path to the GeoPackage file.
- layer_name: Name of the layer in the GeoPackage to convert.
- output_path: Path where the output GeoJSON file should be saved.
"""
# Load the specified layer from the GeoPackage
gdf = gpd.read_file(gpkg_path, layer=layer_name)

# Check and transform the coordinate system to EPSG:4326 if needed
if gdf.crs.to_string() != "EPSG:4326":
gdf = gdf.to_crs(epsg=4326)

# Save the GeoDataFrame as a GeoJSON file
gdf.to_crs(f"EPSG:4326").to_file(output_path, driver="GeoJSON")


def get_remove_unimportant_nexus():
pass
gdf.to_file(output_path, driver="GeoJSON")


def publish_gpkg_layer_to_geoserver(
Expand All @@ -38,20 +114,16 @@ def publish_gpkg_layer_to_geoserver(
geoserver_username,
geoserver_password,
):
# convert to shapefile to uploaded to geoserver
gdf = gpd.read_file(gpkg_path, layer=layer_name)

gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp")
gdf.to_crs(f"EPSG:4326").to_file(f"{shp_path}.shp", driver="ESRI Shapefile")

# zip it
shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]]

# Using shutil to create a zip archive containing all shapefile components
with shutil.ZipFile(f"{shp_path}.zip", "w") as zipf:
for file in shp_files:
zipf.write(file, arcname=os.path.basename(file))

# Optional: Remove the shapefile files after zipping, if you don't need them
for file in shp_files:
os.remove(file)

Expand All @@ -60,17 +132,16 @@ def publish_gpkg_layer_to_geoserver(
username=geoserver_username,
password=geoserver_password,
)
geo.create_workspace(workspace="ngiab")
geo.create_workspace(workspace="nextgen")

geo.create_shp_datastore(
path=f"{shp_path}.zip",
store_name="hydrofabrics",
workspace="ngiab",
workspace="nextgen",
)


def main():
# Setup argument parser
parser = argparse.ArgumentParser(
description="Convert a GeoPackage layer to GeoJSON format with projection to EPSG:4326."
)
Expand All @@ -81,12 +152,25 @@ def main():
"--publish", action="store_true", help="Publish the layer to GeoServer."
)
parser.add_argument("--shp_path", help="Path to save the shapefile for GeoServer.")
# Parse arguments
parser.add_argument("--geoserver_host", help="GeoServer host.")
parser.add_argument("--geoserver_port", help="GeoServer port.")
parser.add_argument("--geoserver_username", help="GeoServer username.")
parser.add_argument("--geoserver_password", help="GeoServer password.")

args = parser.parse_args()
# Call the function with parsed arguments

convert_gpkg_to_geojson(args.gpkg_path, args.layer_name, args.output_path)

if args.publish:
publish_gpkg_layer_to_geoserver(args.gpkg_path, args.shp_path)
publish_gpkg_layer_to_geoserver(
args.gpkg_path,
args.layer_name,
args.shp_path,
args.geoserver_host,
args.geoserver_port,
args.geoserver_username,
args.geoserver_password,
)


if __name__ == "__main__":
Expand Down

0 comments on commit 7e819b5

Please sign in to comment.