From 4252c0b6f67927befc391884b77844b08c41f47e Mon Sep 17 00:00:00 2001 From: Anthony Fok Date: Thu, 17 Jun 2021 05:20:30 -0600 Subject: [PATCH] add_data: Download opendrr-boundaries.dump from S3 - python/Dockerfile: Install zip and aws - sample.env: Add AWS credentials [TODO] Fixes #116 --- python/Dockerfile | 5 ++++- python/add_data.sh | 3 ++- sample.env | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/python/Dockerfile b/python/Dockerfile index eacdf0df..8852731b 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -9,6 +9,9 @@ COPY . . #RUN chmod +x docker-entrypoint.sh && chmod +x add_data.sh RUN chmod +x add_data.sh && \ apt-get update && \ - apt-get install -y eatmydata git git-lfs jq time + apt-get install -y eatmydata git git-lfs jq time zip && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install #ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"] CMD ./add_data.sh diff --git a/python/add_data.sh b/python/add_data.sh index 7b7a94fd..20ca008a 100755 --- a/python/add_data.sh +++ b/python/add_data.sh @@ -429,7 +429,8 @@ import_census_boundaries() { LOG "## Importing Census Boundaries" INFO "Trying to download pre-generated PostGIS database dump (for speed)..." - if RUN curl -O -v --retry 999 --retry-max-time 0 https://opendrr.eccp.ca/file/OpenDRR/opendrr-boundaries.dump || \ + if RUN aws s3 sync s3://opendrr-api-prebuilt-cache-1/boundaries/ . --exclude "*" --include "opendrr-boundaries.dump" || \ + RUN curl -O -v --retry 999 --retry-max-time 0 https://opendrr.eccp.ca/file/OpenDRR/opendrr-boundaries.dump || \ RUN curl -O -v --retry 999 --retry-max-time 0 https://f000.backblazeb2.com/file/OpenDRR/opendrr-boundaries.dump then RUN pg_restore -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$DB_NAME" \ diff --git a/sample.env b/sample.env index 76242d86..c3f994e0 100644 --- a/sample.env +++ b/sample.env @@ -20,6 +20,11 @@ loadPhysicalExposure=true loadRiskDynamics=true loadSocialFabric=true +# For downloading pre-built data, e.g. opendrr-boundaries.dump +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_DEFAULT_REGION=ca-central-1 + # For testing and debugging #ADD_DATA_DRY_RUN=true #ADD_DATA_DOWNLOAD_ONLY=true