feat: enable ssh tunneling using env variables for superset

docker-compose.superset-dev.yml

@@ -0,0 +1,21 @@
+x-superset-volumes:
+  &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./superset:/app/docker
+  - superset_home:/app/superset_home
+
+version: "3.7"
+services:
+  superset:
+    env_file: superset/.env-non-dev
+    image: apache/superset:3.0.2
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    volumes: *superset-volumes
+
+volumes:
+  superset_home:
+    external: false

docker-compose.superset-prod.yml

@@ -0,0 +1,84 @@
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis
+x-superset-volumes:
+  &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./superset:/app/docker
+  - superset_home:/app/superset_home
+
+version: "3.7"
+services:
+  redis:
+    image: redis:7
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+
+  superset:
+    env_file: superset/.env-non-dev
+    image: apache/superset:3.0.2
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  db:
+    env_file: superset/.env-non-dev
+    image: postgres:15
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+      - ./superset/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+
+  superset-init:
+    image: apache/superset:3.0.2
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: superset/.env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      disable: true
+
+  superset-worker:
+    image: apache/superset:3.0.2
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: superset/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
+        ]
+
+  superset-worker-beat:
+    image: apache/superset:3.0.2
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: superset/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      disable: true
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker-compose.yml

@@ -14,14 +14,6 @@ services:
       - COUCHDB_SECURE=${COUCHDB_SECURE:-true}
       - HTTP_ENDPOINT=postgrest:3000
 
-  superset:
-    build: 
-      context: ./superset/
-      args:
-        SUPERSET_PASSWORD: ${SUPERSET_PASSWORD:-password}
-        SUPERSET_ADMIN_EMAIL: ${SUPERSET_ADMIN_EMAIL:[email protected]}
-    ports:
-      - 8080:8088
 
   dbt:
     platform: linux/amd64

env.template

@@ -26,3 +26,4 @@ COUCHDB_SECURE=false
 # superset: required environment variables for 'gamma', 'prod' and 'local'
 SUPERSET_PASSWORD=password
 [email protected]
+SUPERSET_SSH_TUNNELING=false

scripts/config.ts

@@ -40,9 +40,9 @@ export const ENV_COMMANDS: Record<string, any> = {
     "COUCHDB_HOST=adp-sandbox.dev.medicmobile.org COUCHDB_DB=medic COUCHDB_USER=medic \
     docker-compose -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f docker-compose.yml \
       up -d logstash postgres postgrest dbt",
-  prod: "docker-compose -f docker-compose.couchdb.yml -f docker-compose.postgrest.yml -f docker-compose.yml \
-  up -d logstash postgrest dbt",
+  prod: "docker-compose -f docker-compose.postgrest.yml -f docker-compose.superset-prod.yml \
+  -f docker-compose.yml up -d",
   local:
     "docker-compose -f docker-compose.couchdb.yml -f docker-compose.postgres.yml \
-    -f docker-compose.yml up -d",
+    -f docker-compose.superset-dev.yml -f docker-compose.yml up -d",
 };

superset/.env-non-dev

@@ -0,0 +1,28 @@
+COMPOSE_PROJECT_NAME=superset
+
+# database configurations (do not modify)
+DATABASE_DIALECT=postgresql
+
+EXAMPLES_DB=examples
+EXAMPLES_HOST=db
+EXAMPLES_USER=examples
+EXAMPLES_PASSWORD=examples
+EXAMPLES_PORT=5432
+
+# database engine specific environment variables
+# change the below if you prefer another database engine
+POSTGRES_DB=superset
+POSTGRES_USER=superset
+POSTGRES_PASSWORD=superset
+
+# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
+PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+SUPERSET_ENV=production
+SUPERSET_LOAD_EXAMPLES=no
+SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
+CYPRESS_CONFIG=false
+SUPERSET_PORT=8088
+MAPBOX_API_KEY=''

superset/README.md

@@ -0,0 +1,75 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Getting Started with Superset using Docker
+
+Docker is an easy way to get started with Superset.
+
+## Prerequisites
+
+1. [Docker](https://www.docker.com/get-started)
+2. [Docker Compose](https://docs.docker.com/compose/install/)
+
+## Configuration
+
+The `/app/pythonpath` folder is mounted from [`./docker/pythonpath_dev`](./pythonpath_dev)
+which contains a base configuration [`./docker/pythonpath_dev/superset_config.py`](./pythonpath_dev/superset_config.py)
+intended for use with local development.
+
+### Local overrides
+
+In order to override configuration settings locally, simply make a copy of [`./docker/pythonpath_dev/superset_config_local.example`](./pythonpath_dev/superset_config_local.example)
+into `./docker/pythonpath_dev/superset_config_docker.py` (git ignored) and fill in your overrides.
+
+### Local packages
+
+If you want to add Python packages in order to test things like databases locally, you can simply add a local requirements.txt (`./docker/requirements-local.txt`)
+and rebuild your Docker stack.
+
+Steps:
+
+1. Create `./docker/requirements-local.txt`
+2. Add your new packages
+3. Rebuild docker compose
+    1. `docker compose down -v`
+    2. `docker compose up`
+
+## Initializing Database
+
+The database will initialize itself upon startup via the init container ([`superset-init`](./docker-init.sh)). This may take a minute.
+
+## Normal Operation
+
+To run the container, simply run: `docker compose up`
+
+After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088)
+to start your journey.
+
+## Developing
+
+While running, the container server will reload on modification of the Superset Python and JavaScript source code.
+Don't forget to reload the page to take the new frontend into account though.
+
+## Production
+
+It is possible to run Superset in non-development mode by using [`docker-compose-non-dev.yml`](../docker-compose-non-dev.yml). This file excludes the volumes needed for development and uses [`./docker/.env-non-dev`](./.env-non-dev) which sets the variable `SUPERSET_ENV` to `production`.
+
+## Resource Constraints
+
+If you are attempting to build on macOS and it exits with 137 you need to increase your Docker resources. See instructions [here](https://docs.docker.com/docker-for-mac/#advanced) (search for memory)

superset/docker-bootstrap.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+#
+# playwright is an optional package - run only if it is installed
+#
+if command -v playwright > /dev/null 2>&1; then
+  playwright install-deps
+  playwright install chromium
+fi
+
+case "${1}" in
+  worker)
+    echo "Starting Celery worker..."
+    celery --app=superset.tasks.celery_app:app worker -O fair -l INFO
+    ;;
+  beat)
+    echo "Starting Celery beat..."
+    rm -f /tmp/celerybeat.pid
+    celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+    ;;
+  app)
+    echo "Starting web app (using development server)..."
+    flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+    ;;
+  app-gunicorn)
+    echo "Starting web app..."
+    /usr/bin/run-server.sh
+    ;;
+  *)
+    echo "Unknown Operation!!!"
+    ;;
+esac

superset/docker-ci.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+/app/docker/docker-init.sh
+
+# TODO: copy config overrides from ENV vars
+
+# TODO: run celery in detached state
+export SERVER_THREADS_AMOUNT=8
+# start up the web server
+
+/usr/bin/run-server.sh

superset/docker-frontend.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+# Packages needed for puppeteer:
+apt update
+apt install -y chromium
+
+cd /app/superset-frontend
+npm install -f --no-optional --global webpack webpack-cli
+npm install -f --no-optional
+
+echo "Running frontend"
+npm run dev