From 2fe205deaca2dd3335d27dce7fa120537ed70410 Mon Sep 17 00:00:00 2001 From: Aaron Gundel Date: Thu, 31 Oct 2024 07:16:06 -0600 Subject: [PATCH] update script and env variables --- arches_lingo/settings.py | 13 +- docker/entrypoint.sh | 99 +++++++------- docker/production/Dockerfile | 2 +- docker/production/entrypoint.sh | 231 -------------------------------- 4 files changed, 57 insertions(+), 288 deletions(-) delete mode 100644 docker/production/entrypoint.sh diff --git a/arches_lingo/settings.py b/arches_lingo/settings.py index 65fcb67a..77bc5b57 100644 --- a/arches_lingo/settings.py +++ b/arches_lingo/settings.py @@ -39,11 +39,14 @@ def get_optional_env_variable(var_name, default=None) -> str: SECRETS_MODE = get_optional_env_variable("ARCHES_SECRETS_MODE", "ENV") APP_ROOT = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -DB_NAME = get_optional_env_variable("ARCHES_DB_NAME", APP_NAME) -DB_USER = get_optional_env_variable("ARCHES_PGUSERNAME", "postgres") -DB_PASSWORD = get_optional_env_variable("ARCHES_PGPASSWORD", "postgis") -DB_HOST = get_optional_env_variable("ARCHES_PGHOST", "localhost") -DB_PORT = get_optional_env_variable("ARCHES_PGPORT", "5432") + +# environment variable names for postgres are built-ins for the pg client, do not modify +DB_NAME = get_optional_env_variable("PGDATABASE", APP_NAME) +DB_USER = get_optional_env_variable("PGUSER", "postgres") +DB_PASSWORD = get_optional_env_variable("PGPASSWORD", "postgis") +DB_HOST = get_optional_env_variable("PGHOST", "localhost") +DB_PORT = get_optional_env_variable("PGPORT", "5432") + ES_USER = get_optional_env_variable("ARCHES_ESUSER", "elastic") ES_PASSWORD = get_optional_env_variable("ARCHES_ESPASSWORD", "E1asticSearchforArche5") ES_HOST = get_optional_env_variable("ARCHES_ESHOST", "localhost") diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 34f10c3a..9f1d76ec 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,17 +6,10 @@ if [[ -z ${ARCHES_PROJECT} ]]; then APP_FOLDER=${ARCHES_ROOT} PACKAGE_JSON_FOLDER=${ARCHES_ROOT} else - APP_FOLDER=${WEB_ROOT}/${ARCHES_PROJECT_ROOT_DIRECTORY} + APP_FOLDER=${WEB_ROOT}/${ARCHES_PROJECT} PACKAGE_JSON_FOLDER=${ARCHES_ROOT} fi -# npm_MODULES_FOLDER=${PACKAGE_JSON_FOLDER}/$(awk \ -# -F '--install.modules-folder' '{print $2}' ${PACKAGE_JSON_FOLDER}/.npmrc \ -# | awk '{print $1}' \ -# | tr -d $'\r' \ -# | tr -d '"' \ -# | sed -e "s/^\.\///g") - # Environmental Variables export DJANGO_PORT=${DJANGO_PORT:-8000} @@ -25,7 +18,7 @@ wait_for_db() { echo "Testing if database server is up..." while [[ ! ${return_code} == 0 ]] do - psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -c "select 1" >&/dev/null + psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSER} --dbname=postgres -c "select 1" >&/dev/null return_code=$? sleep 1 done @@ -34,7 +27,7 @@ wait_for_db() { echo "Testing if Elasticsearch is up..." while [[ ! ${return_code} == 0 ]] do - curl -s "http://${ESHOST}:${ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/null + curl -s "http://${ARCHES_ESHOST}:${ARCHES_ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/nullB return_code=$? sleep 1 done @@ -43,7 +36,7 @@ wait_for_db() { db_exists() { echo "Checking if database "${PGDBNAME}" exists..." - count=`psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -Atc "SELECT COUNT(*) FROM pg_catalog.pg_database WHERE datname='${PGDBNAME}'"` + count=`psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSER} --dbname=postgres -Atc "SELECT COUNT(*) FROM pg_catalog.pg_database WHERE datname='${PGDBNAME}'"` # Check if returned value is a number and not some error message re='^[0-9]+$' @@ -94,30 +87,11 @@ init_arches() { fi } -# npm -# install_npm_components() { -# if [[ ! -d ${npm_MODULES_FOLDER} ]] || [[ ! "$(ls ${npm_MODULES_FOLDER})" ]]; then -# echo "npm modules do not exist, installing..." -# cd ${PACKAGE_JSON_FOLDER} -# npm install -# fi -# } - -#### Misc -copy_settings_local() { - # The settings_local.py in ${ARCHES_ROOT}/arches/ gets ignored if running manage.py from a custom Arches project instead of Arches core app - echo "Copying ${APP_FOLDER}/docker/settings_docker.py to ${APP_FOLDER}/${ARCHES_PROJECT}/settings_docker.py..." - cp ${APP_FOLDER}/docker/settings_docker.py ${APP_FOLDER}/${ARCHES_PROJECT}/settings_docker.py - - # Copy settings_local if it does not exist - cp -n ${APP_FOLDER}/docker/settings_local.py ${APP_FOLDER}/${ARCHES_PROJECT}/settings_local.py -} - #### Run commands start_celery_supervisor() { cd ${APP_FOLDER} - supervisord -c docker/arches-lingo-supervisor.conf + supervisord -c docker/arches-supervisor.conf } run_migrations() { @@ -125,7 +99,7 @@ run_migrations() { echo "----- RUNNING DATABASE MIGRATIONS -----" echo "" cd ${APP_FOLDER} - ../ENV/bin/python manage.py migrate + python3 manage.py migrate } run_setup_db() { @@ -133,7 +107,7 @@ run_setup_db() { echo "----- RUNNING SETUP_DB -----" echo "" cd ${APP_FOLDER} - ../ENV/bin/python manage.py setup_db --force + python3 manage.py setup_db --force } run_load_package() { @@ -141,7 +115,7 @@ run_load_package() { echo "----- *** LOADING PACKAGE: ${ARCHES_PROJECT} *** -----" echo "" cd ${APP_FOLDER} - ../ENV/bin/python manage.py packages -o load_package -s arches_lingo/pkg -db -dev -y + python3 manage.py packages -o load_package -a arches_lingo -db -dev -y } # "exec" means that it will finish building??? @@ -150,25 +124,54 @@ run_django_server() { echo "----- *** RUNNING DJANGO DEVELOPMENT SERVER *** -----" echo "" cd ${APP_FOLDER} + echo "Running Django" + exec /bin/bash -c "source ${WEB_ROOT}/ENV/bin/activate && gunicorn arches_lingo.wsgi" +} + +run_dev_server() { + echo "" + echo "----- *** RUNNING DJANGO DEVELOPMENT SERVER *** -----" + echo "" + cd ${APP_FOLDER} echo "Running Django" exec /bin/bash -c "source ../ENV/bin/activate && pip3 install debugpy -t /tmp && python -Wdefault /tmp/debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:${DJANGO_PORT}" } -#### Main commands -run_arches() { - init_arches - run_django_server +# "exec" means that it will finish building??? +run_gunicorn() { + echo "" + echo "----- *** RUNNING DJANGO PRODUCTION SERVER *** -----" + echo "" + cd ${APP_ROOT} + echo "Running Django" + exec /bin/bash -c "source ../ENV/bin/activate && (/etc/init.d/nginx start&) && gunicorn arches_lingo.wsgi" } -run_webpack() { + +reset_database() { echo "" - echo "----- *** RUNNING WEBPACK DEVELOPMENT SERVER *** -----" + echo "----- RESETTING DATABASE -----" echo "" - cd ${APP_FOLDER} - # echo "Running Webpack" - eval `ssh-agent -s` && cat /run/secrets/ssh_passphrase | SSH_ASKPASS=/bin/cat setsid -w ssh-add 2>> /dev/null - exec /bin/bash -c "source ../ENV/bin/activate && cd /web_root/arches-lingo && npm i && wait-for-it arches-lingo:80 -t 1200 && npm start" + cd ${APP_ROOT} + pwd && ../ENV/bin/python --version + (test $(echo "SELECT FROM pg_database WHERE datname = 'template_postgis'" | ../ENV/bin/python manage.py dbshell | grep -c "1 row") = 1 || \ + (echo "CREATE DATABASE template_postgis" | ../ENV/bin/python manage.py dbshell --database postgres && \ + echo "CREATE EXTENSION postgis" | ../ENV/bin/python manage.py dbshell --database postgres)) + ../ENV/bin/python manage.py packages -o load_package -a arches_lingo -db -dev -y + ../ENV/bin/python manage.py packages -o import_business_data -s tests/fixtures/data/aat_entries_scheme_examples.json -ow overwrite + ../ENV/bin/python manage.py packages -o import_business_data -s tests/fixtures/data/aat_entries_concept_examples.json -ow overwrite + ../ENV/bin/python manage.py es reindex_database -mp +} + +activate_virtualenv() { + . ${WEB_ROOT}/ENV/bin/activate +} + +#### Main commands +run_arches() { + run_django_server } + ### Starting point ### # Use -gt 1 to consume two arguments per pass in the loop @@ -193,29 +196,23 @@ do case ${key} in run_arches) start_celery_supervisor - copy_settings_local wait_for_db run_arches ;; setup_arches) start_celery_supervisor - copy_settings_local wait_for_db setup_arches ;; run_tests) - copy_settings_local wait_for_db run_tests ;; run_migrations) - copy_settings_local wait_for_db run_migrations ;; - install_npm_components) - install_npm_components - ;; + help|-h) display_help ;; diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index ca2f70c5..24c28bb7 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -64,7 +64,7 @@ WORKDIR ${WEB_ROOT} # From here, run commands from ARCHES_ROOT RUN source ${WEB_ROOT}/ENV/bin/activate -COPY docker/production/entrypoint.sh ${WEB_ROOT}/entrypoint.sh +COPY docker/entrypoint.sh ${WEB_ROOT}/entrypoint.sh RUN chmod -R 700 ${WEB_ROOT}/entrypoint.sh &&\ dos2unix ${WEB_ROOT}/entrypoint.sh diff --git a/docker/production/entrypoint.sh b/docker/production/entrypoint.sh deleted file mode 100644 index 4e7f6b38..00000000 --- a/docker/production/entrypoint.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/bin/bash - -# APP and npm folder locations -# ${WEB_ROOT} and ${ARCHES_ROOT} is defined in the Dockerfile, ${ARCHES_PROJECT} in env_file.env -if [[ -z ${ARCHES_PROJECT} ]]; then - APP_FOLDER=${ARCHES_ROOT} - PACKAGE_JSON_FOLDER=${ARCHES_ROOT} -else - APP_FOLDER=${WEB_ROOT}/${ARCHES_PROJECT} - PACKAGE_JSON_FOLDER=${ARCHES_ROOT} -fi - -# Environmental Variables -export DJANGO_PORT=${DJANGO_PORT:-8000} - -#Utility functions that check db status -wait_for_db() { - echo "Testing if database server is up..." - while [[ ! ${return_code} == 0 ]] - do - psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -c "select 1" >&/dev/null - return_code=$? - sleep 1 - done - echo "Database server is up" - - echo "Testing if Elasticsearch is up..." - while [[ ! ${return_code} == 0 ]] - do - curl -s "http://${ESHOST}:${ESPORT}/_cluster/health?wait_for_status=green&timeout=60s" >&/dev/null - return_code=$? - sleep 1 - done - echo "Elasticsearch is up" -} - -db_exists() { - echo "Checking if database "${PGDBNAME}" exists..." - count=`psql --host=${PGHOST} --port=${PGPORT} --user=${PGUSERNAME} --dbname=postgres -Atc "SELECT COUNT(*) FROM pg_catalog.pg_database WHERE datname='${PGDBNAME}'"` - - # Check if returned value is a number and not some error message - re='^[0-9]+$' - if ! [[ ${count} =~ $re ]] ; then - echo "Error: Something went wrong when checking if database "${PGDBNAME}" exists..." >&2; - echo "Exiting..." - exit 1 - fi - - # Return 0 (= true) if database exists - if [[ ${count} > 0 ]]; then - return 0 - else - return 1 - fi -} - -#### Install -init_arches() { - echo "Checking if Arches project "${ARCHES_PROJECT}" exists..." - if [[ ! -d ${APP_FOLDER} ]] || [[ ! "$(ls ${APP_FOLDER})" ]]; then - echo "" - echo "----- Custom Arches project '${ARCHES_PROJECT}' does not exist. -----" - echo "----- Creating '${ARCHES_PROJECT}'... -----" - echo "" - - cd ${WEB_ROOT} - - arches-project create ${ARCHES_PROJECT} - run_setup_db - - exit_code=$? - if [[ ${exit_code} != 0 ]]; then - echo "Something went wrong when creating your Arches project: ${ARCHES_PROJECT}." - echo "Exiting..." - exit ${exit_code} - fi - else - echo "Custom Arches project '${ARCHES_PROJECT}' exists." - wait_for_db - if db_exists; then - echo "Database ${PGDBNAME} already exists." - echo "Skipping Package Loading" - else - echo "Database ${PGDBNAME} does not exists yet." - run_load_package #change to run_load_package if preferred - fi - fi -} - -#### Misc -copy_settings_local() { - # The settings_local.py in ${ARCHES_ROOT}/arches/ gets ignored if running manage.py from a custom Arches project instead of Arches core app - echo "Copying ${APP_FOLDER}/docker/settings_docker.py to ${APP_FOLDER}/${ARCHES_PROJECT}/settings_docker.py..." - cp ${APP_FOLDER}/docker/settings_docker.py ${APP_FOLDER}/${ARCHES_PROJECT}/settings_docker.py - - # Copy settings_local if it does not exist - cp -n ${APP_FOLDER}/docker/settings_local.py ${APP_FOLDER}/${ARCHES_PROJECT}/settings_local.py -} - -#### Run commands - -start_celery_supervisor() { - cd ${APP_FOLDER} - supervisord -c docker/disco-supervisor.conf -} - -run_migrations() { - echo "" - echo "----- RUNNING DATABASE MIGRATIONS -----" - echo "" - cd ${APP_FOLDER} - python3 manage.py migrate -} - -run_setup_db() { - echo "" - echo "----- RUNNING SETUP_DB -----" - echo "" - cd ${APP_FOLDER} - python3 manage.py setup_db --force -} - -run_load_package() { - echo "" - echo "----- *** LOADING PACKAGE: ${ARCHES_PROJECT} *** -----" - echo "" - cd ${APP_FOLDER} - python3 manage.py packages -o load_package -a arches_lingo -db -dev -y -} - -# "exec" means that it will finish building??? -run_django_server() { - echo "" - echo "----- *** RUNNING DJANGO DEVELOPMENT SERVER *** -----" - echo "" - cd ${APP_FOLDER} - echo "Running Django" - exec /bin/bash -c "source ${WEB_ROOT}/ENV/bin/activate && gunicorn arches_lingo.wsgi" -} - -# "exec" means that it will finish building??? -run_gunicorn() { - echo "" - echo "----- *** RUNNING DJANGO PRODUCTION SERVER *** -----" - echo "" - cd ${APP_ROOT} - echo "Running Django" - exec /bin/bash -c "source ../ENV/bin/activate && (/etc/init.d/nginx start&) && gunicorn arches_lingo.wsgi" -} - - -reset_database() { - echo "" - echo "----- RESETTING DATABASE -----" - echo "" - cd ${APP_ROOT} - pwd && ../ENV/bin/python --version - (test $(echo "SELECT FROM pg_database WHERE datname = 'template_postgis'" | ../ENV/bin/python manage.py dbshell | grep -c "1 row") = 1 || \ - (echo "CREATE DATABASE template_postgis" | ../ENV/bin/python manage.py dbshell --database postgres && \ - echo "CREATE EXTENSION postgis" | ../ENV/bin/python manage.py dbshell --database postgres)) - ../ENV/bin/python manage.py setup_db --force - ../ENV/bin/python manage.py packages -o load_package -a arches_lingo -db -dev -y - ../ENV/bin/python manage.py packages -o import_business_data -s tests/fixtures/data/aat_entries_scheme_examples.json -ow overwrite - ../ENV/bin/python manage.py packages -o import_business_data -s tests/fixtures/data/aat_entries_concept_examples.json -ow overwrite -} - -activate_virtualenv() { - . ${WEB_ROOT}/ENV/bin/activate -} - -#### Main commands -run_arches() { - run_django_server -} - -### Starting point ### - -# Use -gt 1 to consume two arguments per pass in the loop -# (e.g. each argument has a corresponding value to go with it). -# Use -gt 0 to consume one or more arguments per pass in the loop -# (e.g. some arguments don't have a corresponding value to go with it, such as --help ). - -# If no arguments are supplied, assume the server needs to be run -if [[ $# -eq 0 ]]; then - start_celery_supervisor - wait_for_db - run_arches -fi - -# Else, process arguments -echo "Full command: $@" -while [[ $# -gt 0 ]] -do - key="$1" - echo "Command: ${key}" - - case ${key} in - run_arches) - start_celery_supervisor - copy_settings_local - wait_for_db - run_arches - ;; - setup_arches) - start_celery_supervisor - copy_settings_local - wait_for_db - setup_arches - ;; - run_tests) - copy_settings_local - wait_for_db - run_tests - ;; - run_migrations) - copy_settings_local - wait_for_db - run_migrations - ;; - - help|-h) - display_help - ;; - *) - cd ${APP_FOLDER} - "$@" - exit 0 - ;; - esac - shift # next argument or value -done