diff --git a/.docker-setup.sh b/.docker-setup.sh deleted file mode 100755 index dfdc797a38..0000000000 --- a/.docker-setup.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -#automate the small things for setting up docker containers -#This file sets up the backend and the frontend and optional database container. -#NOTE: The frontend is currently under construction. -# -#The script is needed to handle: -# -Environment variables for -# -Runtime values needing to be set and accurate for the backend's database credentials and github api key -# -Pre-runtime values needing to be set and accurate for the database hostname and type (whether it is test data or not). The ip address needs to be added in the extra_hosts argument of the yml markup. -# -Setting up a network alias in order to let the docker container communicate with local hosts. -# -Easily seeing console output and process statistics from one convienient window. -# -Easily save console output to logs. -# -#This file uses two environment files -# - One called docker_env.txt which holds the runtime enviroment variables that the container itself uses -# - One called .env which holds the environment variables that docker-compose.yml uses and holds the database type. -#TODO: - #Let users know how to configure the database to work for local connection because its not *that* clear right now. Its in the docs at least. - #Make container work with gitlab key - #Test this script on macOS -# -missingModules="" - -#Check everything that needs to be in the $PATH is in there. -#Bash doesn't let this work if this is in an if statement for some reason it has to be chained -type -P "docker" &>/dev/null && echo "docker found..." || missingModules="${missingModules} docker" -type -P "ifconfig" &>/dev/null && echo "ifconfig found..." || missingModules="${missingModules} ifconfig (part of net-tools)" -type -P "psql" &>/dev/null && echo "psql found..." || missingModules="${missingModules} psql" -type -P "watch" &>/dev/null && echo "watch found..." || missingModules="${missingModules} watch" - -if [ ! -z "$missingModules" ] -then - echo "One or more modules required to run this script is missing or not in your \$PATH:" - echo "Note: OSX users will need to install watch with \"brew install watch\"" - echo "Including:$missingModules" - exit 1 -fi -unset $missingModules - -if [ "$EUID" -ne 0 ]; - then echo "Please run as root" - exit 1 -fi - -#Always use a clean .env file because it is a subset of docker_env.txt so we can just generate it from that. -if [[ -f ".env" ]] -then - rm .env -fi -touch .env - -#This is differant for MacOS -#Script uses an alias for localhost that is the below ip -echo "Setting up network alias..." -#Check kernel for OS, assumes either linux or macOS -if [ "$(uname -s)" == "Linux" ] -then - ifconfig lo:0 10.254.254.254 - ifconfig lo:0 - echo "Linux detected..." -else - ifconfig lo0 alias 10.254.254.254 - ifconfig lo0 -fi - -#Prompt for deploy type. -echo "Types of docker deployment: " -echo -echo "1. Deploy the backend using docker connected to a non-docker database." -echo "2. Deploy backend and database together in docker containers." -echo "3. Deploy the backend and database together in docker containers using premade test data." -echo -read -p "Would you like to use : " deployChoice - -case $deployChoice in - - 1) - #Start script to set up just two containers - exec scripts/docker/docker-setup-external.sh - ;; - - 2) - #Start script to set up all three containers. - #Set env variable to not use test data - echo "AUGUR_DB_TYPE=database" >> .env - exec scripts/docker/docker-setup-database.sh - ;; - - 3) - #Start script to set up all three containers - #Set env variable to use test data. - echo "AUGUR_DB_TYPE=test_data" >> .env - exec scripts/docker/docker-setup-database.sh - ;; - - *) - echo "Invalid choice!" - exit 1 - ;; -esac diff --git a/.github/workflows/auto_merge.yml b/.github/workflows/auto_merge.yml new file mode 100644 index 0000000000..889ab241e8 --- /dev/null +++ b/.github/workflows/auto_merge.yml @@ -0,0 +1,22 @@ +name: Merge main into dev + +on: + push: + branches: + - main + +jobs: + update-dev: + permissions: write-all + name: update-dev + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + git checkout dev + git merge main + - name: Push to dev + uses: CasperWA/push-protected@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: dev \ No newline at end of file diff --git a/docs/new-install.md b/docs/new-install.md index 060703e691..409d8fe856 100644 --- a/docs/new-install.md +++ b/docs/new-install.md @@ -374,3 +374,5 @@ You can stop augur with `augur backend stop`, followed by `augur backend kill`. 4. `sudo docker build -t augur-new -f docker/backend/Dockerfile .` 5. `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to run the database in a Docker Container or `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to connect to an already running database. + +_Note: `AUGUR\_DB` and `RABBIT\_*` variables are optional when using the default docker-comopse.yml. `docker-compose-externalDB` does require `AUGUR\_DB` set to a postgresql installation._ diff --git a/docs/source/docker/usage.rst b/docs/source/docker/usage.rst index 03e4c92deb..4201f3dc32 100644 --- a/docs/source/docker/usage.rst +++ b/docs/source/docker/usage.rst @@ -43,33 +43,6 @@ By default, the only logs shown by the container are the logs of Augur's main da As for worker logs. They are currently a work in progress to be made easier to view. Shortly, they will automatically populate on the host machine and it will not be necessary to step inside the container. -Using the repo loading UI [NOT RECOMMENDED] -------------------------------------------- - -.. warning:: - The frontend is very out of date and will *very likely* not work with the backend. It is still available,however. The recommended way of accessing augur through docker is through standard api calls at ``localhost:5000``. - - -Augur offers a special graphical interface for loading repository groups when using the Docker containers. This component is called ``augurface``, and is available anytime you are using the ``backend`` and ``frontend`` services together. - -To use it first, start the two services (we recommend using `Docker Compose Script `_ for this): - -.. code-block:: bash - - # this example uses the docker-setup script - $ sudo ./docker-setup.sh - -Then, navigate to ``http://localhost:8080/augurface/`` in your browser - **note the trailing slash!** Once you're on this page, you'll need to enter your Augur API key in the box on the top right. On a default Docker installation, you can use ``docker_key``, but we recommend changing this as soon as possible if you are planning to use the instance long-term. Commands for working with the API keys can be found `here <../getting-started/command-line-interface/db.html>`_. - -Once you've entered your API key, you will be able to use the UI to automatically import GitHub organizations as a repo group, or manually create and edit repo groups yourself. Deleting repos or repo groups is are currently supported, but would be a great contribution! - -.. warning:: - - Because the UI only requires an API key to edit the database, **we recommend that you do not publicly deploy any Docker instance outside your local network or intranet.** The Docker build is intended ONLY for short-term data collection usage and local development. We understand if this inconveniences you, but the functionality is very new to Augur and still needs time to be production-ready. **You have been warned.** - -If are not using the ``frontend`` service, you can use the `database CLI <../getting-started/command-line-interface/db.html>`_ from within the container to add repos. See below for how to start a shell within the container. - - Conclusion ----------- diff --git a/environment.txt b/environment.txt index b8d72f052b..7c9ebfa380 100644 --- a/environment.txt +++ b/environment.txt @@ -3,4 +3,5 @@ AUGUR_GITHUB_USERNAME= AUGUR_GITLAB_API_KEY= AUGUR_GITLAB_USERNAME= AUGUR_DB= - +AUGUR_RABBITMQ_USERNAME= +AUGUR_RABBITMQ_PASSWORD= diff --git a/scripts/docker/docker-setup-database.sh b/scripts/docker/docker-setup-database.sh deleted file mode 100755 index c682938051..0000000000 --- a/scripts/docker/docker-setup-database.sh +++ /dev/null @@ -1,66 +0,0 @@ -#This is where the scripts are differant. -githubAPIKey="dockerkey" - -if [[ -f "docker_env.txt" ]] -then - githubAPIKey=$(awk -F= -v key="AUGUR_GITHUB_API_KEY" '$1==key {print $2}' docker_env.txt) - test -z "$githubAPIKey" && echo "No existing API key could be found!" || echo "Discovered existing api key $githubAPIKey" - rm docker_env.txt -fi - -touch docker_env.txt -echo "AUGUR_GITHUB_API_KEY=$githubAPIKey" >> docker_env.txt -echo "AUGUR_DB_SCHEMA_BUILD=0" >> docker_env.txt - -dbHostname=10.254.254.254 -#docker_env.txt is differant than '.env', '.env' is for the enviroment variables used in the docker-compose.yml file -echo "AUGUR_DB_HOST=$dbHostname" >> docker_env.txt -echo "AUGUR_DB_HOST=$dbHostname" >> .env -echo "AUGUR_DB_NAME=augur" >> docker_env.txt -echo "AUGUR_DB_PORT=5434" >> docker_env.txt -echo "AUGUR_DB_USER=augur" >> docker_env.txt -echo "AUGUR_DB_PASSWORD=augur" >> docker_env.txt -echo "AUGUR_RABBITMQ_USERNAME=augur" >> .env -echo "AUGUR_RABBITMQ_PASSWORD=password123" >> .env - - -#This checks the .env file's environment variables NOT docker_env.txt. -#AUGUR_DB_TYPE is set by the parent script in the .env file. -dbType=$(awk -F= -v key="AUGUR_DB_TYPE" '$1==key {print $2}' .env) -#Handle test data. -if [ "$dbType" == "test_data" ]; then - sed -i -r '/AUGUR_DB_NAME/ s/(^.*)(=.*)/\1=test_data/g' docker_env.txt -fi - -#Ask the user if they want to be prompted or use an existing config file (docker_env.txt) -read -p "Would you like to be prompted for an api key? [y/N] " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - - #Prompt for user info - read -p "Please input Github API key: " githubAPIKey - echo - sed -i -r '/AUGUR_GITHUB_API_KEY/ s/(^.*)(=.*)/\1=1/g' docker_env.txt -fi - - -echo "Tearing down old docker stack..." -docker compose -f docker-compose.yml -f database-compose.yml down --remove-orphans - -#Run docker stack in background to catch up to later -echo "Starting set up of docker stack..." -nohup docker compose -f docker-compose.yml -f database-compose.yml up --no-recreate &>/tmp/dockerComposeLog & -PIDOS=$! - -#While the containers are up show a watch monitor that shows container status and live feed from logs -printf "\nNow showing active docker containers:\n" -watch -n1 --color 'docker compose -f docker-compose.yml -f database-compose.yml ps && echo && tail -n 30 /tmp/dockerComposeLog && echo "Ctrl+C to Exit"' -printf "\n" - -#Stop the process and clean up dead containers on SIGINT. -kill -15 $PIDOS -#Cleaning up dead containers -echo "Cleaning up dead containers... " -docker compose -f docker-compose.yml -f database-compose.yml down --remove-orphans -bash scripts/docker/cleanup.sh diff --git a/scripts/docker/docker-setup-external.sh b/scripts/docker/docker-setup-external.sh deleted file mode 100755 index be6336807c..0000000000 --- a/scripts/docker/docker-setup-external.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -#Ask the user if they want to be prompted or use an existing config file (docker_env.txt) -read -p "Would you like to be prompted for database credentials? [y/N] " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - - if [[ -f "docker_env.txt" ]] - then - rm docker_env.txt - fi - - touch docker_env.txt - - #Prompt for user info - read -p "Please input Github API key: " githubAPIKey - echo "AUGUR_GITHUB_API_KEY=$githubAPIKey" >> docker_env.txt - echo - - echo "Please choose which database hostname to use in the form of an ip or 'localhost'" - read -p "Plase input database hostname: " dbHostname - - #Do a quick translate from 'localhost' to the network alias on lo:0 - if [ "$dbHostname" == "localhost" ]; then - dbHostname=10.254.254.254 - echo "$dbHostname" - echo "AUGUR_DB_SCHEMA_BUILD=1" >> docker_env.txt - #Pass it to build argument in yml - echo "AUGUR_DB_SCHEMA_BUILD=1" >> .env - else - echo "AUGUR_DB_SCHEMA_BUILD=0" >> docker_env.txt - fi - - echo "What port is Postgresql running on?" - read -p "Please input the database port: " dbPort - - #docker_env.txt is differant than '.env', '.env' is for the enviroment variables used in the docker-compose.yml file - echo "AUGUR_DB_HOST=$dbHostname" >> docker_env.txt - echo "AUGUR_DB_HOST=$dbHostname" >> .env - echo "AUGUR_DB_PORT=$dbPort" >> docker_env.txt - echo "AUGUR_RABBITMQ_USERNAME=augur" >> .env - echo "AUGUR_RABBITMQ_PASSWORD=password123" >> .env - - - read -p "Plase input database name: " dbName - #Pretty sure these stay constant among augur databases - echo "AUGUR_DB_NAME=$dbName" >> docker_env.txt - #echo "AUGUR_DB_PORT=5432" >> docker_env.txt - echo "AUGUR_DB_USER=augur" >> docker_env.txt - - #Password is blurred out because thats the standard - read -s -p "Please input database password: " dbPassword - echo - - #If blank, use default password 'password' - if [ -z "$dbPassword" ] - then - dbPassword="password" - fi - echo "AUGUR_DB_PASSWORD=$dbPassword" >> docker_env.txt -else - #docker_env.txt should always be present in a docker compose build otherwise it can cause issues for the database. - if [[ ! -f "docker_env.txt" ]] - then - echo "docker_env.txt not found. Please add environment variables in this file or restart the script and choose to prompt db credentials." - exit 1 - fi - - #Copy host name to docker compose env file. - cat docker_env.txt | grep AUGUR_DB_HOST >> .env -fi - -#Test the database connection -testHost=$(awk -F= -v key="AUGUR_DB_HOST" '$1==key {print $2}' docker_env.txt) -testPassword=$(awk -F= -v key="AUGUR_DB_PASSWORD" '$1==key {print $2}' docker_env.txt) -testName=$(awk -F= -v key="AUGUR_DB_NAME" '$1==key {print $2}' docker_env.txt) -testUser=$(awk -F= -v key="AUGUR_DB_USER" '$1==key {print $2}' docker_env.txt) - -#Test variables that can't be verified with a test database connection. (This includes testPassword but that is needed for the later db test) -missingEnv="" -testGithubKey=$(awk -F= -v key="AUGUR_GITHUB_API_KEY" '$1==key {print $2}' docker_env.txt) -testSchemaBuild=$(awk -F= -v key="AUGUR_DB_SCHEMA_BUILD" '$1==key {print $2}' docker_env.txt) -testPort=$(awk -F= -v key="AUGUR_DB_PORT" '$1==key {print $2}' docker_env.txt) - -test -z "$testGithubKey" && missingEnv="${missingEnv} AUGUR_GITHUB_API_KEY" -test -z "$testSchemaBuild" && missingEnv="${missingEnv} AUGUR_DB_SCHEMA_BUILD" -test -z "$testPort" && missingEnv="${missingEnv} AUGUR_DB_PORT" -test -z "$testPassword" && missingEnv="${missingEnv} AUGUR_DB_PASSWORD" - -if [ ! -z "$missingEnv" ] -then - echo "One or more environment variables required to run this script is not in docker_env.txt" - echo "Including $missingEnv" - exit 1 -fi -unset missingEnv -unset testGithubKey -unset testSchemaBuild -#unset testPort - -#Test connection using quick bash database request. $? now holds the exit code. -psql -d "postgresql://$testUser:$testPassword@$testHost/$testName" -p $testPort -c "select now()" # &>/dev/null -if [[ ! "$?" -eq 0 ]] -then - echo "Database could not be reached!" - #This prompts the user because the *.conf files are in differant places on differant distros/OS's - echo "Check pg_hba.conf and postgres.conf" - exit 1 -fi - -#Ask the user if they need augur to build schema or use an existing schema -#This is important because if it builds schema when it already exists it can cause probems. -#Its also important because if there is no schema augur throws an error (obviously). -read -p "Do you want to build schema on the database? WARNING: ONLY ANSWER YES IF THE DATABASE IS WITHOUT SCHEMA [y/N] " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]] -then - #use some regex to set the schema build in docker_env.txt - sed -i -r -E '/AUGUR_DB_SCHEMA_BUILD/ s/(^.*)(=.*)/\1=1/g' docker_env.txt -else - sed -i -r -E '/AUGUR_DB_SCHEMA_BUILD/ s/(^.*)(=.*)/\1=0/g' docker_env.txt -fi - -echo "Tearing down old docker stack..." -docker compose -f docker-compose.yml down --remove-orphans - -#Run docker stack in background to catch up to later -echo "Starting set up of docker stack..." -nohup docker compose -f docker-compose.yml up --no-recreate &>/tmp/dockerComposeLog & -PIDOS=$! - -#While the containers are up show a watch monitor that shows container status and live feed from logs -printf "\nNow showing active docker containers:\n" -watch -n1 --color 'docker compose ps && echo && tail -n 30 /tmp/dockerComposeLog && echo "Ctrl+C to Exit"' -printf "\n" - -#Stop the process and clean up dead containers on SIGINT. -kill -15 $PIDOS -#Cleaning up dead containers -echo "Cleaning up dead containers... " -docker compose -f docker-compose.yml down --remove-orphans -bash scripts/docker/cleanup.sh