From 122261d34eb0e98d0660cbfd2007423cdd465b46 Mon Sep 17 00:00:00 2001 From: Sylvan Ridderinkhof Date: Thu, 22 Aug 2024 17:28:49 +0200 Subject: [PATCH] feat: updated installation scripts --- direct_indexing/solr/update_solr_cores.sh | 25 +++- docker-compose.yml | 20 --- docs/DOCKER.md | 3 + scripts/build.sh | 26 ++++ scripts/setup.sh | 80 ++++++++++++ scripts/setup/install_docker.sh | 30 +++++ scripts/setup/install_nginx.sh | 90 ++++++++++++++ scripts/setup/install_submodules.sh | 98 +++++++++++++++ scripts/setup/nginx_host_machine/flower | 7 ++ .../setup/nginx_host_machine/iati.cloud | 117 ++++++------------ .../nginx_host_machine}/iati.cloud-redirect | 4 +- scripts/setup/setup_solr.sh | 22 ++++ scripts/start.sh | 25 ++++ scripts/stop.sh | 25 ++++ services/nginx/Dockerfile | 35 ------ services/nginx/NGINX.md | 9 -- services/nginx/hostfile | 7 -- services/nginx/nginx.conf | 32 ----- services/nginx/proxy_params | 4 - services/nginx/sites-enabled/flower | 8 -- services/nginx/sites-enabled/solr | 8 -- 21 files changed, 466 insertions(+), 209 deletions(-) create mode 100644 scripts/build.sh create mode 100644 scripts/setup.sh create mode 100644 scripts/setup/install_docker.sh create mode 100644 scripts/setup/install_nginx.sh create mode 100644 scripts/setup/install_submodules.sh create mode 100644 scripts/setup/nginx_host_machine/flower rename services/nginx/sites-enabled/datastore => scripts/setup/nginx_host_machine/iati.cloud (75%) rename {services/nginx/sites-enabled => scripts/setup/nginx_host_machine}/iati.cloud-redirect (74%) create mode 100644 scripts/setup/setup_solr.sh create mode 100644 scripts/start.sh create mode 100644 scripts/stop.sh delete mode 100644 services/nginx/Dockerfile delete mode 100644 services/nginx/NGINX.md delete mode 100644 services/nginx/hostfile delete mode 100644 services/nginx/nginx.conf delete mode 100644 services/nginx/proxy_params delete mode 100644 services/nginx/sites-enabled/flower delete mode 100644 services/nginx/sites-enabled/solr diff --git a/direct_indexing/solr/update_solr_cores.sh b/direct_indexing/solr/update_solr_cores.sh index 81664874e..a0ba8a466 100644 --- a/direct_indexing/solr/update_solr_cores.sh +++ b/direct_indexing/solr/update_solr_cores.sh @@ -4,6 +4,22 @@ if [ "$EUID" -ne 0 ] exit fi +# Function to prompt user for Y/n choice +ask_for_confirmation() { + read -rp "$1 (Y/n): " choice + case "$choice" in + ""|y|Y ) + return 0 # Default to Y if user presses Enter without typing anything + ;; + n|N ) + return 1 + ;; + * ) + ask_for_confirmation "$1" # Ask again if input is not recognized + ;; + esac +} + # Ask the user if they are sure they want to copy these schemas, if not exit read -p "Are you sure you want to copy the schemas? (y/N) " -n 1 -r echo # (optional) move to a new line @@ -32,7 +48,12 @@ docker cp ./direct_indexing/solr/cores/transaction/managed-schema $solr_containe docker cp ./direct_indexing/solr/cores/activity/xslt $solr_container_id:/bitnami/solr/server/solr/activity/conf/ # Ask the user if this is mounted locally, default to no. If it is, chown the files to 1001:root -read -p "Is this mounted locally? (y/N) " -n 1 -r - +if ask_for_confirmation "Are the files locally mounted (f.ex. on extra mounted volume)?"; then + df -h + read -p "Enter your mounted directory: " mounted_dir + sudo chown -R 1001:root $mounted_dir/solr_data/ +else + echo "Skipping mounted solr directory." +fi echo "Done!" diff --git a/docker-compose.yml b/docker-compose.yml index fd7baa195..1f561d408 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -153,26 +153,6 @@ services: ports: - 5555:5555 - nginx: - container_name: nginx - image: iaticloud/nginx - env_file: - - .env - build: - context: ./services/nginx - args: - - IC_DOMAIN=${IC_DOMAIN} - - SOLR_AUTH_ENCODED=${SOLR_AUTH_ENCODED} - volumes: - - ./:/app - - ./static:/static - ports: - - 80:80 - depends_on: - - celeryflower - - iaticloud - - solr - volumes: db_data: rabbitmq_data: diff --git a/docs/DOCKER.md b/docs/DOCKER.md index d2dc61672..be7d1f2e9 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -38,6 +38,9 @@ Please check out the [reference of .env under the local docs](./LOCAL.md#env). T | nginx | nginx | 80 | ./services/nginx | Runs NGINX and enables the flower and datastore subdomains for a provided domain. For local development it also allows subdomains. Customize `SOLR_AUTH_ENCODED` and `IC_DOMAIN`. iati.cloud-redirect is available but not enabled by default. The docker image is more described [here](../services/nginx/NGINX.md). | ## Running +We recommend using the `./scripts/setup.sh` script to get everything set up for you, then running `sudo docker compose up -d` to start the required processes. + +### Old installation steps We use a git submodule to ensure the Django static is set up correctly. Run the following to prepare to use that submodule: ``` git submodule init diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 000000000..479794ed9 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Help +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + echo "Used to (re)build services running through docker. Optionally specify service names." + echo "" + echo "Usage: bash $0 [service name (optional) (up to 10 service names)]" + exit 0 +fi + +# Start +# Get additional arguments (service names) +I1="$1" +I2="$2" +I3="$3" +I4="$4" +I5="$5" +I6="$6" +I7="$7" +I8="$8" +I9="$9" +I10="$10" + +sudo docker compose build $I1 $I2 $I3 $I4 $I5 $I6 $I7 $I8 $I9 $I10 + +echo "Build script is done. You can now start the services using the './scripts/start.sh $MODE' script." diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 000000000..65aedb307 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Help +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + echo "Used to set up the repository. Specify the environment type." + echo "Optionally initialises git submodules and their setups." + echo "Removes any .env file, and creates a copy of .env.example with the appropriate name, then symlinks it to .env." + echo "Optionally prepopulates DX with data" + echo "" + echo "Usage: . $0" + exit 0 +fi + +# Function to prompt user for Y/n choice +ask_for_confirmation() { + read -rp "$1 (Y/n): " choice + case "$choice" in + ""|y|Y ) + return 0 # Default to Y if user presses Enter without typing anything + ;; + n|N ) + return 1 + ;; + * ) + ask_for_confirmation "$1" # Ask again if input is not recognized + ;; + esac +} + +echo "" +echo "" +if ask_for_confirmation "Do you want to install Docker?"; then + . ./scripts/setup/install_docker.sh +else + echo "Skipping Docker installation." +fi + +echo "" +echo "" +if ask_for_confirmation "Do you want to install NodeJS v16, npm and yarn?"; then + . ./scripts/setup/install_node.sh +else + echo "Skipping NodeJS, npm and yarn installation." +fi + +echo "" +echo "" +if ask_for_confirmation "Do you want to initialise the submodules?"; then + . ./scripts/setup/install_submodules.sh +else + echo "Skipping the submodules." +fi + +echo "" +echo "" +if ask_for_confirmation "Do you want to set up the query builder?"; then + . ./scripts/setup/setup_iati_cloud_frontend.sh +else + echo "Skipping the Query Builder." +fi + +echo "" +echo "" +if ask_for_confirmation "Do you want to install NGINX with SSL enabled?"; then + . ./scripts/setup/install_nginx.sh +else + echo "Skipping NGINX." +fi + +echo "" +echo "" +if ask_for_confirmation "Do you want to set up Solr (must be done before stack can be activated)?"; then + . ./scripts/setup/setup_solr.sh +else + echo "Skipping NGINX." +fi + +echo "" +echo "" +echo "Setup script is done, please set up your env, and run 'bash ./scripts/build.sh ' to build the project." diff --git a/scripts/setup/install_docker.sh b/scripts/setup/install_docker.sh new file mode 100644 index 000000000..84001e5fb --- /dev/null +++ b/scripts/setup/install_docker.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +print_status() { + echo " + +====================================================== + Status Update +------------------------------------------------------ +$1 +====================================================== +" +} + +print_status "Installing Docker..." +# Copied from https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install ca-certificates curl gnupg -y +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg + +# Add the repository to Apt sources: +echo \ +"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ +"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ +sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y +print_status "Done installing Docker." diff --git a/scripts/setup/install_nginx.sh b/scripts/setup/install_nginx.sh new file mode 100644 index 000000000..1421ba234 --- /dev/null +++ b/scripts/setup/install_nginx.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Function to prompt user for Y/n choice +ask_for_confirmation() { + read -rp "$1 (Y/n): " choice + case "$choice" in + ""|y|Y ) + return 0 # Default to Y if user presses Enter without typing anything + ;; + n|N ) + return 1 + ;; + * ) + ask_for_confirmation "$1" # Ask again if input is not recognized + ;; + esac +} + +print_status() { + echo " + +====================================================== + Status Update +------------------------------------------------------ +$1 +====================================================== +" +} + +if ask_for_confirmation "Do NGINX and Certbot need to be installed before setup?"; then + echo "Installing NGINX and Certbot..." + print_status "Installing NGINX..." + sudo apt update + sudo apt install nginx -y + + print_status "Installing certbot..." + sudo apt install software-properties-common -y + sudo add-apt-repository universe -y + sudo apt-get update -y + sudo apt-get install certbot python3-certbot-nginx -y +else + echo "Skipping NGINX and Certbot installation." +fi + +# Configure nginx + +print_status "Setting up NGINX configuration..." +# Function to configure NGINX for a given environment + +# datastore.iati.cloud +# Ask the user for the server name +read -rp "What is the iati.cloud server name as defined in the DNS records (for example, datastore.iati.cloud)?: " server_name +sudo cp ./scripts/setup/nginx_host_machine/iati.cloud "/etc/nginx/sites-available/iati-cloud" +sudo sed -i "s/REPL_SERVER_NAME/$server_name/g" "/etc/nginx/sites-available/iati-cloud" +read -p "Re-Enter your solr username: " username +read -sp "Re-Enter your solr password: " password +encoded_base64=$(echo -n "$username:$password" | base64) +sudo sed -i "s/REPL_AUTH/$encoded_base64/g" "/etc/nginx/sites-available/iati-cloud" +sudo ln -s "/etc/nginx/sites-available/iati-cloud" /etc/nginx/sites-enabled/ + +# flower +sudo cp ./scripts/setup/nginx_host_machine/flower "/etc/nginx/sites-available/flower" +sudo sed -i "s/REPL_SERVER_NAME/$server_name/g" "/etc/nginx/sites-available/flower" +sudo ln -s "/etc/nginx/sites-available/flower" /etc/nginx/sites-enabled/ + +if ask_for_confirmation "Do you want to set up the redirect for iati.cloud -> datastore.iati.cloud?: "; then + sudo cp ./scripts/setup/nginx_host_machine/iati.cloud-redirect "/etc/nginx/sites-available/iati-cloud-redirect" + sudo ln -s "/etc/nginx/sites-available/iati-cloud-redirect" /etc/nginx/sites-enabled/ +fi + +# Restart nginx +print_status "Restarting NGINX..." +sudo service nginx restart + +print_status "Setting up SSL certificates..." +if ask_for_confirmation "Do you want to set up SSL certificates for your domains?"; then + # Set up the ssl certificate, this will require some user input. + echo "Setting up SSL certificates..." + sudo certbot --nginx + + echo "Setting up cron job to renew SSL certificates..." + # update crontab with `0 5 1 * * sudo certbot renew --preferred-challenges http-01` + cron_command="0 5 1 * * sudo certbot renew --preferred-challenges http-01" + temp_cron_file=$(mktemp) + echo "$cron_command" > "$temp_cron_file" + crontab "$temp_cron_file" + rm "$temp_cron_file" +fi + +print_status "Done installing NGINX." diff --git a/scripts/setup/install_submodules.sh b/scripts/setup/install_submodules.sh new file mode 100644 index 000000000..3cf6cd609 --- /dev/null +++ b/scripts/setup/install_submodules.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +print_status() { + echo " + +====================================================== + Status Update +------------------------------------------------------ +$1 +====================================================== +" +} +# Function to prompt user for Y/n choice +ask_for_confirmation() { + read -rp "$1 (Y/n): " choice + case "$choice" in + ""|y|Y ) + return 0 # Default to Y if user presses Enter without typing anything + ;; + n|N ) + return 1 + ;; + * ) + ask_for_confirmation "$1" # Ask again if input is not recognized + ;; + esac +} + +print_status "Installing submodules..." +git submodule init +git submodule update + +print_status "Preparing environment files." +# cp ./monitoring/.env.example ./monitoring/.env +cp ./.env.example.docker ./.env.dev +cp ./.env.example.docker ./.env.prod +cp ./.env.example.docker ./.env.staging +cp ./.env.example.docker ./.env.test +ln -s ./.env.prod ./.env + +# update the .env files with their correct values + +print_status "Updating .env files with correct values. Please provide these required values:" +# Ask the user for their username and password +read -p "Enter your username: " username +read -sp "Enter your password: " password +read -p "Enter your django superuser email: " email +read -p "Enter your IATI.cloud domain (ex.: localhost, datastore.iati.cloud): " domain +read -p "Enter your trusted origin (ex.: https://datastore.iati.cloud): " trusted_origin +echo "" +# password encoded: +encoded_base64=$(echo -n "$username:$password" | base64) + +# List of .env files +env_files=(.env.dev .env.test .env.staging .env.prod) +# Loop through each file and perform the replacement +for env_file in "${env_files[@]}"; do + # Postgres + sed -i "s/POSTGRES_USER=iati_cloud/POSTGRES_USER=$username/g" "$env_file" + sed -i "s/POSTGRES_PASSWORD=oipa/POSTGRES_PASSWORD=$password/g" "$env_file" + # Solr + sed -i "s/SOLR_ADMIN_USERNAME=admin_example/SOLR_ADMIN_USERNAME=$username/g" "$env_file" + sed -i "s|SOLR_ADMIN_PASSWORD=exampl3_123!|SOLR_ADMIN_PASSWORD=$password|g" "$env_file" + sed -i "s|SOLR_BASE_URL=http://admin_example:exampl3_123!@solr:8983/solr|SOLR_BASE_URL=http://$username:$password@solr:8983/solr|g" "$env_file" # NOQA + sed -i "s|SOLR_AUTH_ENCODED=YWRtaW5fZXhhbXBsZTpleGFtcGwzXzEyMyE=|SOLR_AUTH_ENCODED=$encoded_base64|g" "$env_file" + # Flower + sed -i "s/CELERYFLOWER_USER=zz/CELERYFLOWER_USER=$username/g" "$env_file" + sed -i "s/CELERYFLOWER_PASSWORD=zz/CELERYFLOWER_PASSWORD=$password/g" "$env_file" + # Django + sed -i "s/DJANGO_SUPERUSER_USERNAME=admin_example/DJANGO_SUPERUSER_USERNAME=$username/g" "$env_file" + sed -i "s|DJANGO_SUPERUSER_PASSWORD=exampl3_123!|DJANGO_SUPERUSER_PASSWORD=$password|g" "$env_file" + sed -i "s|DJANGO_SUPERUSER_EMAIL=example@zimmerman.team|DJANGO_SUPERUSER_EMAIL=$email|g" "$env_file" + # Mongo + sed -i "s/MONGO_INITDB_ROOT_USERNAME=admin_example/MONGO_INITDB_ROOT_USERNAME=$username/g" "$env_file" + sed -i "s|MONGO_INITDB_ROOT_PASSWORD=exampl3_123!|MONGO_INITDB_ROOT_PASSWORD=$password|g" "$env_file" + sed -i "s|MONGO_CONNECTION_STRING=mongodb://admin_example:exampl3_123!@mongo:27017|MONGO_CONNECTION_STRING=mongodb://$username:$password@mongo:27017|g" "$env_file" # NOQA + # IC_DOMAIN and CSRF_TRUSTED_ORIGINS + sed -i "s|IC_DOMAIN=localhost|IC_DOMAIN=$domain|g" "$env_file" + sed -i "s|CSRF_TRUSTED_ORIGINS=https://|CSRF_TRUSTED_ORIGINS=$trusted_origin|g" "$env_file" +done + +print_status "Done... (By default, .env has been symlinked to .env.prod.)" + +echo "" +echo "" +if ask_for_confirmation "Do you want to set up a mounted solr directory?"; then + df -h + read -p "Enter your mounted directory: " mounted_dir + sudo mkdir $mounted_dir/solr_data + sudo chown -R 1001:root $mounted_dir/solr_data/ + # replace the string "solr_data:/bitnami" in docker-compose.yml + sed -i "s|solr_data:/bitnami|$mounted_dir/solr_data:/bitnami|g" docker-compose.yml +else + echo "Skipping mounted solr directory." +fi + +print_status "Copying the static files..." +cp -r ./static /static diff --git a/scripts/setup/nginx_host_machine/flower b/scripts/setup/nginx_host_machine/flower new file mode 100644 index 000000000..8501d5a38 --- /dev/null +++ b/scripts/setup/nginx_host_machine/flower @@ -0,0 +1,7 @@ +server { + server_name flower.REPL_SERVER_NAME; + + location / { + proxy_pass http://localhost:5555; + } +} diff --git a/services/nginx/sites-enabled/datastore b/scripts/setup/nginx_host_machine/iati.cloud similarity index 75% rename from services/nginx/sites-enabled/datastore rename to scripts/setup/nginx_host_machine/iati.cloud index bc55a5514..0a4d79838 100644 --- a/services/nginx/sites-enabled/datastore +++ b/scripts/setup/nginx_host_machine/iati.cloud @@ -1,6 +1,5 @@ server { - server_name datastore.IC_DOMAIN; - listen 80; + server_name REPL_SERVER_NAME; charset utf-8; client_max_body_size 75M; @@ -9,7 +8,7 @@ server { # Lib files for saving file streams on client location /streamsaver { - alias /static/streamsaver; + alias /static/streamsaver; } # React App @@ -18,19 +17,10 @@ server { try_files $uri /index.html; } - location /admin { - proxy_pass http://iaticloud:8000/admin; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - } - - # Django Media -# location /media { -# alias /root/IATI.cloud/OIPA/OIPA/media; # amend as required -# alias /static/build; # amend as required -# } + location /media { + alias /static/build; # amend as required + } # Django Static location /static { @@ -40,11 +30,19 @@ server { alias /static; } + # Django API + location /api { + include proxy_params; + proxy_pass http://unix:/tmp/oipa.sock; + } + # Django Admin -# location /admin { -# include proxy_params; -# proxy_pass http://unix:/tmp/oipa.sock; -# } + location /admin { + proxy_pass http://localhost:8000/admin; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } # Old endpoint redirect location ~ ^/search/(.*) { @@ -68,11 +66,11 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/activity/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/activity/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } - - # SOLR Transaction + + # SOLR Transaction location /api/v2/transaction { if ($args !~* fl=) { set $args $1$args&fl=iati_identifier,reporting_org_ref,title_narrative,activity_description_narrative,activity_recipient_country_code,activity_recipient_region_code,activity_sector_code,transaction_ref,transaction_humanitarian,transaction_type,transaction_date_iso_date,transaction_value_currency,transaction_value_date,transaction_value,transaction_usd_conversion_rate,transaction_value_usd,transaction_provider_org_ref,transaction_provider_org_provider_activity_id,transaction_provider_org_type,transaction_provider_org_narrative_text,transaction_receiver_org_ref,transaction_receiver_org_receiver_activity_id,transaction_receiver_org_type,transaction_receiver_org_narrative,transaction_disburstment_channel_code,transaction_sector_vocabulary,transaction_sector_code,transaction_recipient_country_code,transaction_recipient_region_code,transaction_flow_type_code,transaction_finance_type_code,transaction_aid_type_code,transaction_aid_type_vocabulary,transaction_tied_status_code,transaction_description_narrative,default_currency,default_lang,default_flow_type_code,default_aid_type_code,default_tied_status_code,default_humanitarian; @@ -84,21 +82,8 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/transaction/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; - } - - # SOLR Transaction POST - location /api/v2/transaction-post { - if ($http_origin !~ '^http?://(localhost)') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - include proxy_params; - proxy_pass http://solr:8983/solr/transaction/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/transaction/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } # SOLR Publisher @@ -110,8 +95,8 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/publisher/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/publisher/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } # SOLR Organisation @@ -123,8 +108,8 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/organisation/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/organisation/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } # SOLR Dataset @@ -136,8 +121,8 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/dataset/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/dataset/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } # SOLR Result @@ -149,8 +134,8 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/result/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/result/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } # SOLR Budget @@ -166,42 +151,10 @@ server { add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } include proxy_params; - proxy_pass http://solr:8983/solr/budget/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; + proxy_pass http://localhost:8983/solr/budget/select; + proxy_set_header Authorization "Basic REPL_AUTH"; } - # SOLR Activity-Sector - location /api/v2/activity-sector { - if ($request_uri !~ .*wt=xslt*) - { - return 404; - } - - if ($http_origin !~ '^http?://(localhost)') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - include proxy_params; - proxy_pass http://solr:8983/solr/activity/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; - } + listen 80; - # SOLR Activity-Recipient-Country - location /api/v2/activity-country { - if ($request_uri !~ .*wt=xslt*) - { - return 404; - } - if ($http_origin !~ '^http?://(localhost)') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - include proxy_params; - proxy_pass http://solr:8983/solr/activity/select; - proxy_set_header Authorization "Basic SOLR_AUTH_ENCODED"; - } -} +} \ No newline at end of file diff --git a/services/nginx/sites-enabled/iati.cloud-redirect b/scripts/setup/nginx_host_machine/iati.cloud-redirect similarity index 74% rename from services/nginx/sites-enabled/iati.cloud-redirect rename to scripts/setup/nginx_host_machine/iati.cloud-redirect index 07b03c2cf..f1a14e4db 100644 --- a/services/nginx/sites-enabled/iati.cloud-redirect +++ b/scripts/setup/nginx_host_machine/iati.cloud-redirect @@ -4,5 +4,5 @@ server { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - return 301 http://datastore.IC_DOMAIN$request_uri; -} \ No newline at end of file + return 301 https://datastore.iati.cloud$request_uri; +} diff --git a/scripts/setup/setup_solr.sh b/scripts/setup/setup_solr.sh new file mode 100644 index 000000000..43094ab38 --- /dev/null +++ b/scripts/setup/setup_solr.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +print_status() { + echo " + +====================================================== + Status Update +------------------------------------------------------ +$1 +====================================================== +" +} + +print_status "Setting up Solr..." + +sudo docker pull bitnami/solr:9.1.1 +sudo docker compose up solr -d +sleep 60 +sudo bash ./direct_indexing/solr/update_solr_cores.sh +sudo docker compose down + +print_status "Done setting up Solr." diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 000000000..3054f4736 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Help +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + echo "Used to start services running through docker. Optionally specify service names." + echo "" + echo "Usage: bash $0 [service name (optional) (up to 9 service names, and -d)]" + exit 0 +fi + +# Start +# Get additional arguments (service names) +I1="$1" +I2="$2" +I3="$3" +I4="$4" +I5="$5" +I6="$6" +I7="$7" +I8="$8" +I9="$9" +I10="$10" +sudo docker compose up $I1 $I2 $I3 $I4 $I5 $I6 $I7 $I8 $I9 $I10 + +echo "Start script is done." diff --git a/scripts/stop.sh b/scripts/stop.sh new file mode 100644 index 000000000..ca85add59 --- /dev/null +++ b/scripts/stop.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Help +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + echo "Used to stop services running through docker. Optionally specify service names." + echo "" + echo "Usage: bash $0 [service name (optional) (up to 9 service names, and -d)]" + exit 0 +fi + +# Start +# Get additional arguments (service names) +I1="$1" +I2="$2" +I3="$3" +I4="$4" +I5="$5" +I6="$6" +I7="$7" +I8="$8" +I9="$9" +I10="$10" +sudo docker compose down $I1 $I2 $I3 $I4 $I5 $I6 $I7 $I8 $I9 $I10 + +echo "Stop script is done." diff --git a/services/nginx/Dockerfile b/services/nginx/Dockerfile deleted file mode 100644 index 6934e3c74..000000000 --- a/services/nginx/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM nginx:latest - -ARG IC_DOMAIN -ARG SOLR_AUTH_ENCODED - -COPY ./nginx.conf /etc/nginx/nginx.conf -COPY ./hostfile /etc/hosts -COPY ./proxy_params /etc/nginx/proxy_params - -# Flower -COPY ./sites-enabled/flower /etc/nginx/sites-enabled/flower -RUN sed -i "s|IC_DOMAIN|${IC_DOMAIN}|g" /etc/nginx/sites-enabled/flower - -# Solr -COPY ./sites-enabled/solr /etc/nginx/sites-enabled/solr -RUN sed -i "s|IC_DOMAIN|${IC_DOMAIN}|g" /etc/nginx/sites-enabled/solr - -# Django -COPY ./sites-enabled/datastore /etc/nginx/sites-enabled/datastore -RUN sed -i "s|IC_DOMAIN|${IC_DOMAIN}|g" /etc/nginx/sites-enabled/datastore -# RUN sed -i "s|http://solr:8983|http://${SOLR_ADMIN_USERNAME}:${SOLR_ADMIN_PASSWORD}@solr:8983|g" /etc/nginx/sites-enabled/datastore -RUN sed -i "s|SOLR_AUTH_ENCODED|${SOLR_AUTH_ENCODED}|g" /etc/nginx/sites-enabled/datastore - -# Redirect -- disabled by default, feel free to enable in production -# COPY ./sites-enabled/iati.cloud-redirect /etc/nginx/sites-enabled/iati.cloud-redirect -# RUN sed -i "s|IC_DOMAIN|${IC_DOMAIN}|g" /etc/nginx/sites-enabled/iati.cloud-redirect - -# Change the user and ownership to user 'iaticloud' -RUN useradd -ms /bin/bash iaticloud -RUN chown -R iaticloud:iaticloud /etc/nginx && \ - chown -R iaticloud:iaticloud /var/cache/nginx && \ - chown -R iaticloud:iaticloud /var/log/nginx -RUN touch /var/run/nginx.pid && \ - chown -R iaticloud:iaticloud /var/run/nginx.pid -USER iaticloud diff --git a/services/nginx/NGINX.md b/services/nginx/NGINX.md deleted file mode 100644 index fe93ad3bb..000000000 --- a/services/nginx/NGINX.md +++ /dev/null @@ -1,9 +0,0 @@ -# NGINX Dockerfile -This image uses the latest NGINX docker image, and then adds the following changes: - -- [nginx.conf](./nginx.conf): changed to include `include /etc/nginx/sites-enabled/*;` in http. -- [hostfile](./hostfile): changed to include `127.0.0.1 localhost *.localhost` to allow for localhost subdomains. -- [proxy_params](./proxy_params): included to support default proxy params. -- [sites-enabled/datastore](./sites-enabled/datastore): enables the django administration panel, and connects incoming requests to solr. (needs elaboration) -- [sites-enabled/flower](./sites-enabled/flower): enables the flower interface. -- [sites-enabled/iati.cloud-redirect](./sites-enabled/iati.cloud-redirect): enables redirects from the original iati.cloud domain to the current domain. diff --git a/services/nginx/hostfile b/services/nginx/hostfile deleted file mode 100644 index 1bac60822..000000000 --- a/services/nginx/hostfile +++ /dev/null @@ -1,7 +0,0 @@ -127.0.0.1 localhost *.localhost -::1 localhost ip6-localhost ip6-loopback -fe00::0 ip6-localnet -ff00::0 ip6-mcastprefix -ff02::1 ip6-allnodes -ff02::2 ip6-allrouters -172.31.0.2 5873e714efa1 \ No newline at end of file diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf deleted file mode 100644 index 4f02f6133..000000000 --- a/services/nginx/nginx.conf +++ /dev/null @@ -1,32 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} \ No newline at end of file diff --git a/services/nginx/proxy_params b/services/nginx/proxy_params deleted file mode 100644 index 11c0f2c43..000000000 --- a/services/nginx/proxy_params +++ /dev/null @@ -1,4 +0,0 @@ -proxy_set_header Host $http_host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; \ No newline at end of file diff --git a/services/nginx/sites-enabled/flower b/services/nginx/sites-enabled/flower deleted file mode 100644 index 88c2db928..000000000 --- a/services/nginx/sites-enabled/flower +++ /dev/null @@ -1,8 +0,0 @@ -server { - server_name flower.IC_DOMAIN; - listen 80; - - location / { - proxy_pass http://celeryflower:5555; - } -} diff --git a/services/nginx/sites-enabled/solr b/services/nginx/sites-enabled/solr deleted file mode 100644 index f4a40b8e1..000000000 --- a/services/nginx/sites-enabled/solr +++ /dev/null @@ -1,8 +0,0 @@ -server { - server_name solr.IC_DOMAIN; - listen 80; - - location / { - proxy_pass http://solr:8983; - } -}