diff --git a/backend/db-init/create-test-db.sh b/backend/db-init/create-test-db.sh index 77190622..6d766685 100755 --- a/backend/db-init/create-test-db.sh +++ b/backend/db-init/create-test-db.sh @@ -3,7 +3,7 @@ set -e set -u -psql -v ON_ERROR_STOP=1 --username "$DB_USERNAME" <<-EOSQL +psql -v ON_ERROR_STOP=1 --username "$DB_USERNAME" <<- EOSQL CREATE DATABASE crossfeed_test; GRANT ALL PRIVILEGES ON DATABASE crossfeed_test TO $DB_USERNAME; -EOSQL \ No newline at end of file +EOSQL diff --git a/backend/src/api/search/buildRequest.ts b/backend/src/api/search/buildRequest.ts old mode 100755 new mode 100644 diff --git a/backend/src/api/search/buildRequestFilter.ts b/backend/src/api/search/buildRequestFilter.ts old mode 100755 new mode 100644 diff --git a/backend/tools/build-worker.sh b/backend/tools/build-worker.sh index 8ad5aa04..32facfc3 100755 --- a/backend/tools/build-worker.sh +++ b/backend/tools/build-worker.sh @@ -8,4 +8,4 @@ set -e docker build -t crossfeed-worker -f Dockerfile.worker . -docker build -t pe-worker -f Dockerfile.pe . \ No newline at end of file +docker build -t pe-worker -f Dockerfile.pe . diff --git a/backend/tools/deploy-worker.sh b/backend/tools/deploy-worker.sh index 085583d5..38177bd3 100755 --- a/backend/tools/deploy-worker.sh +++ b/backend/tools/deploy-worker.sh @@ -13,8 +13,8 @@ PE_WORKER_TAG=${1:-pe-staging-worker} ./tools/build-worker.sh aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $AWS_ECR_DOMAIN -docker tag crossfeed-worker:latest $AWS_ECR_DOMAIN/$WORKER_TAG:latest -docker push $AWS_ECR_DOMAIN/$WORKER_TAG:latest +docker tag crossfeed-worker:latest $AWS_ECR_DOMAIN/"$WORKER_TAG":latest +docker push $AWS_ECR_DOMAIN/"$WORKER_TAG":latest -docker tag pe-worker:latest $AWS_ECR_DOMAIN/$PE_WORKER_TAG:latest -docker push $AWS_ECR_DOMAIN/$PE_WORKER_TAG:latest +docker tag pe-worker:latest $AWS_ECR_DOMAIN/"$PE_WORKER_TAG":latest +docker push $AWS_ECR_DOMAIN/"$PE_WORKER_TAG":latest diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 0bcfb88e..6a461dbe 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -1,7 +1,7 @@ #!/bin/bash # Generate database.ini -cat < pe-reports/src/pe_reports/data/database.ini +cat << EOF > pe-reports/src/pe_reports/data/database.ini [postgres] host=${DB_HOST} database=${PE_DB_NAME} @@ -54,4 +54,4 @@ pe_reports_path="${pe_reports_path%/pe-reports}/pe_reports" # Copy database.ini to the module's installation directory cp /app/pe-reports/src/pe_reports/data/database.ini "${pe_reports_path}/data/" -exec "$@" \ No newline at end of file +exec "$@" diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index 4a441751..1579bdcd 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -16,22 +16,22 @@ fi # Function to retrieve a message from RabbitMQ queue get_rabbitmq_message() { curl -s -u "guest:guest" \ - -H "content-type:application/json" \ - -X POST "http://rabbitmq:15672/api/queues/%2F/$SERVICE_QUEUE_URL/get" \ - --data '{"count": 1, "requeue": false, "encoding": "auto", "ackmode": "ack_requeue_false"}' + -H "content-type:application/json" \ + -X POST "http://rabbitmq:15672/api/queues/%2F/$SERVICE_QUEUE_URL/get" \ + --data '{"count": 1, "requeue": false, "encoding": "auto", "ackmode": "ack_requeue_false"}' } - while true; do # Receive message from the Scan specific queue if [ "$IS_LOCAL" = true ]; then echo "Running local RabbitMQ logic..." # Call the function and capture the response - RESPONSE=$(get_rabbitmq_message) && - echo "Response from get_rabbitmq_message: $RESPONSE" && - # Extract the JSON payload from the response body - MESSAGE=$(echo "$RESPONSE" | jq -r '.[0].payload') - MESSAGE=$(echo "$MESSAGE" | sed 's/\\"/"/g') + RESPONSE=$(get_rabbitmq_message) \ + && echo "Response from get_rabbitmq_message: $RESPONSE" \ + && + # Extract the JSON payload from the response body + MESSAGE=$(echo "$RESPONSE" | jq -r '.[0].payload') + MESSAGE=${MESSAGE//\\\"/\"} echo "MESSAGE: $MESSAGE" else @@ -41,7 +41,7 @@ while true; do fi # Check if there are no more messages. If no more, then exit Fargate container - if [ -z "$MESSAGE" ] || [ "$MESSAGE" == "null" ]; then + if [ -z "$MESSAGE" ] || [ "$MESSAGE" == "null" ]; then echo "No more messages in the queue. Exiting." break fi @@ -53,15 +53,15 @@ while true; do ORG=$(echo "$MESSAGE" | jq -r '.Messages[0].Body | fromjson | .org') fi - if [[ "$SERVICE_TYPE" = *"shodan"* ]]; then + if [[ "$SERVICE_TYPE" = *"shodan"* ]]; then COMMAND="pe-source shodan --soc_med_included --org=$ORG" - elif [[ "$SERVICE_TYPE" = *"dnstwist"* ]]; then + elif [[ "$SERVICE_TYPE" = *"dnstwist"* ]]; then COMMAND="pe-source dnstwist --org=$ORG" - elif [[ "$SERVICE_TYPE" = *"hibp"* ]]; then + elif [[ "$SERVICE_TYPE" = *"hibp"* ]]; then COMMAND="pe-source hibp --org=$ORG" - elif [[ "$SERVICE_TYPE" = *"intelx"* ]]; then + elif [[ "$SERVICE_TYPE" = *"intelx"* ]]; then COMMAND="pe-source intelx --org=$ORG --soc_med_included" - elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then + elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included" else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" @@ -71,9 +71,8 @@ while true; do echo "Running $COMMAND" # Run the pe-source command - eval "$COMMAND" && - - cat /app/pe_reports_logging.log + eval "$COMMAND" \ + && cat /app/pe_reports_logging.log # Delete the processed message from the queue if [ "$IS_LOCAL" = true ]; then @@ -84,4 +83,4 @@ while true; do aws sqs delete-message --queue-url "$SERVICE_QUEUE_URL" --receipt-handle "$RECEIPT_HANDLE" echo "Done with $ORG" fi -done \ No newline at end of file +done diff --git a/backend/worker/pe_scripts/runPeAlerts.sh b/backend/worker/pe_scripts/runPeAlerts.sh index 03c1ba36..d8cd96cc 100755 --- a/backend/worker/pe_scripts/runPeAlerts.sh +++ b/backend/worker/pe_scripts/runPeAlerts.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source cybersixgill --cybersix-methods=alerts --soc_med_included \ No newline at end of file +pe-source cybersixgill --cybersix-methods=alerts --soc_med_included diff --git a/backend/worker/pe_scripts/runPeCredentials.sh b/backend/worker/pe_scripts/runPeCredentials.sh index 69c7d6ba..ab0335f1 100755 --- a/backend/worker/pe_scripts/runPeCredentials.sh +++ b/backend/worker/pe_scripts/runPeCredentials.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source cybersixgill --cybersix-methods=credentials --soc_med_included \ No newline at end of file +pe-source cybersixgill --cybersix-methods=credentials --soc_med_included diff --git a/backend/worker/pe_scripts/runPeDnsMonitor.sh b/backend/worker/pe_scripts/runPeDnsMonitor.sh index 7cfe5b26..f8668ea3 100755 --- a/backend/worker/pe_scripts/runPeDnsMonitor.sh +++ b/backend/worker/pe_scripts/runPeDnsMonitor.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source dnsmonitor \ No newline at end of file +pe-source dnsmonitor diff --git a/backend/worker/pe_scripts/runPeDnstwist.sh b/backend/worker/pe_scripts/runPeDnstwist.sh index ca83662e..95da4379 100755 --- a/backend/worker/pe_scripts/runPeDnstwist.sh +++ b/backend/worker/pe_scripts/runPeDnstwist.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports/src/adhoc +cd /app/pe-reports/src/adhoc || return -python3 run_dnstwist.py \ No newline at end of file +python3 run_dnstwist.py diff --git a/backend/worker/pe_scripts/runPeHibp.sh b/backend/worker/pe_scripts/runPeHibp.sh index 0fe9479d..9d8b12a2 100755 --- a/backend/worker/pe_scripts/runPeHibp.sh +++ b/backend/worker/pe_scripts/runPeHibp.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports/src/adhoc +cd /app/pe-reports/src/adhoc || return -python3 hibp_latest.py \ No newline at end of file +python3 hibp_latest.py diff --git a/backend/worker/pe_scripts/runPeIntelx.sh b/backend/worker/pe_scripts/runPeIntelx.sh index ed3ad5ea..de52eaac 100755 --- a/backend/worker/pe_scripts/runPeIntelx.sh +++ b/backend/worker/pe_scripts/runPeIntelx.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source intelx \ No newline at end of file +pe-source intelx diff --git a/backend/worker/pe_scripts/runPeMentions.sh b/backend/worker/pe_scripts/runPeMentions.sh index 3d47692f..cb4a6365 100755 --- a/backend/worker/pe_scripts/runPeMentions.sh +++ b/backend/worker/pe_scripts/runPeMentions.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source cybersixgill --cybersix-methods=mentions --soc_med_included \ No newline at end of file +pe-source cybersixgill --cybersix-methods=mentions --soc_med_included diff --git a/backend/worker/pe_scripts/runPeShodan.sh b/backend/worker/pe_scripts/runPeShodan.sh index 7a73cf60..1338766e 100755 --- a/backend/worker/pe_scripts/runPeShodan.sh +++ b/backend/worker/pe_scripts/runPeShodan.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source shodan --soc_med_included \ No newline at end of file +pe-source shodan --soc_med_included diff --git a/backend/worker/pe_scripts/runPeTopCVEs.sh b/backend/worker/pe_scripts/runPeTopCVEs.sh index d51f246a..89307ab3 100755 --- a/backend/worker/pe_scripts/runPeTopCVEs.sh +++ b/backend/worker/pe_scripts/runPeTopCVEs.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd /app/pe-reports +cd /app/pe-reports || return -pe-source cybersixgill --cybersix-methods=topCVEs --soc_med_included \ No newline at end of file +pe-source cybersixgill --cybersix-methods=topCVEs --soc_med_included diff --git a/backend/worker/shodan.sh b/backend/worker/shodan.sh index d73535fa..7cdba0cd 100644 --- a/backend/worker/shodan.sh +++ b/backend/worker/shodan.sh @@ -8,4 +8,4 @@ echo "Starting Shodan" pe-source shodan --orgs=DHS --soc_med_included -echo "Done" \ No newline at end of file +echo "Done" diff --git a/backend/worker/worker-entry.sh b/backend/worker/worker-entry.sh index 2a510ab8..09ab0780 100755 --- a/backend/worker/worker-entry.sh +++ b/backend/worker/worker-entry.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Sets up an explicit proxy using mitmproxy. set -e @@ -30,4 +31,4 @@ echo "Printing pm2 error logs (if available):" cat ~/pm2-error.log -echo "Done" \ No newline at end of file +echo "Done" diff --git a/build.sh b/build.sh index c8e87c31..db4b9a3c 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,5 @@ +#!/bin/bash + docker-compose down --volumes --rmi all cd backend && npm run build-worker && cd .. && npm start -cd backend && npm run syncdb && npm run syncdb -- -d populate \ No newline at end of file +cd backend && npm run syncdb && npm run syncdb -- -d populate diff --git a/frontend/src/context/SearchProvider/applyDisjunctiveFaceting.js b/frontend/src/context/SearchProvider/applyDisjunctiveFaceting.js old mode 100755 new mode 100644 diff --git a/frontend/src/context/SearchProvider/buildState.js b/frontend/src/context/SearchProvider/buildState.js old mode 100755 new mode 100644 diff --git a/frontend/src/context/SearchProvider/buildStateFacets.js b/frontend/src/context/SearchProvider/buildStateFacets.js old mode 100755 new mode 100644 diff --git a/infrastructure/ssm-agent-install.sh b/infrastructure/ssm-agent-install.sh index f17227f9..90537a70 100644 --- a/infrastructure/ssm-agent-install.sh +++ b/infrastructure/ssm-agent-install.sh @@ -1,7 +1,8 @@ #!/bin/bash + sudo mkdir /tmp/ssm -cd /tmp/ssm +cd /tmp/ssm || return wget https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_amd64/amazon-ssm-agent.deb sudo dpkg -i amazon-ssm-agent.deb sudo systemctl enable amazon-ssm-agent -rm amazon-ssm-agent.deb \ No newline at end of file +rm amazon-ssm-agent.deb diff --git a/rebuild.sh b/rebuild.sh index d41a5fae..ef12dd47 100755 --- a/rebuild.sh +++ b/rebuild.sh @@ -1 +1,3 @@ -docker-compose up -d --build \ No newline at end of file +#!/bin/bash + +docker-compose up -d --build diff --git a/setup-matomo.sh b/setup-matomo.sh index ce8d8122..e49dd9b6 100755 --- a/setup-matomo.sh +++ b/setup-matomo.sh @@ -1,4 +1,6 @@ +#!/bin/bash + # Fixes the trusted_hosts issue to allow Matomo to run on a custom port locally, as # a workaround for https://github.com/matomo-org/matomo/issues/9549. # Run this after initially setting up Matomo through the UI. -docker-compose exec matomo sed -i 's/"localhost"/"localhost:3000"/g' /var/www/html/config/config.ini.php \ No newline at end of file +docker-compose exec matomo sed -i 's/"localhost"/"localhost:3000"/g' /var/www/html/config/config.ini.php