LME 2 Dashboard Update and Bug Fixes #402
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cluster Run - Minimega | |
| on: | |
| pull_request: | |
| branches: | |
| - '*' | |
| workflow_dispatch: | |
| inputs: | |
| azure_region: | |
| description: 'Azure region to deploy resources' | |
| required: true | |
| default: 'centralus' | |
| type: choice | |
| options: | |
| - centralus | |
| - eastus | |
| - eastus2 | |
| - westus | |
| - westus2 | |
| - westus3 | |
| - northcentralus | |
| - southcentralus | |
| - canadacentral | |
| - canadaeast | |
| - uksouth | |
| - ukwest | |
| - northeurope | |
| - westeurope | |
| jobs: | |
| build-and-test-cluster: | |
| runs-on: self-hosted | |
| env: | |
| UNIQUE_ID: ${{ github.run_id }}-${{ github.run_number }} | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| IP_ADDRESS: "" | |
| LS1_IP: "" | |
| elastic: "" | |
| AZURE_IP: "" | |
| MINIMEGA_IP: "" | |
| ENROLLMENT_TOKEN: "" | |
| ES_PASSWORD: "" | |
| KIBANA_PASSWORD: "" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/[email protected] | |
| - name: Setup environment variables | |
| run: | | |
| PUBLIC_IP=$(curl -s https://api.ipify.org) | |
| echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV | |
| - name: Get branch name | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV | |
| else | |
| echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV | |
| fi | |
| - name: Set the environment for docker-compose | |
| run: | | |
| cd testing/v2/development | |
| # Get the UID and GID of the current user | |
| echo "HOST_UID=$(id -u)" > .env | |
| echo "HOST_GID=$(id -g)" >> .env | |
| cat .env | |
| - name: Build pipeline container | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} build pipeline --no-cache | |
| - name: Start pipeline container | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline | |
| - name: Install Python requirements | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers/azure && \ | |
| pip install -r requirements.txt | |
| " | |
| - name: Build an Azure instance | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| python3 ./azure/build_azure_linux_network.py \ | |
| -g pipe-${{ env.UNIQUE_ID }} \ | |
| -s ${{ env.IP_ADDRESS }}/32 \ | |
| -vs Standard_D8_v4 \ | |
| -l ${{ inputs.azure_region || 'centralus' }} \ | |
| -ast 23:00 \ | |
| -y | |
| " | |
| - name: Install minimega on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| sleep 30 | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./minimega/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" | |
| " | |
| - name: Install Linux in minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./ubuntu_qcow_maker/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" | |
| " | |
| - name: Check if linux is running in minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| sleep 120 && \ | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ssh lme-user@\$IP_ADDRESS 'sudo /opt/minimega/bin/minimega -e vm info' | |
| " | |
| - name: Get Azure and Minimega IP addresses | |
| run: | | |
| cd testing/v2/development | |
| AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt") | |
| echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV | |
| echo "Azure IP:$AZURE_IP" | |
| MINIMEGA_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true .filter name=\"linux-runner\" vm info | jq -r \".[].Data[].Networks[].IP4\"' | |
| " ) | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "Failed to get Minimega IP. Exit code: $EXIT_CODE" >&2 | |
| exit 1 | |
| fi | |
| if [ -z "$MINIMEGA_IP" ]; then | |
| echo "Minimega IP is empty" >&2 | |
| exit 1 | |
| fi | |
| echo "MINIMEGA_IP=$MINIMEGA_IP" >> $GITHUB_ENV | |
| echo "Azure IP:$AZURE_IP Minimega IP:$MINIMEGA_IP" | |
| - name: Run a command in Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la' | |
| " | |
| - name: Install LME on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ./install_v2/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" ${{ env.BRANCH_NAME }} | |
| " | |
| - name: Retrieve Elastic password | |
| env: | |
| AZURE_IP: ${{ env.AZURE_IP }} | |
| run: | | |
| cd testing/v2/development | |
| ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$elastic'" | tail -n 1 | tr -d '\n') | |
| echo "::add-mask::$ES_PASSWORD" | |
| echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV | |
| echo "Elastic password retrieved successfully: $ES_PASSWORD" | |
| KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$kibana_system'" | tail -n 1 | tr -d '\n') | |
| echo "::add-mask::$KIBANA_PASSWORD" | |
| echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV | |
| echo "Kibana password retrieved successfully: $KIBANA_PASSWORD" | |
| - name: Install test requirements on Azure instance | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ | |
| ssh lme-user@\$IP_ADDRESS 'whoami && hostname && \ | |
| wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ | |
| sudo apt install -y ./google-chrome-stable_current_amd64.deb && \ | |
| cd /home/lme-user/LME/testing/tests && \ | |
| python3 -m venv venv && \ | |
| source venv/bin/activate && \ | |
| pip install -r requirements.txt ' | |
| " | |
| - name: Retrieve Elastic policy ID and enrollment token | |
| env: | |
| KIBANA_URL: "https://localhost:5601" | |
| ES_USERNAME: "elastic" | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| # Retrieve policy ID | |
| POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} ' | |
| curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \ | |
| -H \"kbn-xsrf: true\" \ | |
| -H \"Content-Type: application/json\" | | |
| jq -r '.items[0].id' | |
| ' | |
| ") | |
| echo "Retrieved Policy ID: $POLICY_ID" | |
| # Retrieve enrollment token using the policy ID | |
| ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} ' | |
| curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \ | |
| -H \"kbn-xsrf: true\" \ | |
| -H \"Content-Type: application/json\" \ | |
| -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" | | |
| jq -r .item.api_key | |
| ' | |
| ") | |
| echo "Retrieved enrollment token: $ENROLLMENT_TOKEN" | |
| # Mask the enrollment token in logs and set it as an environment variable | |
| echo "::add-mask::$ENROLLMENT_TOKEN" | |
| echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV | |
| echo "Policy ID and Enrollment Token retrieved successfully" | |
| - name: Copy the Elastic Agent installer to Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ | |
| 'sudo scp -p -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/lib/install_agent_linux.sh vmuser@${{ env.MINIMEGA_IP }}:~' | |
| " | |
| - name: Run a command in Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la' | |
| " | |
| - name: Install the Elastic Agent in Minimega | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} chmod +x ./install_agent_linux.sh ' && \ | |
| ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ./install_agent_linux.sh --token ${{ env.ENROLLMENT_TOKEN }}' | |
| " | |
| - name: Check if the Elastic agent is reporting | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| run: | | |
| sleep 360 | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ | |
| 'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh' | |
| " | |
| - name: Run api tests on Azure instance | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ | |
| echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ | |
| echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ | |
| echo elastic=\"$ES_PASSWORD\" >> .env && \ | |
| source venv/bin/activate && \ | |
| pytest -v api_tests/' | |
| " | |
| - name: Run selenium tests on Azure instance | |
| env: | |
| ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ | |
| echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ | |
| echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ | |
| echo elastic=\"$ES_PASSWORD\" >> .env && \ | |
| source venv/bin/activate && \ | |
| pytest -v selenium_tests/' | |
| " | |
| - name: Cleanup Azure resources | |
| if: always() | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT | |
| az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait | |
| " | |
| - name: Stop and remove containers | |
| if: always() | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} down | |
| docker system prune -af |