ci.yaml

name: Build and Test

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_call:
  pull_request:

jobs:
  build:
    name: Build Snap
    runs-on: ubuntu-latest
    timeout-minutes: 60
    outputs:
      snap-file: ${{ steps.build-snap.outputs.snap }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install required dependencies
        run: |
          sudo snap install yq

      - name: Upgrade linux deps
        run: |
          sudo apt-get update
          
          # install security updates
          sudo apt-get -s dist-upgrade \
            | grep "^Inst" \
            | grep -i securi \
            | awk -F " " {'print $2'} \
            | xargs sudo apt-get install -y
          
          sudo apt-get autoremove -y
          sudo apt-get clean -y
          sudo snap refresh snapd

      - id: build-snap
        name: Build snap
        uses: snapcore/action-build@v1
        with:
          snapcraft-channel: 7.x/candidate

      - name: Upload built snap job artifact
        uses: actions/upload-artifact@v3
        with:
          name: opensearch_snap_amd64
          path: "opensearch_*.snap"

  test:
    name: Test Snap
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs:
      - build
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Upgrade snapd
        run: |
          sudo snap refresh snapd

      - name: Download snap file
        uses: actions/download-artifact@v3
        with:
          name: opensearch_snap_amd64
          path: .

      - name: Install snap file
        run: |
          version="$(cat snap/snapcraft.yaml | yq .version)"
          
          sudo snap remove --purge opensearch
          sudo snap install opensearch_${version}_amd64.snap --dangerous --jailmode

      - name: Setup the required system configs
        run: |
          sudo sysctl -w vm.swappiness=0
          sudo sysctl -w vm.max_map_count=262144
          sudo sysctl -w net.ipv4.tcp_retries2=5

      - name: Connect required interfaces
        run: |
          sudo snap connect opensearch:log-observe
          sudo snap connect opensearch:mount-observe
          sudo snap connect opensearch:process-control
          sudo snap connect opensearch:system-observe
          sudo snap connect opensearch:sys-fs-cgroup-service
          sudo snap connect opensearch:shmem-perf-analyzer

      - name: Setup and Start OpenSearch
        run: |
          # create the certificates
          sudo snap run opensearch.setup \
              --node-name cm0 \
              --node-roles cluster_manager,data \
              --tls-priv-key-root-pass root1234 \
              --tls-priv-key-admin-pass admin1234 \
              --tls-priv-key-node-pass node1234 \
              --tls-init-setup yes                 # this creates the root and admin certs as well.

          # start opensearch
          sudo snap start opensearch.daemon

          # wait a bit for it to fully initialize
          sleep 15s

          # create the security index
          sudo snap run opensearch.security-init --tls-priv-key-admin-pass=admin1234

      - name: Ensure the cluster is reachable and node created
        run: |
          sudo snap install yq
          
          sudo cp /var/snap/opensearch/current/etc/opensearch/certificates/node-cm0.pem ./
          cert=./node-cm0.pem
          
          # Check node name
          cluster_resp=$(curl --cacert ${cert} -XGET https://localhost:9200 -u 'admin:admin')
          echo -e "Cluster Response: \n ${cluster_resp}"
          node_name=$(echo "${cluster_resp}" | yq -r .name)
          if [ "${node_name}" != "cm0" ]; then
              exit 1
          fi

          # Check cluster health
          health_resp=$(curl --cacert "${cert}" -XGET https://localhost:9200/_cluster/health -u 'admin:admin')
          echo -e "Cluster Health Response: \n ${health_resp}"
          cluster_status=$(echo "${health_resp}" | yq -r .status)
          
          # TODO: once this https://github.com/opensearch-project/OpenSearch/issues/8862 is fixed
          # replace the following condition by "${cluster_status}" != "green"  
          if [ "${cluster_status}" == "red" ]; then
              curl --cacert ${cert} -XGET https://localhost:9200/_cat/shards -u 'admin:admin'
              exit 1
          fi

      - name: Check if Prometheus Exporter and repository plugins are available
        env:
          OPENSEARCH_JAVA_HOME: /snap/opensearch/current/usr/lib/jvm/java-17-openjdk-amd64
          OPENSEARCH_BIN: /snap/opensearch/current/usr/share/opensearch/bin
          OPENSEARCH_PATH_CONF: /var/snap/opensearch/current/etc/opensearch
          OPENSEARCH_HOME: /var/snap/opensearch/current/usr/share/opensearch
          OPENSEARCH_LIB: /var/snap/opensearch/current/usr/share/opensearch/lib
          OPENSEARCH_PATH_CERTS: /var/snap/opensearch/current/etc/opensearch/certificates
        run: |
          # Prometheus Exporter appears in plugins listing
          prometheus_is_there=$(sudo -E "${OPENSEARCH_BIN}"/opensearch-plugin list | grep prometheus-exporter)
          if [ ! "$prometheus_is_there" ]; then
            exit 1
          fi
          # repository-s3 appears in plugins listing
          repository_s3_is_there=$(sudo -E "${OPENSEARCH_BIN}"/opensearch-plugin list | grep repository-s3)
          if [ ! "$repository_s3_is_there" ]; then
            exit 1
          fi
          # repository-gcs appears in plugins listing
          repository_gcs_is_there=$(sudo -E "${OPENSEARCH_BIN}"/opensearch-plugin list | grep repository-gcs)
          if [ ! "$repository_gcs_is_there" ]; then
            exit 1
          fi

          # Prometheus exporter can be queried
          sudo cp /var/snap/opensearch/current/etc/opensearch/certificates/node-cm0.pem ./
          cert=./node-cm0.pem
          resp=$(curl -I --cacert ${cert} -XGET https://localhost:9200/_prometheus/metrics -u 'admin:admin')
          if [[ "$resp" != *"200 OK"* ]]; then
            exit 1
          fi

          # Checking that Prometheus configuration is correct
          prometheus_lines=$(sudo grep prometheus ${OPENSEARCH_PATH_CONF}/opensearch.yml | wc -l)
          if [ "$prometheus_lines" != "4" ]; then
            exit 1
          fi

      - name: Check COS logs slot is available
        run: |
          snap_slot=$( sudo snap connections opensearch | grep content )
          if [ ! "$snap_slot" ]; then
            exit 1
          fi 

      - name: Configure backup plugin 
        env:
          OPENSEARCH_JAVA_HOME: /snap/opensearch/current/usr/lib/jvm/java-17-openjdk-amd64
          OPENSEARCH_BIN: /snap/opensearch/current/usr/share/opensearch/bin
          OPENSEARCH_PATH_CONF: /var/snap/opensearch/current/etc/opensearch
          OPENSEARCH_HOME: /var/snap/opensearch/current/usr/share/opensearch
          OPENSEARCH_LIB: /var/snap/opensearch/current/usr/share/opensearch/lib
          OPENSEARCH_PATH_CERTS: /var/snap/opensearch/current/etc/opensearch/certificates
        run: |
          echo "TEST" | sudo tee -a testkey

          sudo -E "${OPENSEARCH_BIN}"/opensearch-keystore add-file s3.client.default.access_key ${PWD}/testkey
          sudo -E "${OPENSEARCH_BIN}"/opensearch-keystore add-file s3.client.default.secret_key ${PWD}/testkey

          sudo snap restart opensearch.daemon
          sleep 20s

          sudo cp /var/snap/opensearch/current/etc/opensearch/certificates/node-cm0.pem ./
          cert=./node-cm0.pem

          cluster_resp=$(curl --cacert ${cert} -XGET https://localhost:9200 -u 'admin:admin')
          echo -e "Cluster Response: \n ${cluster_resp}"
          node_name=$(echo "${cluster_resp}" | yq -r .name)
          if [ "${node_name}" != "cm0" ]; then
              exit 1
          fi

      - name: Upgrade snap
        run: |
          version="$(cat snap/snapcraft.yaml | yq .version)"
          sudo snap install opensearch_${version}_amd64.snap --dangerous --jailmode

          if [ "$(ls /var/snap/opensearch/x2)" ]; then
              echo "Snap upgraded."
          else
              exit 1
          fi

      - name: Ensure the cluster is reachable and node created after upgrade
        run: |
          # start opensearch after upgrade
          sudo snap restart opensearch.daemon
          # Give some time for the service to come back up
          sleep 20s

          sudo cp /var/snap/opensearch/current/etc/opensearch/certificates/node-cm0.pem ./
          cert=./node-cm0.pem

          # Check node name
          cluster_resp=$(curl --cacert ${cert} -XGET https://localhost:9200 -u 'admin:admin')
          echo -e "Cluster Response: \n ${cluster_resp}"
          node_name=$(echo "${cluster_resp}" | yq -r .name)
          if [ "${node_name}" != "cm0" ]; then
              exit 1
          fi

          # Check cluster health
          health_resp=$(curl --cacert ${cert} -XGET https://localhost:9200/_cluster/health -u 'admin:admin')
          echo -e "Cluster Health Response: \n ${health_resp}"
          # TODO: once this https://github.com/opensearch-project/OpenSearch/issues/8862 is fixed
          # replace the following condition by "${cluster_status}" != "green"  
          if [ "${cluster_status}" == "red" ]; then
              curl --cacert ${cert} -XGET https://localhost:9200/_cat/shards -u 'admin:admin'
              exit 1
          fi

      - name: Remove backup plugin
        env:
          OPENSEARCH_JAVA_HOME: /snap/opensearch/current/usr/lib/jvm/java-17-openjdk-amd64
          OPENSEARCH_BIN: /snap/opensearch/current/usr/share/opensearch/bin
          OPENSEARCH_PATH_CONF: /var/snap/opensearch/current/etc/opensearch
          OPENSEARCH_HOME: /var/snap/opensearch/current/usr/share/opensearch
          OPENSEARCH_LIB: /var/snap/opensearch/current/usr/share/opensearch/lib
          OPENSEARCH_PATH_CERTS: /var/snap/opensearch/current/etc/opensearch/certificates
        run: |
          sudo -E "${OPENSEARCH_BIN}"/opensearch-plugin remove repository-s3
          sudo -E "${OPENSEARCH_BIN}"/opensearch-keystore remove s3.client.default.access_key
          sudo -E "${OPENSEARCH_BIN}"/opensearch-keystore remove s3.client.default.secret_key