LME 2 Dashboard Update and Bug Fixes

.github/workflows/cluster.yml

@@ -1,10 +1,31 @@
 name: Cluster Run - Minimega
 
 on:
+  pull_request:
+    # branches:
+    #   - '*'
   workflow_dispatch:
-  # pull_request:
-  #  branches:
-  #    - '*'
+    inputs:
+      azure_region:
+        description: 'Azure region to deploy resources'
+        required: true
+        default: 'centralus'
+        type: choice
+        options:
+          - centralus
+          - eastus
+          - eastus2
+          - westus
+          - westus2
+          - westus3
+          - northcentralus
+          - southcentralus
+          - canadacentral
+          - canadaeast
+          - uksouth
+          - ukwest
+          - northeurope
+          - westeurope
 
 jobs:
   build-and-test-cluster:
@@ -84,7 +105,7 @@ jobs:
                 -g pipe-${{ env.UNIQUE_ID }} \
                 -s ${{ env.IP_ADDRESS }}/32 \
                 -vs Standard_D8_v4 \
-                -l centralus \
+                -l ${{ inputs.azure_region || 'centralus' }} \
                 -ast 23:00 \
                 -y
             "
@@ -199,7 +220,7 @@ jobs:
           # Retrieve policy ID
           POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
             ssh lme-user@${{ env.AZURE_IP }} '
-              curl -k -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
+              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
                 -H \"kbn-xsrf: true\" \
                 -H \"Content-Type: application/json\" |
               jq -r '.items[0].id'
@@ -210,7 +231,7 @@ jobs:
           # Retrieve enrollment token using the policy ID
           ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
             ssh lme-user@${{ env.AZURE_IP }} '
-              curl -k -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
+              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
                 -H \"kbn-xsrf: true\" \
                 -H \"Content-Type: application/json\" \
                 -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" |

.github/workflows/linux_only.yml

@@ -118,6 +118,7 @@ jobs:
         KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }}
         AZURE_IP: ${{ env.AZURE_IP }}
       run: |
+        sleep 360
         cd testing/v2/development
         docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
           cd /home/lme-user/LME/testing/v2/installers && \

ansible/post_install_local.yml

@@ -7,7 +7,7 @@
 
   vars:
     headers:
-      kbn-version: "8.12.2"
+      kbn-version: "8.15.3"
       kbn-xsrf: "kibana"
       Content-Type: "application/json"
     max_retries: 60
@@ -465,7 +465,7 @@
       register: dashboards
 
     - name: Upload dashboards to Kibana
-      shell: 'curl -X POST -k --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
+      shell: 'curl -X POST -kL --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
       loop: "{{ dashboards.files | map(attribute='path') | list }}"
       args:
         warn: false
@@ -610,7 +610,7 @@
       register: dashboards
 
     - name: Upload dashboards to Kibana
-      shell: 'curl -X POST -k --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
+      shell: 'curl -X POST -kL --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
       args:
         warn: false
       loop: "{{ dashboards.files | map(attribute='path') | list }}"
@@ -901,7 +901,7 @@
       shell: >
         curl -X POST "{{ local_es_url }}/_security/user/readonly_user"
         -u "{{ elastic_username }}:{{ elastic_password }}"
-        -k
+        -kL
         -H "Content-Type: application/json"
         -d '{
               "password": "{{ read_only_password.stdout }}",

ansible/set_fleet.yml

@@ -6,7 +6,7 @@
 
   vars:
     headers:
-      kbn-version: "8.12.2"
+      kbn-version: "8.15.3"
       kbn-xsrf: "kibana"
       Content-Type: "application/json"
     max_retries: 60
@@ -109,7 +109,7 @@
         max_attempts=30
         delay=10
         while [ $attempt -lt $max_attempts ]; do
-          response=$(curl -s -o /dev/null -w "%{http_code}" -k -u elastic:{{ elastic_password }} {{ local_kbn_url }}/api/fleet/agents/setup)
+          response=$(curl -s -o /dev/null -w "%{http_code}" -kL -u elastic:{{ elastic_password }} {{ local_kbn_url }}/api/fleet/agents/setup)
           if [ "$response" = "200" ]; then
             echo "Fleet API is ready. Proceeding with configuration..."
             exit 0

config/containers.txt

@@ -1,6 +1,6 @@
 docker.io/caddy:2-alpine
-docker.elastic.co/elasticsearch/elasticsearch:8.12.2
-docker.elastic.co/beats/elastic-agent:8.12.2
-docker.elastic.co/kibana/kibana:8.12.2
-docker.io/wazuh/wazuh-manager:4.7.5
+docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+docker.elastic.co/beats/elastic-agent:8.15.3
+docker.elastic.co/kibana/kibana:8.15.3
+docker.io/wazuh/wazuh-manager:4.9.1
 docker.io/jertel/elastalert2:2.20.0

config/elastalert2/misc/smtp_auth.yml

@@ -0,0 +1,2 @@
+user: "[email protected]"
+password: "giyq caym zqiw chje" #this is your app password if using gmail

config/elastalert2/rules/example-email-rule.yml

@@ -0,0 +1,21 @@
+name: EMAIL
+type: frequency
+index: wazuh-*
+num_events: 1
+timeframe: 
+  minutes: 1
+filter:
+- query:
+    match_phrase:
+      agent.ip: "10.1.0.4"
+alert: email
+alert_text: "ASDFASDF"
+alert_text_type: alert_text_only
+email:
+  - "[email protected]"
+smtp_ssl: true
+smtp_port: 465
+smtp_host: "smtp.gmail.com"
+from_addr: "[email protected]"
+smtp_auth_file: /opt/elastalert/misc/smtp_auth.yml
+

config/example.env

@@ -23,7 +23,7 @@ LOCAL_ES_URL=https://127.0.0.1:9200
 #################
 
 # Version of Elastic products
-STACK_VERSION=8.12.2
+STACK_VERSION=8.15.3
 # Testing pre-releases? Use the SNAPSHOT option below:
 # STACK_VERSION=8.11.0-SNAPSHOT
 #

config/setup/acct-init.sh

@@ -15,7 +15,7 @@ if [ ! -f "${CERTS_DIR}/ACCOUNTS_CREATED" ]; then
   until curl -s --cacert config/certs/ca/ca.crt https://lme-elasticsearch:9200 | grep -q "missing authentication credentials"; do echo "WAITING"; sleep 30; done;
 
   echo "Setting kibana_system password";
-  until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://lme-elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 2; done;
+  until curl -L -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://lme-elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 2; done;
 
   echo "All done!" | tee "${CERTS_DIR}/ACCOUNTS_CREATED" ;
 fi

config/wazuh_cluster/wazuh_manager.conf

@@ -101,91 +101,26 @@
     <skip_nfs>yes</skip_nfs>
   </sca>
 
-  <vulnerability-detector>
-    <enabled>yes</enabled>
-    <interval>5m</interval>
-    <min_full_scan_interval>6h</min_full_scan_interval>
-    <run_on_start>yes</run_on_start>
-
-    <!-- Ubuntu OS vulnerabilities -->
-    <provider name="canonical">
-      <enabled>yes</enabled>
-      <os>trusty</os>
-      <os>xenial</os>
-      <os>bionic</os>
-      <os>focal</os>
-      <os>jammy</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Debian OS vulnerabilities -->
-    <provider name="debian">
-      <enabled>no</enabled>
-      <os>buster</os>
-      <os>bullseye</os>
-      <os>bookworm</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- RedHat OS vulnerabilities -->
-    <provider name="redhat">
-      <enabled>no</enabled>
-      <os>5</os>
-      <os>6</os>
-      <os>7</os>
-      <os>8</os>
-      <os>9</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Amazon Linux OS vulnerabilities -->
-    <provider name="alas">
-      <enabled>no</enabled>
-      <os>amazon-linux</os>
-      <os>amazon-linux-2</os>
-      <os>amazon-linux-2022</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- SUSE OS vulnerabilities -->
-    <provider name="suse">
-      <enabled>no</enabled>
-      <os>11-server</os>
-      <os>11-desktop</os>
-      <os>12-server</os>
-      <os>12-desktop</os>
-      <os>15-server</os>
-      <os>15-desktop</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Arch OS vulnerabilities -->
-    <provider name="arch">
-      <enabled>no</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Alma Linux OS vulnerabilities -->
-    <provider name="almalinux">
-      <enabled>no</enabled>
-      <os>8</os>
-      <os>9</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Windows OS vulnerabilities -->
-    <provider name="msu">
-      <enabled>yes</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
+  <vulnerability-detection>
+     <enabled>yes</enabled>
+     <index-status>yes</index-status>
+     <feed-update-interval>60m</feed-update-interval>
+  </vulnerability-detection>
 
-    <!-- Aggregate vulnerabilities -->
-    <provider name="nvd">
-      <enabled>yes</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
-
-  </vulnerability-detector>
+  <indexer>
+    <enabled>yes</enabled>
+    <hosts>
+      <!-- TODO make this use the official one we set in environment variables -->
+      <host>https://lme-elasticsearch:9200</host>
+    </hosts>
+    <ssl>
+      <certificate_authorities>
+        <ca>/etc/wazuh-manager/certs/ca/ca.crt</ca>
+      </certificate_authorities>
+      <certificate>/etc/wazuh-manager/certs/wazuh-manager/wazuh-manager.crt</certificate>
+      <key>/etc/wazuh-manager/certs/wazuh-manager/wazuh-manager.key</key>
+    </ssl>
+  </indexer>
 
   <!-- File integrity monitoring -->
   <syscheck>