diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 6e78a086..3dba4e20 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_linting.yml b/.github/workflows/test_linting.yml index cbb0a4ca..fa0da927 100644 --- a/.github/workflows/test_linting.yml +++ b/.github/workflows/test_linting.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3. uses: actions/setup-python@v4 diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index 75de6cc5..3bd80ab0 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3. uses: actions/setup-python@v4 @@ -70,7 +70,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 @@ -117,7 +117,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v4 @@ -157,7 +157,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 @@ -197,7 +197,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.9.14 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml index 82741d96..45544ba6 100644 --- a/.github/workflows/test_role_beats.yml +++ b/.github/workflows/test_role_beats.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml index 606b92e2..b0da7d9b 100644 --- a/.github/workflows/test_role_elasticsearch.yml +++ b/.github/workflows/test_role_elasticsearch.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml index 2d75cb63..914464eb 100644 --- a/.github/workflows/test_role_kibana.yml +++ b/.github/workflows/test_role_kibana.yml @@ -56,7 +56,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index e13e8384..14198928 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index ab3441c0..7963ce6b 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index ad50534f..525f1691 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml index 8cba6694..1e836b59 100644 --- a/molecule/elasticsearch_default/converge.yml +++ b/molecule/elasticsearch_default/converge.yml @@ -12,6 +12,7 @@ elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticsearch_heap: "1" + elasticstack_no_log: false tasks: - name: Include Elastics repos role ansible.builtin.include_role: diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 61ec7344..009b0fdd 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -12,8 +12,9 @@ vars: elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true + elasticsearch_monitoring_enabled: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticsearch_heap: "1" + elasticsearch_heap: "2" elasticstack_full_stack: true elasticstack_no_log: false logstash_pipeline_unsafe_shutdown: true diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index 671296ab..122ee248 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -10,7 +10,6 @@ platforms: groups: - beats - logstash - - kibana - elasticsearch image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} @@ -22,7 +21,6 @@ platforms: - name: "elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}" groups: - beats - - logstash - kibana - elasticsearch image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest" diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index abceef3f..3a2e8430 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -59,42 +59,46 @@ msg: "Elasticsearch received {{ logstash_count.stdout }} events so far" when: "'elasticsearch' in group_names" - - name: fetch kibana.yml - ansible.builtin.command: cat /etc/kibana/kibana.yml - register: kibanayml - - - name: Show kibana.yml - ansible.builtin.debug: - var: kibanayml.stdout_lines - - - name: Check for Kibana port - ansible.builtin.wait_for: - port: 5601 - timeout: 120 - - - name: Connect to Kibana - ansible.builtin.command: - curl - -s - -u elastic:{{ elastic_pass.stdout }} - http://{{ ansible_hostname }}:5601/api/status - register: curl_out - failed_when: - - "'green' not in curl_out.stdout" - - "'Elasticsearch is available' not in curl_out.stdout" - - # The following might be nicer but doesn't work - #- name: Connect to Kibana - # ansible.builtin.uri: - # url: http://ansible-role-kibana_full_stack:5601/api/status - # user: elastic - # password: "{{ elastic_password.stdout }}" - # return_content: yes - # register: kibana_status - # #failed_when: "'"title": "Green"' not in kibana_status.content" - # failed_when: "'Green' not in kibana_status.content" - - - name: Health check + - name: Run Kibana checks + when: "'kibana' in group_names" + block: + + - name: Fetch kibana.yml + ansible.builtin.command: cat /etc/kibana/kibana.yml + register: kibanayml + + - name: Show kibana.yml + ansible.builtin.debug: + var: kibanayml.stdout_lines + + - name: Check for Kibana port + ansible.builtin.wait_for: + port: 5601 + timeout: 120 + + - name: Connect to Kibana + ansible.builtin.command: + curl + -s + -u elastic:{{ elastic_pass.stdout }} + http://{{ ansible_hostname }}:5601/api/status + register: curl_out + failed_when: + - "'green' not in curl_out.stdout" + - "'Elasticsearch is available' not in curl_out.stdout" + + # The following might be nicer but doesn't work + #- name: Connect to Kibana + # ansible.builtin.uri: + # url: http://ansible-role-kibana_full_stack:5601/api/status + # user: elastic + # password: "{{ elastic_password.stdout }}" + # return_content: yes + # register: kibana_status + # #failed_when: "'"title": "Green"' not in kibana_status.content" + # failed_when: "'Green' not in kibana_status.content" + + - name: Elasticsearch health check ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health method: GET @@ -110,7 +114,7 @@ delay: 10 when: groups['elasticsearch'] | length > 1 - - name: Node check + - name: Elasticsearch Node check ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes method: GET diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index f242c0a8..9d521bb1 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -82,3 +82,10 @@ - name: Import Metricbeat tasks ansible.builtin.import_tasks: metricbeat.yml when: beats_metricbeat | bool + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index bcadfb3d..3f531606 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -48,6 +48,12 @@ elasticsearch_cert_expiration_buffer: 30 elasticstack_ca_will_expire_soon: false elasticsearch_cert_will_expire_soon: false +# only used internally +elasticsearch_freshstart: + changed: false +elasticsearch_freshstart_security: + changed: false + # "global" variables for all roles elasticstack_release: 8 diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index b39f884a..ff3b5ab5 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -5,7 +5,10 @@ name: elasticsearch state: restarted daemon_reload: yes - when: elasticsearch_enable | bool + when: + - elasticsearch_enable | bool + - not elasticsearch_freshstart.changed | bool + - not elasticsearch_freshstart_security.changed | bool - name: Restart kibana if available for elasticsearch certificates ansible.builtin.include_tasks: handlers/restart_kibana.yml diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 0b1a95ac..48bcb2aa 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -352,25 +352,31 @@ name: elasticsearch state: started enabled: yes + register: elasticsearch_freshstart_security - name: Wait for all instances to start ansible.builtin.include_tasks: wait_for_instance.yml loop: "{{ groups['elasticsearch'] }}" -- name: Force all notified handlers to run at this point, not waiting for normal sync points - ansible.builtin.meta: flush_handlers - tags: - - certificates - - renew_ca - - renew_es_cert - -- name: Wait for all instances to start - ansible.builtin.include_tasks: wait_for_instance.yml - loop: "{{ groups['elasticsearch'] }}" - tags: - - certificates - - renew_ca - - renew_es_cert +- name: Restart if Elasticsearch was already running + when: + - not elasticsearch_freshstart.changed | bool + - not elasticsearch_freshstart_security.changed | bool + block: + - name: Force all notified handlers to run at this point, not waiting for normal sync points + ansible.builtin.meta: flush_handlers + tags: + - certificates + - renew_ca + - renew_es_cert + + - name: Wait for all instances to start + ansible.builtin.include_tasks: wait_for_instance.yml + loop: "{{ groups['elasticsearch'] }}" + tags: + - certificates + - renew_ca + - renew_es_cert - name: Check for passwords being set ansible.builtin.stat: @@ -383,6 +389,25 @@ elasticsearch_http_protocol: "https" when: elasticsearch_http_security +- name: Check for API with bootstrap password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}" + user: elastic + password: "{{ elasticsearch_bootstrap_pw }}" + validate_certs: false + register: elasticsearch_api_status_bootstrap + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - not elasticsearch_passwords_file.stat.exists | bool + - groups['elasticsearch'] | length > 1 + until: elasticsearch_api_status_bootstrap.json.cluster_name is defined + retries: 5 + delay: 10 + +# We need this check twice. One to wait for the API to be actually available. And a second time to +# check the actual return code. Should not cause a huge delay. + - name: Check for cluster status with bootstrap password ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" @@ -410,6 +435,57 @@ delegate_to: "{{ elasticstack_ca }}" when: elasticsearch_passwords_file.stat.exists | bool +- name: Check for API availability with elastic password + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}" + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: false + register: elasticsearch_api_status + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - elasticsearch_passwords_file.stat.exists | bool + - groups['elasticsearch'] | length > 1 + until: elasticsearch_api_status.json.cluster_name is defined + retries: 20 + delay: 10 + +- name: Work around low ressources on CI/CD nodes + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" + block: + # Free up some space to let elsticsearch allocate replica in GitHub Action + - name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + + - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + curl + -k + -X PUT + "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings" + -H 'Content-Type: application/json' -d + ' + { + "persistent": { + "cluster.routing.allocation.disk.watermark.low": "97%", + "cluster.routing.allocation.disk.watermark.high": "98%", + "cluster.routing.allocation.disk.watermark.flood_stage": "99%", + "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%" + } + } + ' + changed_when: false + no_log: "{{ elasticstack_no_log }}" + when: + - elasticstack_password.stdout is defined + +# We need this check twice. One to wait for the API to be actually available. And a second time to +# check the actual return code. Should not cause a huge delay. + - name: Check for cluster status with elastic password ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 4b813117..0632f870 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -178,9 +178,8 @@ when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" # Free up some space to let elsticsearch allocate replica in GitHub Action -- name: Remove cache # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; +- name: Remove cache + ansible.builtin.command: > rm -rf /var/cache/* changed_when: false when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" @@ -200,6 +199,7 @@ name: elasticsearch state: started enabled: yes + register: elasticsearch_freshstart - name: Handle cluster setup without security when: not elasticsearch_security | bool @@ -237,8 +237,6 @@ group: root mode: 0644 backup: "{{ elasticsearch_config_backup }}" - notify: - - Restart Elasticsearch when: elasticsearch_manage_yaml | bool - name: Show Info about heap diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml index cc21f125..7aa06f7b 100644 --- a/roles/kibana/defaults/main.yml +++ b/roles/kibana/defaults/main.yml @@ -18,6 +18,9 @@ kibana_cert_will_expire_soon: false kibana_sniff_on_start: false kibana_sniff_on_connection_fault: false +kibana_freshstart: + changed: false + # "global" variables for all roles elasticstack_release: 8 elasticstack_full_stack: true diff --git a/roles/kibana/handlers/main.yml b/roles/kibana/handlers/main.yml index 81ffa146..532d014a 100644 --- a/roles/kibana/handlers/main.yml +++ b/roles/kibana/handlers/main.yml @@ -4,3 +4,5 @@ ansible.builtin.service: name: kibana state: restarted + when: + - not kibana_freshstart.changed | bool diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index b5b3ebb9..74e91344 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -82,6 +82,7 @@ state: started enabled: yes when: kibana_enable | bool + register: kibana_freshstart # the following is useful when running tests or extra tasks that need to # have Kibana running. Escape it on Rocky8, because it gets time out with Elastic 8 @@ -90,3 +91,10 @@ ansible.builtin.wait_for: host: localhost port: 5601 + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker" diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 8ff97804..1941e792 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -73,6 +73,11 @@ logstash_pipeline_identifier: true logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false +# Only for internal use + +logstash_freshstart: + changed: false + elasticstack_ca_dir: /opt/es-ca elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_ca_pass: PleaseChangeMe diff --git a/roles/logstash/handlers/main.yml b/roles/logstash/handlers/main.yml index eb55a868..08b3b71b 100644 --- a/roles/logstash/handlers/main.yml +++ b/roles/logstash/handlers/main.yml @@ -4,7 +4,9 @@ ansible.builtin.service: name: logstash state: restarted - when: logstash_enable | bool + when: + - logstash_enable | bool + - not logstash_freshstart.changed | bool - name: Restart Logstash noauto ansible.builtin.service: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 1dcee30b..a1d1b3de 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -230,3 +230,11 @@ state: started enabled: yes when: logstash_enable | bool + register: logstash_freshstart + +# Free up some space to let elsticsearch allocate replica in GitHub Action +- name: Remove cache + ansible.builtin.command: > + rm -rf /var/cache/* + changed_when: false + when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"