From 06c54bf4f70d3a397d86667dae5ed1dcf648612b Mon Sep 17 00:00:00 2001
From: Wojciech Zyla <wzya@splunk.com>
Date: Thu, 31 Oct 2024 17:48:27 +0100
Subject: [PATCH] feat: add documentation and configuration for deploying SC4S
 on RKE2 with ansible

feat: remove ip and personal uid
---
 ansible/inventory/inventory_rke2.yml          |   5 +
 ansible/inventory/inventory_rke2_ha.yml       |  23 ++++
 ansible/playbooks/rke2.yml                    |  65 ++++++++++
 ansible/resources/metallb-config.yaml         |  15 +++
 ansible/tasks/rke2/add_token_to_config.yml    |   8 ++
 ansible/tasks/rke2/copy_config.yml            |  14 +++
 ansible/tasks/rke2/deploy_app.yml             |  21 ++++
 ansible/tasks/rke2/deploy_secrets.yml         |  29 +++++
 ansible/tasks/rke2/get_registration_token.yml |  16 +++
 ansible/tasks/rke2/install_agents.yml         |   9 ++
 ansible/tasks/rke2/install_first_server.yml   |  11 ++
 ansible/tasks/rke2/install_helm_repo.yml      |   6 +
 ansible/tasks/rke2/install_metallb.yml        |  27 ++++
 ansible/tasks/rke2/install_other_servers.yml  |  11 ++
 ansible/tasks/rke2/provide_kubectl.yml        |  14 +++
 docs/gettingstarted/ansible-rke2.md           | 117 ++++++++++++++++++
 mkdocs.yml                                    |   1 +
 17 files changed, 392 insertions(+)
 create mode 100644 ansible/inventory/inventory_rke2.yml
 create mode 100644 ansible/inventory/inventory_rke2_ha.yml
 create mode 100644 ansible/playbooks/rke2.yml
 create mode 100644 ansible/resources/metallb-config.yaml
 create mode 100644 ansible/tasks/rke2/add_token_to_config.yml
 create mode 100644 ansible/tasks/rke2/copy_config.yml
 create mode 100644 ansible/tasks/rke2/deploy_app.yml
 create mode 100644 ansible/tasks/rke2/deploy_secrets.yml
 create mode 100644 ansible/tasks/rke2/get_registration_token.yml
 create mode 100644 ansible/tasks/rke2/install_agents.yml
 create mode 100644 ansible/tasks/rke2/install_first_server.yml
 create mode 100644 ansible/tasks/rke2/install_helm_repo.yml
 create mode 100644 ansible/tasks/rke2/install_metallb.yml
 create mode 100644 ansible/tasks/rke2/install_other_servers.yml
 create mode 100644 ansible/tasks/rke2/provide_kubectl.yml
 create mode 100644 docs/gettingstarted/ansible-rke2.md

diff --git a/ansible/inventory/inventory_rke2.yml b/ansible/inventory/inventory_rke2.yml
new file mode 100644
index 0000000000..0d350106f6
--- /dev/null
+++ b/ansible/inventory/inventory_rke2.yml
@@ -0,0 +1,5 @@
+control_nodes:
+  hosts:
+    token_node:
+      ansible_host:
+      config_file:
\ No newline at end of file
diff --git a/ansible/inventory/inventory_rke2_ha.yml b/ansible/inventory/inventory_rke2_ha.yml
new file mode 100644
index 0000000000..b3692a4d9a
--- /dev/null
+++ b/ansible/inventory/inventory_rke2_ha.yml
@@ -0,0 +1,23 @@
+control_nodes:
+  hosts:
+    token_node:
+      ansible_host:
+      config_file:
+    optional_control_node_1:
+      ansible_host:
+      config_file:
+    optional_control_node_2:
+      ansible_host:
+      config_file:
+
+agent_nodes:
+  hosts:
+    optional_agent_1:
+      ansible_host:
+      config_file:
+    optional_agent_2:
+      ansible_host:
+      config_file:
+    optional_agent_3:
+      ansible_host:
+      config_file:
\ No newline at end of file
diff --git a/ansible/playbooks/rke2.yml b/ansible/playbooks/rke2.yml
new file mode 100644
index 0000000000..73bf27e1d4
--- /dev/null
+++ b/ansible/playbooks/rke2.yml
@@ -0,0 +1,65 @@
+---
+- name: Copy rke2 configuration files
+  hosts: all
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/copy_config.yml
+
+- name: Install and run rke2-server.service on first control node
+  hosts: control_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/install_first_server.yml
+
+- name: Get node-token from a control node
+  hosts: control_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/get_registration_token.yml
+
+- name: Add node-token to other control nodes and agent nodes configuration
+  hosts: control_nodes:agent_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/add_token_to_config.yml
+
+- name: Install and run rke2-server.service on rest of the control nodes
+  hosts: control_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/install_other_servers.yml
+
+- name: Install and run rke2-agent.service on agent nodes
+  hosts: agent_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/install_agents.yml
+
+- name: Make kubectl executable available for ansible_user
+  hosts: control_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/provide_kubectl.yml
+
+- name: Deploy k8s secrets
+  hosts: control_nodes
+  become: true
+  tasks:
+    - include_tasks: ../tasks/rke2/deploy_secrets.yml
+
+- name: Install metallb
+  hosts: control_nodes
+  tasks:
+    - include_tasks: ../tasks/rke2/install_metallb.yml
+
+- name: Install SC4S helm repo
+  hosts: control_nodes
+  tasks:
+    - include_tasks: ../tasks/rke2/install_helm_repo.yml
+
+- name: Deploy SC4S app
+  hosts: control_nodes
+  tasks:
+    - include_tasks: ../tasks/rke2/deploy_app.yml
+
+
diff --git a/ansible/resources/metallb-config.yaml b/ansible/resources/metallb-config.yaml
new file mode 100644
index 0000000000..6adde98b1a
--- /dev/null
+++ b/ansible/resources/metallb-config.yaml
@@ -0,0 +1,15 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  namespace: metallb
+  name: my-ip-pool
+spec:
+  addresses:
+  # Configure address pool for metallb
+  #- 1.2.3.4/32
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  namespace: metallb
+  name: l2-advertisement
diff --git a/ansible/tasks/rke2/add_token_to_config.yml b/ansible/tasks/rke2/add_token_to_config.yml
new file mode 100644
index 0000000000..b002a84e29
--- /dev/null
+++ b/ansible/tasks/rke2/add_token_to_config.yml
@@ -0,0 +1,8 @@
+---
+- name: Add rke2 token to config
+  lineinfile:
+    path: /etc/rancher/rke2/config.yaml
+    regexp: '^token:'
+    line: "token: {{ hostvars['token_node'].rke2_token }}"
+    create: yes
+  when: inventory_hostname != "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/copy_config.yml b/ansible/tasks/rke2/copy_config.yml
new file mode 100644
index 0000000000..ab580383d1
--- /dev/null
+++ b/ansible/tasks/rke2/copy_config.yml
@@ -0,0 +1,14 @@
+---
+- name: Create /etc/rancher/rke2 directory
+  file:
+    path: /etc/rancher/rke2
+    state: directory
+    mode: u=rw,g=rw,o=r
+
+- name: Copy the configuration file to the remote location
+  copy:
+    src: "{{ config_file }}"
+    dest: /etc/rancher/rke2/config.yaml
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: u=rw,g=rw,o=r
\ No newline at end of file
diff --git a/ansible/tasks/rke2/deploy_app.yml b/ansible/tasks/rke2/deploy_app.yml
new file mode 100644
index 0000000000..506062fbb7
--- /dev/null
+++ b/ansible/tasks/rke2/deploy_app.yml
@@ -0,0 +1,21 @@
+---
+- name: Copying values.yml file on the server
+  copy:
+    src: /opt/charts/splunk-connect-for-syslog/values.yaml
+    dest: "/home/{{ ansible_user }}/values.yaml"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: u=rw,g=rw,o=r
+
+- name: Deploy app or update it with new values if already deployed
+  block:
+    - name: Deploy sc4s app from templates with overwrites from values.yml
+      ansible.builtin.shell: helm install sc4s splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml
+      args:
+        chdir: "/home/{{ ansible_user }}/"
+  rescue:
+      - name: Update app with new values.yml
+        ansible.builtin.shell: helm upgrade sc4s splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml
+        args:
+          chdir: "/home/{{ ansible_user }}/"
+  when: inventory_hostname == "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/deploy_secrets.yml b/ansible/tasks/rke2/deploy_secrets.yml
new file mode 100644
index 0000000000..ed71e4a233
--- /dev/null
+++ b/ansible/tasks/rke2/deploy_secrets.yml
@@ -0,0 +1,29 @@
+---
+- name: Load k8s secrets
+  include_vars:
+    file: "{{ item }}"
+  with_first_found:
+    - files:
+      - /opt/ansible/resources/k8s_secrets.yaml
+      - /opt/charts/splunk-connect-for-syslog/secrets.yaml
+
+- name: Export kubectl bin path
+  shell: export PATH=$PATH:/var/lib/rancher/rke2/bin/
+
+- name: Create mTLS secret
+  ansible.builtin.shell: |
+    /var/lib/rancher/rke2/bin/kubectl apply -f - <<EOF
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: {{ hec_tls.secret }}
+    type: Opaque
+    data:
+      key.pem: {{ hec_tls.value.key | b64encode }}
+      cert.pem: {{ hec_tls.value.cert | b64encode }}
+      ca_cert.pem: {{ hec_tls.value.ca | b64encode }}
+    EOF
+  when:
+    - hec_tls is defined
+    - ('secret' in hec_tls) and ('value' in hec_tls)
+    - inventory_hostname == "token_node"
diff --git a/ansible/tasks/rke2/get_registration_token.yml b/ansible/tasks/rke2/get_registration_token.yml
new file mode 100644
index 0000000000..fa973e7631
--- /dev/null
+++ b/ansible/tasks/rke2/get_registration_token.yml
@@ -0,0 +1,16 @@
+---
+- name: Wait for rke2 token to be ready
+  wait_for:
+    path: /var/lib/rancher/rke2/server/node-token
+    timeout: 60
+  when: inventory_hostname == "token_node"
+
+- name: Retrieve the rke2 node token
+  shell: cat /var/lib/rancher/rke2/server/node-token
+  register: node_token
+  when: inventory_hostname == "token_node"
+
+- name: Set the node token as a fact for worker nodes
+  set_fact:
+    rke2_token: "{{ node_token.stdout }}"
+  when: inventory_hostname == "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/install_agents.yml b/ansible/tasks/rke2/install_agents.yml
new file mode 100644
index 0000000000..7d35c32eac
--- /dev/null
+++ b/ansible/tasks/rke2/install_agents.yml
@@ -0,0 +1,9 @@
+---
+- name: Run the installer
+  shell: curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -
+
+- name: Enable and start rke2-agent.service
+  service:
+    name: rke2-agent.service
+    enabled: yes
+    state: started
\ No newline at end of file
diff --git a/ansible/tasks/rke2/install_first_server.yml b/ansible/tasks/rke2/install_first_server.yml
new file mode 100644
index 0000000000..64598237b0
--- /dev/null
+++ b/ansible/tasks/rke2/install_first_server.yml
@@ -0,0 +1,11 @@
+---
+- name: Run the installer
+  shell: curl -sfL https://get.rke2.io | sh -
+  when: inventory_hostname == "token_node"
+
+- name: Enable and start rke2-server.service
+  service:
+    name: rke2-server.service
+    enabled: yes
+    state: started
+  when: inventory_hostname == "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/install_helm_repo.yml b/ansible/tasks/rke2/install_helm_repo.yml
new file mode 100644
index 0000000000..2b37a135e8
--- /dev/null
+++ b/ansible/tasks/rke2/install_helm_repo.yml
@@ -0,0 +1,6 @@
+---
+- name: Add sc4s helm repository
+  shell: helm repo add splunk-connect-for-syslog https://splunk.github.io/splunk-connect-for-syslog
+
+- name: Separately update the repository cache
+  shell: helm repo update
diff --git a/ansible/tasks/rke2/install_metallb.yml b/ansible/tasks/rke2/install_metallb.yml
new file mode 100644
index 0000000000..f7acaec1aa
--- /dev/null
+++ b/ansible/tasks/rke2/install_metallb.yml
@@ -0,0 +1,27 @@
+---
+- name: Add metallb helm repository
+  shell: helm repo add metallb https://metallb.github.io/metallb
+
+- name: Create metallb namespace
+  shell: /var/lib/rancher/rke2/bin/kubectl create namespace metallb
+  when: inventory_hostname == "token_node"
+
+- name: Install metallb
+  shell: helm install metallb metallb/metallb -n metallb
+  when: inventory_hostname == "token_node"
+
+- name: Copy metallb-config.yaml file on the server
+  copy:
+    src: /opt/ansible/resources/metallb-config.yaml
+    dest: "/home/{{ ansible_user }}/metallb-config.yaml"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: u=rw,g=rw,o=r
+
+- name: Apply metallb config
+  shell: /var/lib/rancher/rke2/bin/kubectl apply -f metallb-config.yaml
+  register: result
+  retries: 10
+  delay: 60
+  until: result.stderr == ""
+  when: inventory_hostname == "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/install_other_servers.yml b/ansible/tasks/rke2/install_other_servers.yml
new file mode 100644
index 0000000000..0c5f88deea
--- /dev/null
+++ b/ansible/tasks/rke2/install_other_servers.yml
@@ -0,0 +1,11 @@
+---
+- name: Run the installer
+  shell: curl -sfL https://get.rke2.io | sh -
+  when: inventory_hostname != "token_node"
+
+- name: Enable and start rke2-server.service
+  service:
+    name: rke2-server.service
+    enabled: yes
+    state: started
+  when: inventory_hostname != "token_node"
\ No newline at end of file
diff --git a/ansible/tasks/rke2/provide_kubectl.yml b/ansible/tasks/rke2/provide_kubectl.yml
new file mode 100644
index 0000000000..b2ab066311
--- /dev/null
+++ b/ansible/tasks/rke2/provide_kubectl.yml
@@ -0,0 +1,14 @@
+---
+    - name: Copy kubectl config to ~/.kube/config
+      shell: |
+        mkdir -p /home/{{ ansible_user }}/.kube && \
+        cp /etc/rancher/rke2/rke2.yaml /home/{{ ansible_user }}/.kube/config && \
+        chown {{ ansible_user }} /home/{{ ansible_user }}/.kube/config
+
+    - name: Add rke2 bin path to .bashrc
+      lineinfile:
+        path: "/home/{{ ansible_user }}/.bashrc"
+        line: 'export PATH=$PATH:/var/lib/rancher/rke2/bin/'
+        create: yes
+        state: present
+        insertafter: EOF
\ No newline at end of file
diff --git a/docs/gettingstarted/ansible-rke2.md b/docs/gettingstarted/ansible-rke2.md
new file mode 100644
index 0000000000..cee9f6ec97
--- /dev/null
+++ b/docs/gettingstarted/ansible-rke2.md
@@ -0,0 +1,117 @@
+# Rancher Kubernetes Engine 2
+
+SC4S can be deployed on Rancher Kubernetes Engine 2 (RKE). Provisioning of the infrastructure can be automated using
+Ansible. This instruction assumes knowledge of RKE2 and assumes that [helm](https://helm.sh/) is installed on all control nodes. 
+
+## Step 1: Prepare your initial configuration
+
+1. Before you run SC4S with Ansible, update `values.yaml` with your Splunk endpoint and HEC token. 
+You can find the [values.yaml file here](https://github.com/splunk/splunk-connect-for-syslog/blob/main/charts/splunk-connect-for-syslog/values.yaml).
+
+2. In the inventory file, provide a list of hosts on which you want to run your cluster and the host application. In case of single node deployment inventory should look like this:
+``` yaml
+--8<---- "ansible/inventory/inventory_rke2.yml"
+```
+
+- `token_node` is the host name which must be kept unchanged. In case of high availability cluster this node is used to generate registration token. For single node deployment this feature is not used, but the `token_node` hostname is referenced in the ansible playbook.
+
+- `ansible_host` - address of the node
+
+- `config_file` - absolute path to the RKE2 configuration file. In most basic configuration this can be empty `.yaml` file. Different configuration variables can be found in [RKE2 documentation](https://docs.rke2.io/).
+
+3. Alternatively, you can spin up a high-availability cluster. Variables for all hosts are the same as for single node deployment with one difference, that ansible host names for other nodes apart from `token_node` can be custom:
+``` yaml
+--8<---- "ansible/inventory/inventory_rke2_ha.yml"
+```
+
+For high-availability cluster some basic configuration is required inside `config_file`. Examples below show how these files should be structures, more details can be found in [RKE2 documentation](https://docs.rke2.io/install/ha):
+
+- `token_node` configuration:
+``` yaml
+# The following configuration is needed only when multiple control nodes are installed inside the cluster.
+tls-san:
+  - my-kubernetes-domain.com
+```
+
+- Other control nodes configuration:
+``` yaml
+# 'token' variable should be left empty. It will be updated automatically by ansible playbook.
+server: https://my-kubernetes-domain.com:9345
+token:
+tls-san:
+  - my-kubernetes-domain.com
+```
+
+- Agent nodes configuration:
+``` yaml
+# 'token' variable should be left empty. It will be updated automatically by ansible playbook.
+server: https://my-kubernetes-domain.com:9345
+token:
+```
+
+4. Configure address pool used by Metallb in [ansible/resources/metallb-config.yaml](https://github.com/splunk/splunk-connect-for-syslog/blob/main/ansible/resources/metallb-config.yaml) file.
+
+## Step 2: Deploy SC4S on your configuration
+1. If you have Ansible installed on your host, run the Ansible playbook to deploy SC4S. Otherwise, use the Docker Ansible image provided in the package:
+```bash
+# From repository root
+docker-compose -f ansible/docker-compose.yml build
+docker-compose -f ansible/docker-compose.yml up -d
+docker exec -it ansible_sc4s /bin/bash
+```
+2. If you used the Docker Ansible image, then from your container remote shell, authenticate to and run the MicroK8s playbook.
+
+* To authenticate with username and password:
+``` bash 
+ansible-playbook -i path/to/inventory_rke2.yaml -u <username> --ask-pass path/to/playbooks/rke2.yml
+```
+
+* To authenitcate if you are running a high-availability cluster:
+``` bash 
+ansible-playbook -i path/to/inventory_rke2_ha.yaml -u <username> --ask-pass path/to/playbooks/rke2.yml
+```
+
+* To authenticate using a key pair:
+``` bash 
+ansible-playbook -i path/to/inventory_rke2.yaml -u <username> --key-file <key_file> path/to/playbooks/rke2.yml
+```
+
+## Step 3: Validate your configuration
+
+SC4S performs checks to ensure that the container starts properly and that the syntax of the underlying syslog-ng
+configuration is correct. Once the checks are complete, validate that SC4S properly communicates with Splunk. To do this, execute the following search in Splunk:
+
+```ini
+index=* sourcetype=sc4s:events "starting up"
+```
+
+This should yield an event similar to the following:
+
+```ini
+syslog-ng starting up; version='3.28.1'
+```
+
+You can verify whether all services in the cluster work by checking the ```sc4s_container``` in Splunk. Each service should have a different container ID. All other fields should be the same.
+
+The startup process should proceed normally without syntax errors. If it does not,
+follow the steps below before proceeding to deeper-level troubleshooting:
+
+1. Verify that the URL, token, and TLS/SSL settings are correct, and that the appropriate firewall ports are open (8088 or 443).
+2. Verify that your indexes are created in Splunk, and that your token has access to them.
+3. If you are using a load balancer, verify that it is operating properly.
+4. Execute the following command to check the SC4S startup process running in the container.
+   
+```bash
+kubectl get pods
+kubectl logs <podname>
+```
+
+You should see events similar to those below in the output:
+
+```ini
+SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:fallback...
+SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:events...
+syslog-ng checking config
+sc4s version=v1.36.0
+starting syslog-ng
+```
diff --git a/mkdocs.yml b/mkdocs.yml
index 54f426b628..cefe30386e 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -52,6 +52,7 @@ nav:
           - Docker Swarm: "gettingstarted/ansible-docker-swarm.md"
           - Podman/Docker: "gettingstarted/ansible-docker-podman.md"
           - mk8s: "gettingstarted/ansible-mk8s.md"
+          - RKE2: "gettingstarted/ansible-rke2.md"
   - Create a parser: "create-parser.md"
   - Configuration: "configuration.md"
   - Destinations: "destinations.md"