updates

infra/README.md

@@ -24,8 +24,9 @@ terraform apply -var-file=your_env.tfvars
 4. Setup ansible, build the inventory, run the playbook using the keyfile generated in 2.
 ```
 cd meshdb/infra/ansible
-ansible-galaxy collection install cloud.
-ansible-playbook -i inventory.yaml install_packages.yaml -v --key-file ../infra/tf/meshdbdev0
+ansible-galaxy collection install cloud.terraform
+ansible-playbook -i inventory.yaml install_packages.yaml -v --key-file ../tf/meshdbdev0
+ansible-playbook -i inventory.yaml k8s_lb.yaml -v --key-file ../tf/meshdbdev0
 ```
 
 <!-- 5. Install the `meshdb-cluster` chart.

infra/ansible/k8s_lb.yaml

@@ -1,74 +1,104 @@
 - name: Setup k8s-lb
-  hosts: k8s-lb
-  vars_files:
-    - k8s_lb_config.yaml
+  hosts: lb
   tasks:
     - name: Install deps
       ansible.builtin.apt:
         update_cache: true
         pkg:
           - iptables-persistent
+      become: true
 
     - name: dummy0 interface
       ansible.builtin.template:
         src: ./lb_config/netplan_dummy0.yaml.j2
         dest: /etc/netplan/dummy0.yaml
-        mode: "640"
+        mode: "600"
+      become: true
 
     - name: eth0 interface
       ansible.builtin.template:
         src: ./lb_config/netplan_50_cloud_init.yaml.j2
         dest: /etc/netplan/50-cloud-init.yaml
-        mode: "640"
+        mode: "600"
+      become: true
 
     - name: Install frr
       ansible.builtin.apt:
         update_cache: true
         pkg:
           - frr
+      become: true
 
     - name: Enable ospfd
       ansible.builtin.lineinfile:
         path: /etc/frr/daemons
         search_string: ospfd=no
         line: "ospfd=yes"
+      become: true
 
     - name: Config template frr
       ansible.builtin.template:
         src: ./lb_config/frr.conf.j2
         dest: /etc/frr/frr.conf
+      become: true
+
+    - name: Install haproxy
+      ansible.builtin.apt:
+        update_cache: true
+        pkg:
+          - haproxy
+      become: true
+
+    - name: Config template haproxy
+      ansible.builtin.template:
+        src: ./lb_config/haproxy.cfg
+        dest: /etc/haproxy/haproxy.cfg
+      become: true
 
     - name: Iptables rules
       ansible.builtin.template:
         src: ./lb_config/iptables.j2
         dest: /etc/iptables/rules.v4
+      become: true
 
     - name: Restore iptables rules
       ansible.builtin.command:
         cmd: "bash -c '/sbin/iptables-restore < /etc/iptables/rules.v4 && touch /tmp/firewall_set'"
         creates: /tmp/firewall_set
+      become: true
 
-    - name: Restore iptables rules
+    - name: Netplan apply
       ansible.builtin.command:
         cmd: "bash -c 'netplan apply && touch /tmp/netplan_applied'"
         creates: /tmp/netplan_applied
+      become: true
 
     - name: Restart and enable iptables service
       ansible.builtin.service:
         name: netfilter-persistent
         state: restarted
         enabled: true
+      become: true
 
     - name: Restart and enable frr service
       ansible.builtin.service:
         name: frr
         state: restarted
         enabled: true
+      become: true
+
+    - name: Restart and enable haproxy service
+      ansible.builtin.service:
+        name: haproxy
+        state: restarted
+        enabled: true
+      become: true
 
     - name: net.ipv4.ip_forward
       ansible.posix.sysctl:
         name: net.ipv4.ip_forward
         value: '1'
         sysctl_set: true
         state: present
-        reload: true
+        reload: true
+      become: true

infra/ansible/lb_config/frr.conf.j2

@@ -11,7 +11,7 @@ interface eth0
 exit
 !
 router ospf
- network {{ INTERNAL_NETWORK_RANGE }} area 0
+ network {{ INTERNAL_NETWORK_BLOCK }} area 0
  network {{ EXTERNAL_LISTEN_IP }}/32 area 0
 exit
 !

infra/ansible/lb_config/haproxy.cfg

@@ -0,0 +1,46 @@
+global
+	log /dev/log	local0
+	log /dev/log	local1 notice
+	chroot /var/lib/haproxy
+	stats socket /run/haproxy/admin.sock mode 660 level admin
+	stats timeout 30s
+	user haproxy
+	group haproxy
+	daemon
+
+	# Default SSL material locations
+	ca-base /etc/ssl/certs
+	crt-base /etc/ssl/private
+
+	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
+        ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+        ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+        ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
+
+defaults
+	log	global
+	mode	http
+	option	httplog
+	option	dontlognull
+        timeout connect 5000
+        timeout client  50000
+        timeout server  50000
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 408 /etc/haproxy/errors/408.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
+	errorfile 504 /etc/haproxy/errors/504.http
+
+frontend meshdb
+  bind {{ EXTERNAL_LISTEN_IP }}:80
+  bind {{ EXTERNAL_LISTEN_IP }}:443
+  default_backend k8s
+
+backend k8s
+#  server metallb 10.70.90.240:80
+  server meshdbarlenenode0 {{ NODE_IP_1 }}:{{ NODE_PORT }}
+  server meshdbarlenenode1 {{ NODE_IP_2 }}:{{ NODE_PORT }}
+  server meshdbarlenenode2 {{ NODE_IP_3 }}:{{ NODE_PORT }}
+

infra/ansible/lb_config/iptables.j2

@@ -2,6 +2,7 @@
 :INPUT ACCEPT [0:0]
 :FORWARD ACCEPT [0:0]
 :OUTPUT ACCEPT [0:0]
+-A INPUT -d {{ inventory_hostname }}/{{ INTERNAL_NETWORK_RANGE }} -p tcp -m tcp --dport 22 -j ACCEPT
 -A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 80 -j ACCEPT
 -A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 443 -j ACCEPT
 -A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -j DROP

infra/ansible/lb_config/netplan_50_cloud_init.yaml.j2

@@ -7,11 +7,11 @@ network:
     version: 2
     ethernets:
         eth0:
-            dhcp4: false
-            dhcp6: false
+            dhcp4: no
+            dhcp6: no
             addresses: [ "{{ inventory_hostname }}/{{ INTERNAL_NETWORK_RANGE }}" ]
             nameservers:
-                addresses: [ "10.10.10.10", "10.10.10.11" ]
+                addresses: [ "10.10.10.10", "10.10.10.11", "8.8.8.8" ]
             match:
-                macaddress: {{ INTERNAL_NETWORK_MAC_ADDRESS }}
+                macaddress: {{ hostvars[inventory_hostname].ansible_default_ipv4.macaddress }}
             set-name: eth0

infra/tf/ansible.tf

@@ -13,6 +13,21 @@ resource "ansible_group" "workers" {
   }
 }
 
+resource "ansible_group" "lb" {
+  name = "lb"
+  variables = {
+    ansible_user = var.meshdb_local_user
+    EXTERNAL_LISTEN_IP = var.meshdb_external_ip
+    LB_HOSTNAME = "k8s-lb"
+    INTERNAL_NETWORK_BLOCK: format("%s/%s", var.meshdb_net_block, var.meshdb_networkrange)
+    INTERNAL_NETWORK_RANGE: var.meshdb_networkrange
+    NODE_IP_1: var.meshdb_ips[0]
+    NODE_IP_2: var.meshdb_ips[1]
+    NODE_IP_3: var.meshdb_ips[2]
+    NODE_PORT = "32192"
+  }
+}
+
 # declare your hosts here
 resource "ansible_host" "meshdbmgr" {
   count  = 1
@@ -25,3 +40,8 @@ resource "ansible_host" "meshdbnode" {
   name   = var.meshdb_ips[count.index] 
   groups = [ansible_group.workers.name]
 }
+
+resource "ansible_host" "k8slb" {
+  name   = var.meshdb_lb_ip
+  groups = [ansible_group.lb.name]
+}

infra/tf/example.tfvars

@@ -17,3 +17,5 @@ meshdb_ips                      = [
 ]
 meshdb_lb_ip                    = "10.70.90.B"
 meshdb_metallb_range            = "10.70.90.80/29"
+meshdb_external_ip              = "1.2.3.4"
+meshdb_net_block                = "10.70.90.0"

infra/tf/vars.tf

@@ -70,7 +70,17 @@ variable "meshdb_networkrange" {
   default      = "24"
 }
 
+variable "meshdb_net_block" {
+  description  = "network range to use for intneral networking"
+  default      = "10.70.90.0"
+}
+
 variable "meshdb_metallb_range" {
   type         = string
   description  = "ip range for metallb"
 }
+
+variable "meshdb_external_ip" {
+  type         = string
+  description  = "external ip for meshdb, assigned to the lb vm"
+}