diff --git a/capi-lab/deploy.yaml b/capi-lab/deploy.yaml index 13c3165..01218c9 100644 --- a/capi-lab/deploy.yaml +++ b/capi-lab/deploy.yaml @@ -14,5 +14,8 @@ - name: prometheus - name: firewall-controller-manager vars: - firewall_controller_manager_namespace: cap-metal-stack + firewall_controller_manager_namespace: capms-system + firewall_controller_manager_ca: "{{ lookup('file', playbook_dir + '/fcm-certs/ca.pem') }}" + firewall_controller_manager_cert: "{{ lookup('file', playbook_dir + '/fcm-certs/tls.crt') }}" + firewall_controller_manager_cert_key: "{{ lookup('file', playbook_dir + '/fcm-certs/tls.key') }}" - name: cluster-api-provider-metal-stack diff --git a/capi-lab/fcm-certs/ca-config.json b/capi-lab/fcm-certs/ca-config.json new file mode 100644 index 0000000..e80812e --- /dev/null +++ b/capi-lab/fcm-certs/ca-config.json @@ -0,0 +1,18 @@ +{ + "signing": { + "default": { + "expiry": "168h" + }, + "profiles": { + "client-server": { + "expiry": "8760h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } +} diff --git a/capi-lab/fcm-certs/ca-csr.json b/capi-lab/fcm-certs/ca-csr.json new file mode 100644 index 0000000..1a3d24b --- /dev/null +++ b/capi-lab/fcm-certs/ca-csr.json @@ -0,0 +1,14 @@ +{ + "CN": "ca", + "key": { + "algo": "ecdsa", + "size": 256 + }, + "names": [ + { + "C": "DE", + "L": "Bavaria", + "ST": "Munich" + } + ] +} diff --git a/capi-lab/fcm-certs/ca-key.pem b/capi-lab/fcm-certs/ca-key.pem new file mode 100644 index 0000000..48fc579 --- /dev/null +++ b/capi-lab/fcm-certs/ca-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIFUGS1Xbmf1C9NcitDjcU3yfM3JUSS8SAeIHAvkHgofhoAoGCCqGSM49 +AwEHoUQDQgAEYPaD8+nz3ffhuV3iq3958NFnO28pCIfXiZOCVLyQYsvlr88eFbrN +vjEHXAmvxTp5X2hlY5dbVh/CPC6FJbBFCw== +-----END EC PRIVATE KEY----- diff --git a/capi-lab/fcm-certs/ca.pem b/capi-lab/fcm-certs/ca.pem new file mode 100644 index 0000000..4c7ebb7 --- /dev/null +++ b/capi-lab/fcm-certs/ca.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBvjCCAWSgAwIBAgIUQBnjRL2py37bbgxj2/pB9TYZdSMwCgYIKoZIzj0EAwIw +PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp +YTELMAkGA1UEAxMCY2EwHhcNMjQxMTIxMTIxMjAwWhcNMjkxMTIwMTIxMjAwWjA9 +MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh +MQswCQYDVQQDEwJjYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGD2g/Pp8933 +4bld4qt/efDRZztvKQiH14mTglS8kGLL5a/PHhW6zb4xB1wJr8U6eV9oZWOXW1Yf +wjwuhSWwRQujQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBT0JWN2t5PTJEOyBBbfGqjUdrsMXTAKBggqhkjOPQQDAgNIADBFAiEA +ojnyHUbtmkx1xnuon+VFZKjccZxyoMaU/0u2Sz0MhWwCICrpHbQTNLoL8Q48UfJK +33EilS1z6lxn/nM6+ql8WVfO +-----END CERTIFICATE----- diff --git a/capi-lab/fcm-certs/roll.sh b/capi-lab/fcm-certs/roll.sh new file mode 100755 index 0000000..b8293d8 --- /dev/null +++ b/capi-lab/fcm-certs/roll.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +echo "generating example certs" +cfssl genkey -initca ca-csr.json | cfssljson -bare ca +cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server tls.json | cfssljson -bare tls +rm *.csr +mv tls.pem tls.crt +mv tls-key.pem tls.key diff --git a/capi-lab/fcm-certs/tls.crt b/capi-lab/fcm-certs/tls.crt new file mode 100644 index 0000000..36f6d89 --- /dev/null +++ b/capi-lab/fcm-certs/tls.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIChjCCAiugAwIBAgIUfIGP//S9eEv3UtQ/ZlfTc579jdowCgYIKoZIzj0EAwIw +PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp +YTELMAkGA1UEAxMCY2EwHhcNMjQxMTIxMTIxMjAwWhcNMjUxMTIxMTIxMjAwWjBE +MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh +MRIwEAYDVQQDEwlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQJ +MoYtsmZB2s53fzS+LXf/rFSI6sHiKJ4kbenK04agoarAsIGniCPgRb4MUj2LvhC5 +1xJJncCC21QVUZCXZb+lo4IBADCB/TAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw +FAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFNDY +PWHrOS2a1CtLw91V3cKk+Y6NMB8GA1UdIwQYMBaAFPQlY3a3k9MkQ7IEFt8aqNR2 +uwxdMH4GA1UdEQR3MHWCCWxvY2FsaG9zdIIsZmlyZXdhbGwtY29udHJvbGxlci1t +YW5hZ2VyLmNhcG1zLXN5c3RlbS5zdmOCOmZpcmV3YWxsLWNvbnRyb2xsZXItbWFu +YWdlci5jYXBtcy1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWwwCgYIKoZIzj0EAwID +SQAwRgIhAKlzsenMaiXH+IqONSjxL/Bk5Xk7HM+sWfbTyVoOHXnhAiEA0nd2f04Z +R36a+jGSXPxMgR2OOmScjfOUk3xnDDInMQE= +-----END CERTIFICATE----- diff --git a/capi-lab/fcm-certs/tls.json b/capi-lab/fcm-certs/tls.json new file mode 100644 index 0000000..ba00cd9 --- /dev/null +++ b/capi-lab/fcm-certs/tls.json @@ -0,0 +1,19 @@ +{ + "CN": "localhost", + "hosts": [ + "localhost", + "firewall-controller-manager.capms-system.svc", + "firewall-controller-manager.capms-system.svc.cluster.local" + ], + "key": { + "algo": "ecdsa", + "size": 256 + }, + "names": [ + { + "C": "DE", + "L": "Bavaria", + "ST": "Munich" + } + ] +} diff --git a/capi-lab/fcm-certs/tls.key b/capi-lab/fcm-certs/tls.key new file mode 100644 index 0000000..5678c7b --- /dev/null +++ b/capi-lab/fcm-certs/tls.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEICCR8PczdJo8Tjpum62cO2hrlS0irQRVAgYhzcAr9raXoAoGCCqGSM49 +AwEHoUQDQgAECTKGLbJmQdrOd380vi13/6xUiOrB4iieJG3pytOGoKGqwLCBp4gj +4EW+DFI9i74QudcSSZ3AgttUFVGQl2W/pQ== +-----END EC PRIVATE KEY----- diff --git a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml index 6bb9002..3e8570b 100644 --- a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml @@ -1,2 +1,29 @@ --- firewall_controller_manager_namespace: "firewall-controller-manager" + +firewall_controller_manager_image_pull_policy: Always +firewall_controller_manager_replicas: 1 +# firewall_controller_manager_pod_annotations: + +firewall_controller_manager_seed_api_url: https://kubernetes +firewall_controller_manager_shoot_api_url: +firewall_controller_manager_cluster_id: + +firewall_controller_manager_metalapi_url: http://metal-api.metal-control-plane.svc.cluster.local:8080 +firewall_controller_manager_metalapi_hmac: metal-admin + +firewall_controller_manager_generic_token_kubeconfig_secret_name: +firewall_controller_manager_ssh_key_secret_name: + +firewall_controller_manager_shoot_access_token_secret: "shoot-access-firewall-controller-manager" + +firewall_controller_manager_ca: +firewall_controller_manager_cert: +firewall_controller_manager_cert_key: + +firewall_controller_manager_pod_annotations: {} + +firewall_controller_manager_crd_fetch_base_url: "https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/heads/" +# TODO: +# firewall_controller_manager_crd_fetch_base_url: "https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/tags/" +firewall_controller_manager_image_tag: initial-firewall-ruleset diff --git a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml index 0a35dd7..15222ce 100644 --- a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml @@ -8,22 +8,36 @@ apiVersion: v1 kind: Namespace metadata: - name: "{{ firewall_controller_manager_namespace }}" + name: "{{ item }}" labels: - name: "{{ firewall_controller_manager_namespace }}" + name: "{{ item }}" + loop: + - "{{ firewall_controller_manager_namespace }}" + - firewall - name: Deploy firewall-controller-manager CRDs k8s: - definition: "{{ lookup('url', 'https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/tags/' + firewall_controller_manager_image_tag + '/config/crds/' + item, split_lines=False) }}" + definition: "{{ lookup('url', firewall_controller_manager_crd_fetch_base_url + firewall_controller_manager_image_tag + '/config/crds/' + item, split_lines=False) }}" namespace: "{{ firewall_controller_manager_namespace }}" + apply: true loop: - firewall.metal-stack.io_firewalldeployments.yaml - firewall.metal-stack.io_firewallmonitors.yaml - firewall.metal-stack.io_firewalls.yaml - firewall.metal-stack.io_firewallsets.yaml -# - name: Deploy firewall-controller-manager -# k8s: -# definition: - -# namespace: "{{ firewall_controller_manager_namespace }}" +- name: Deploy firewall-controller-manager + k8s: + definition: "{{ lookup('template', item) }}" + namespace: "{{ firewall_controller_manager_namespace }}" + apply: true + loop: + - sa.yaml + - cluster-role.yaml + - cluster-role-binding.yaml + - mutatingwebhookconfiguration.yaml + - validatingwebhookconfiguration.yaml + - secret.yaml + - secret-ca.yaml + - deployment.yaml + - service.yaml diff --git a/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml b/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml new file mode 100644 index 0000000..a6366ed --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: firewall-controller-manager +subjects: +- kind: ServiceAccount + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml b/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml new file mode 100644 index 0000000..cd1a73c --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml @@ -0,0 +1,87 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager +rules: +- apiGroups: + - firewall.metal-stack.io + resources: + - firewalls + - firewalls/status + - firewallsets + - firewallsets/status + - firewalldeployments + - firewalldeployments/status + - firewallmonitors + - firewallmonitors/status + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - secrets + - serviceaccounts + - namespaces + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + - clusterroles + - clusterrolebindings + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures + - extensions + verbs: + - get +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures/status + verbs: + - patch +- apiGroups: + - extensions.gardener.cloud + resources: + - extensions + verbs: + - update diff --git a/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml b/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml new file mode 100644 index 0000000..53f5a75 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager +spec: + selector: + matchLabels: + app: firewall-controller-manager + replicas: {{ firewall_controller_manager_replicas }} + template: + metadata: + labels: + app: firewall-controller-manager +{% if firewall_controller_manager_pod_annotations %} + annotations: +{{ firewall_controller_manager_pod_annotations | to_nice_yaml | indent(width=8, first=true) }} +{% endif %} + spec: + serviceAccountName: firewall-controller-manager + containers: + - name: firewall-controller-manager + image: {{ firewall_controller_manager_image_name }}:{{ firewall_controller_manager_image_tag }} + imagePullPolicy: {{ firewall_controller_manager_image_pull_policy }} + args: + - -cert-dir=/certs + - -log-level=info + - -seed-api-url={{ firewall_controller_manager_seed_api_url }} + # - -shoot-api-url={{ firewall_controller_manager_shoot_api_url }} + # - -internal-shoot-api-url=https://kube-apiserver + # - -cluster-id={{ firewall_controller_manager_cluster_id }} + - -enable-leader-election + - -metal-api-url={{ firewall_controller_manager_metalapi_url }} + - -namespace={{ firewall_controller_manager_namespace }} + - -shoot-kubeconfig-secret-name=none + - -shoot-token-secret-name=none + - -ssh-key-secret-name=none + # - -shoot-token-path=/token + env: + - name: METAL_AUTH_HMAC + valueFrom: + secretKeyRef: + name: firewall-controller-manager-config + key: api-hmac + livenessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: webhook-certs + mountPath: "/certs" + readOnly: true + - name: token-dir + mountPath: "/token" + resources: + limits: + cpu: 400m + memory: 400Mi + requests: + cpu: 100m + memory: 20Mi + volumes: + - name: webhook-certs + secret: + secretName: firewall-controller-manager-certs + - name: token-dir + emptyDir: {} diff --git a/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml b/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml new file mode 100644 index 0000000..1a9bfaf --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalldeployments + sideEffects: None diff --git a/capi-lab/roles/firewall-controller-manager/templates/sa.yaml b/capi-lab/roles/firewall-controller-manager/templates/sa.yaml new file mode 100644 index 0000000..0ba4ccf --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/sa.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml b/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml new file mode 100644 index 0000000..aa09697 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: firewall-controller-manager-certs +data: + ca.crt: {{ firewall_controller_manager_ca | b64encode }} + tls.crt: {{ firewall_controller_manager_cert | b64encode }} + tls.key: {{ firewall_controller_manager_cert_key | b64encode }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/secret.yaml b/capi-lab/roles/firewall-controller-manager/templates/secret.yaml new file mode 100644 index 0000000..88eff24 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/secret.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: firewall-controller-manager-config +stringData: + api-hmac: {{ firewall_controller_manager_metalapi_hmac }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/service.yaml b/capi-lab/roles/firewall-controller-manager/templates/service.yaml new file mode 100644 index 0000000..abd4628 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/service.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager +spec: + type: ClusterIP + ports: + - name: webhooks + port: 9443 + protocol: TCP + - name: metrics + port: 2112 + protocol: TCP + selector: + app: firewall-controller-manager diff --git a/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml b/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml new file mode 100644 index 0000000..7be581a --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalldeployments + sideEffects: None diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b8514d4..2c66e33 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -9,8 +9,11 @@ rules: resources: - secrets verbs: + - create + - delete - get - list + - update - watch - apiGroups: - cluster.x-k8s.io diff --git a/config/samples/example-kubeadm.yaml b/config/samples/example-kubeadm.yaml index 8642605..4f71599 100644 --- a/config/samples/example-kubeadm.yaml +++ b/config/samples/example-kubeadm.yaml @@ -27,7 +27,7 @@ spec: size: v1-small-x86 image: firewall-ubuntu-3.0 networks: - - internet-mini-lab + - internet-mini-lab --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: MetalStackMachineTemplate @@ -101,17 +101,17 @@ spec: CNI_PLUGINS_VERSION="v1.3.0" DEST="/opt/cni/bin" mkdir -p "$DEST" - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | sudo tar -C "$DEST" -xz + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | tar -C "$DEST" -xz RELEASE="v1.30.6" cd /usr/local/bin - sudo curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - sudo chmod +x {kubeadm,kubelet,kubectl} + curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} RELEASE_VERSION="v0.16.2" - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | sudo tee /usr/lib/systemd/system/kubelet.service - sudo mkdir -p /usr/lib/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | sudo tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service + mkdir -p /usr/lib/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf systemctl enable kubelet.service - path: /etc/containerd/config.toml diff --git a/go.mod b/go.mod index 40f9470..ed37ffd 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,12 @@ go 1.23.0 require ( github.com/go-logr/logr v1.4.2 - github.com/metal-stack/firewall-controller-manager v0.4.3 + github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b github.com/metal-stack/metal-go v0.37.2 github.com/metal-stack/metal-lib v0.18.4 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 @@ -102,7 +103,6 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect diff --git a/go.sum b/go.sum index e6f1983..3ea7915 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/metal-stack/firewall-controller-manager v0.4.3 h1:WU5bqD710gUtzyA2NdWJuSveCbOhllQ7ybpgUg7aJW8= -github.com/metal-stack/firewall-controller-manager v0.4.3/go.mod h1:J/3LHcvfJCpEEC4yk+WD0exh3btaScCaFkzbnbOsqrY= +github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b h1:MKtYVt1QPVSd9LzTW532QzVz9c+hIUst7+8SmxhM8us= +github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b/go.mod h1:J/3LHcvfJCpEEC4yk+WD0exh3btaScCaFkzbnbOsqrY= github.com/metal-stack/metal-go v0.37.2 h1:SDIuV43y09kmwtHfsReOZoZ7c2F+lNP4iIhazfJL5tQ= github.com/metal-stack/metal-go v0.37.2/go.mod h1:3MJTYCS4YJz8D8oteTKhjpaAKNMMjMKYDrIy9awHGtQ= github.com/metal-stack/metal-lib v0.18.4 h1:7HnfSwSbrKNHU+i6i79YFk/eeuhBhwIEHWpGqS7pYCc= diff --git a/internal/controller/metalstackcluster_controller.go b/internal/controller/metalstackcluster_controller.go index ac33d47..d5e0120 100644 --- a/internal/controller/metalstackcluster_controller.go +++ b/internal/controller/metalstackcluster_controller.go @@ -18,10 +18,15 @@ package controller import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "errors" "fmt" "strconv" + "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,6 +78,7 @@ type clusterReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=metalstackclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=metalstackclusters/finalizers,verbs=update // +kubebuilder:rbac:groups=firewall.metal-stack.io,resources=firewalldeployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete // Reconcile reconciles the reconciled cluster to be reconciled. func (r *MetalStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -166,6 +172,13 @@ func (r *MetalStackClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *clusterReconciler) reconcile() error { + sshPubKey, err := r.ensureSshKeyPair(r.ctx) + if err != nil { + return fmt.Errorf("unable to ensure ssh key pair: %w", err) + } + + r.log.Info("reconciled ssh key pair") + nodeNetworkID, err := r.ensureNodeNetwork() if err != nil { return fmt.Errorf("unable to ensure node network: %w", err) @@ -197,7 +210,7 @@ func (r *clusterReconciler) reconcile() error { return fmt.Errorf("failed to update infra cluster control plane endpoint: %w", err) } - fwdeploy, err := r.ensureFirewallDeployment(nodeNetworkID) + fwdeploy, err := r.ensureFirewallDeployment(nodeNetworkID, sshPubKey) if err != nil { return fmt.Errorf("unable to ensure firewall deployment: %w", err) } @@ -237,9 +250,83 @@ func (r *clusterReconciler) delete() error { r.log.Info("deleted node network") + err = r.deleteSshKeyPair(r.ctx) + if err != nil { + return fmt.Errorf("unable to delete ssh key pair: %w", err) + } + + r.log.Info("deleted ssh key pair") + return err } +func (r *clusterReconciler) ensureSshKeyPair(ctx context.Context) (string, error) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.cluster.Name + "-ssh-keypair", + Namespace: r.cluster.Namespace, + }, + } + + err := r.client.Get(ctx, client.ObjectKeyFromObject(secret), secret) + if err == nil { + if key, ok := secret.Data["id_rsa.pub"]; ok { + return string(key), nil + } + } + if err != nil && !apierrors.IsNotFound(err) { + return "", err + } + + privateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + privateKeyBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + } + + // generate and write public key + pubKey, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return "", err + } + + secret.Data = map[string][]byte{ + "id_rsa": pem.EncodeToMemory(privateKeyBlock), + "id_rsa.pub": ssh.MarshalAuthorizedKey(pubKey), + } + + err = r.client.Create(ctx, secret) + if err != nil { + return "", err + } + + return string(ssh.MarshalAuthorizedKey(pubKey)), nil +} + +func (r *clusterReconciler) deleteSshKeyPair(ctx context.Context) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ssh-keypair-" + r.cluster.Name, + Namespace: r.cluster.Namespace, + }, + } + + err := r.client.Delete(ctx, secret) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + return nil +} + func (r *clusterReconciler) ensureNodeNetwork() (string, error) { nws, err := r.findNodeNetwork() if err != nil { @@ -399,7 +486,7 @@ func (r *clusterReconciler) findControlPlaneIP() ([]*models.V1IPResponse, error) return resp.Payload, nil } -func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcmv2.FirewallDeployment, error) { +func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID, sshPubKey string) (*fcmv2.FirewallDeployment, error) { deploy := &fcmv2.FirewallDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: r.infraCluster.Name, @@ -420,19 +507,60 @@ func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcm deploy.Annotations = map[string]string{} } deploy.Annotations[fcmv2.ReconcileAnnotation] = strconv.FormatBool(true) + deploy.Annotations[fcmv2.FirewallNoControllerConnectionAnnotation] = strconv.FormatBool(true) if deploy.Labels == nil { deploy.Labels = map[string]string{} } - // TODO: this is the selector for the mutating webhook, without it the mutation will not happen, but do we need mutation? - // deploy.Labels[MutatingWebhookObjectSelectorLabel] = cluster.ObjectMeta.Name - deploy.Spec.Replicas = 1 deploy.Spec.Selector = map[string]string{ tag.ClusterID: string(r.infraCluster.GetUID()), } + deploy.Spec.Template.Spec.InitialRuleSet = &fcmv2.InitialRuleSet{ + Egress: []fcmv2.EgressRule{ + { + Comment: "allow outgoing http", + Ports: []int32{80}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing https", + Ports: []int32{443}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing dns via tcp", + Ports: []int32{53}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing dns and ntp via udp", + Ports: []int32{53, 123}, + Protocol: fcmv2.NetworkProtocolUDP, + To: []string{"0.0.0.0/0"}, + }, + }, + Ingress: []fcmv2.IngressRule{ + { + Comment: "allow incoming ssh", + Ports: []int32{22}, + Protocol: fcmv2.NetworkProtocolTCP, + From: []string{"0.0.0.0/0"}, // TODO: restrict cidr + }, + { + Comment: "allow incoming https to kube-apiserver", + Ports: []int32{443}, + Protocol: fcmv2.NetworkProtocolTCP, + From: []string{"0.0.0.0/0"}, // TODO: restrict cidr + }, + }, + } + if deploy.Spec.Template.Labels == nil { deploy.Spec.Template.Labels = map[string]string{} } @@ -454,10 +582,8 @@ func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcm // TODO: we need to allow internet connection for the nodes before the firewall-controller can connect to the control-plane // the FCM currently does not support this - deploy.Spec.Template.Spec.Userdata = "" - - // TODO: do we need to generate ssh keys for the machines and the firewall in this controller? - deploy.Spec.Template.Spec.SSHPublicKeys = nil + deploy.Spec.Template.Spec.Userdata = "{}" + deploy.Spec.Template.Spec.SSHPublicKeys = []string{sshPubKey} // TODO: consider auto update machine image feature diff --git a/internal/controller/metalstackmachine_controller.go b/internal/controller/metalstackmachine_controller.go index 60cc1f9..296e29b 100644 --- a/internal/controller/metalstackmachine_controller.go +++ b/internal/controller/metalstackmachine_controller.go @@ -274,6 +274,22 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { return nil, fmt.Errorf("unable to fetch bootstrap secret: %w", err) } + sshSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.infraCluster.Name + "-ssh-keypair", + Namespace: r.infraCluster.Namespace, + }, + } + err = r.client.Get(r.ctx, client.ObjectKeyFromObject(sshSecret), sshSecret) + if err != nil { + return nil, fmt.Errorf("unable to fetch ssh secret: %w", err) + } + + sshPubKey, ok := sshSecret.Data["id_rsa.pub"] + if !ok { + return nil, errors.New("ssh secret does not contain public key") + } + var ( ips []string nws = []*models.V1MachineAllocationNetwork{ @@ -311,7 +327,7 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { Networks: nws, Ips: ips, UserData: string(bootstrapSecret.Data["value"]), - // TODO: SSHPubKeys, ... + SSHPubKeys: []string{string(sshPubKey)}, }), nil) if err != nil { return nil, fmt.Errorf("failed to allocate machine: %w", err)