diff --git a/.taskfiles/bootstrap/Taskfile.dist.yaml b/.taskfiles/bootstrap/Taskfile.dist.yaml index 82ec576b26..332aa632a1 100644 --- a/.taskfiles/bootstrap/Taskfile.dist.yaml +++ b/.taskfiles/bootstrap/Taskfile.dist.yaml @@ -27,8 +27,10 @@ tasks: PRE_ARGS: '{{.PRE_ARGS}}' POST_ARGS: '{{.POST_ARGS}}' cmds: + #- | + # {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + (.value | @sh)) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}} - | - {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + (.value | @sh)) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}} + {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + .value) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}} 1p-vars-env-run: dir: '{{.USER_WORKING_DIR}}' @@ -70,8 +72,8 @@ tasks: - task: 1p-vars-env-shell vars: C: *c - POST_ARGS: " | kubectl create secret generic -n flux-system sinon-vars --from-env-file=/dev/stdin" - - kubectl create secret generic -n flux-system sinon-secrets # currently nothing hopefully maybe? + POST_ARGS: " | kubectl create secret generic -n flux-system {{.C}}-vars --from-env-file=/dev/stdin" + - kubectl create secret generic -n flux-system {{.C}}-secrets # currently nothing hopefully maybe? - op read "op://{{.C}}/Flux/agekey" | kubectl create secret generic -n flux-system agekey --from-file=age.agekey=/dev/stdin - yq '.spec.values | explode(.)' ./kube/deploy/core/secrets/external-secrets/app/hr.yaml | helm install -n external-secrets external-secrets oci://ghcr.io/external-secrets/charts/external-secrets --version $(yq '.spec.chart.spec.version' ./kube/deploy/core/secrets/external-secrets/app/hr.yaml) -f - - - kubectl apply --server-side -f /{{.ROOT_DIR}}/kube/clusters/sinon/flux/flux-repo.yaml + - kubectl apply --server-side -f /{{.ROOT_DIR}}/kube/clusters/{{.C}}/flux/flux-repo.yaml diff --git a/.taskfiles/k8s/Taskfile.dist.yaml b/.taskfiles/k8s/Taskfile.dist.yaml index bc301f71fa..8d5ea53e0a 100644 --- a/.taskfiles/k8s/Taskfile.dist.yaml +++ b/.taskfiles/k8s/Taskfile.dist.yaml @@ -132,6 +132,7 @@ tasks: - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/${APPNAME}/{{.APP}}/g' % # uppercase, for variable substitution references e.g. ${APP_DNS_AUTHENTIK} - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME}/_{{.APP}}}/g;s/\(_{{.APP}}}\)/\U\1/g' % + - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME:=/_{{.APP}}:=/g' % shortnames: desc: List all installed CRDs and their short names. @@ -146,6 +147,7 @@ tasks: - |- while true; do kubectl delete pod -A --field-selector=status.phase==Failed || true; + kubectl delete pod -A --field-selector=status.phase==Pending || true; kubectl delete pod -A --field-selector=status.phase==Succeeded || true; done @@ -153,9 +155,12 @@ tasks: aliases: [delpvc] desc: Delete PVC which is stuck, e.g. if it's a local-path PVC and the node has been wiped and reset. vars: + #NS: '{{.NS | default "default"}}' NS: '{{ or .NS (fail "Missing `NS` environment variable!") }}' PVC: '{{ or .PVC (fail "Missing `PVC` environment variable!") }}' cmds: + #- | + # kubectl get pvc -n {{.NS}} {{.PVC}} -o jsonpath='{.spec.volumeName}' - | kubectl delete pvc -n {{.NS}} {{.PVC}} --wait=false - | @@ -163,6 +168,17 @@ tasks: - | until kubectl wait --for=delete pvc -n {{.NS}} {{.PVC}} --timeout=3600s; do sleep 1; done + delete-all-reset-node-pvcs: + desc: Delete all PVCs that are stuck because they're tied to a node that has been reset. + vars: + NODE: '{{ or .NODE (fail "Missing `NODE` environment variable!") }}' + FAILCONTINUE: '{{ .FAILCONTINUE | default "false" }}' + cmds: + - | + kubectl get pv -o jsonpath='{range .items[?(@.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]=="{{.NODE}}")]}{"NS="}{.spec.claimRef.namespace}{" "}{"PVC="}{.spec.claimRef.name}{"\n"}{end}' | while read -r i; do + task k8s:delete-stuck-pvc ${i} || {{.FAILCONTINUE}} + done + iperf2: desc: Start a iperf2 server on one node, and iperf2 client on another node, to benchmark network performance. dir: "/{{.ROOT_DIR}}/.taskfiles/k8s/template/iperf2" diff --git a/.taskfiles/rook/Taskfile.dist.yaml b/.taskfiles/rook/Taskfile.dist.yaml index 46453f6b0f..31c6a1e1e0 100644 --- a/.taskfiles/rook/Taskfile.dist.yaml +++ b/.taskfiles/rook/Taskfile.dist.yaml @@ -148,6 +148,19 @@ tasks: NODE: "humming" CEPH_DISK: "/dev/disk/by-id/ata-INTEL_SSDSC2BB016T4_BTWD709202L91P6HGN" + wipe-nodes-sinon: + desc: Wipe all nodes in cluster "sinon" + dir: '/{{.ROOT_DIR}}/.taskfiles/rook' + cmds: + - task: wipe-node + vars: + NODE: "sinon" + CEPH_DISK: "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_5QKKKY0B" + - task: wipe-node + vars: + NODE: "sinon" + CEPH_DISK: "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_5QKLS2WB" + force-delete-cluster: desc: |- Sometimes Rook seems to fail applying the cluster and want to delete it before it even gets anything set up, this Task will force delete all finalizers to delete all unready Ceph resources. @@ -174,9 +187,9 @@ tasks: vars: C: '{{ or .C (fail "Missing `C` environment variable for cluster!") }}' cmds: - - task: cluster:cluster-switch - vars: - C: '{{.C}}' + #- task: cluster:cluster-switch + # vars: + # C: '{{.C}}' - flux suspend ks 0-{{.C}}-config - flux suspend ks 1-core-storage-rook-ceph-app - flux suspend ks 1-core-storage-rook-ceph-cluster-{{.C}} diff --git a/.taskfiles/talos/Taskfile.dist.yaml b/.taskfiles/talos/Taskfile.dist.yaml index df676339e4..57a85cb477 100644 --- a/.taskfiles/talos/Taskfile.dist.yaml +++ b/.taskfiles/talos/Taskfile.dist.yaml @@ -41,7 +41,7 @@ tasks: cmds: - pwd - op user get --me # check signin status, fail if not signed in - - op run -- task --taskfile /{{.ROOT_DIR}}/.taskfiles/bootstrap/Taskfile.dist.yaml 1p-vars-env-run C={{.C}} -- talhelper genconfig + - op --cache=false run -- task --taskfile /{{.ROOT_DIR}}/.taskfiles/bootstrap/Taskfile.dist.yaml 1p-vars-env-run C={{.C}} -- talhelper genconfig bootstrap: aliases: [bs] @@ -74,3 +74,12 @@ tasks: NODE: '{{ or .NODE (fail "Missing `NODE` environment variable!") }}' cmds: - talosctl reboot --nodes $(kubectl get nodes {{.NODE}} -o yaml | yq '.status.addresses | map(select(.type == "InternalIP")) | .[].address') + + gensecret: + desc: Generate secrets and push to 1Password + dir: "/{{.ROOT_DIR}}/kube/clusters/{{.C}}/talos" + vars: + C: '{{ or .C (fail "Missing C environment variable for cluster!") }}' + cmds: + - op user get --me # check signin status, fail if not signed in + - export $(talhelper gensecret | yq --input-format yaml --output-format shell | sed -e 's/\'$//g' -e 's/=\'/=/g'); op item create --category=PASSWORD --title='.{{.C}}-talos' --vault='{{.C}}' --generate-password='64,letters,digits,symbols' Secrets.cluster_id[password]="$cluster_id" Secrets.cluster_secret[password]="$cluster_secret" Secrets.secrets_bootstraptoken[password]="$secrets_bootstraptoken" Secrets.secrets_secretboxencryptionsecret[password]="$secrets_secretboxencryptionsecret" Secrets.trustdinfo_token[password]="$trustdinfo_token" Secrets.certs_etcd_crt[password]="$certs_etcd_crt" Secrets.certs_etcd_key[password]="$certs_etcd_key" Secrets.certs_k8s_crt[password]="$certs_k8s_crt" Secrets.certs_k8s_key[password]="$certs_k8s_key" Secrets.certs_k8saggregator_crt[password]="$certs_k8saggregator_crt" Secrets.certs_k8saggregator_key[password]="$certs_k8saggregator_key" Secrets.certs_k8sserviceaccount_key[password]="$certs_k8sserviceaccount_key" Secrets.certs_os_crt[password]="$certs_os_crt" Secrets.certs_os_key[password]="$certs_os_key" # TODO: this is a fish command, check if it works on Task's sh interpreter diff --git a/.taskfiles/talos/talhelper-secrets-1p.env b/.taskfiles/talos/talhelper-secrets-1p.env new file mode 100644 index 0000000000..de01eb8c43 --- /dev/null +++ b/.taskfiles/talos/talhelper-secrets-1p.env @@ -0,0 +1,14 @@ +cluster_id="op://sinon/.sinon-talos/Secrets/cluster_id" +cluster_secret="op://sinon/.sinon-talos/Secrets/cluster_secret" +secrets_bootstraptoken="op://sinon/.sinon-talos/Secrets/secrets_bootstraptoken" +secrets_secretboxencryptionsecret="op://sinon/.sinon-talos/Secrets/secrets_secretboxencryptionsecret" +trustdinfo_token="op://sinon/.sinon-talos/Secrets/trustdinfo_token" +certs_etcd_crt="op://sinon/.sinon-talos/Secrets/certs_etcd_crt" +certs_etcd_key="op://sinon/.sinon-talos/Secrets/certs_etcd_key" +certs_k8s_crt="op://sinon/.sinon-talos/Secrets/certs_k8s_crt" +certs_k8s_key="op://sinon/.sinon-talos/Secrets/certs_k8s_key" +certs_k8saggregator_crt="op://sinon/.sinon-talos/Secrets/certs_k8saggregator_crt" +certs_k8saggregator_key="op://sinon/.sinon-talos/Secrets/certs_k8saggregator_key" +certs_k8sserviceaccount_key="op://sinon/.sinon-talos/Secrets/certs_k8sserviceaccount_key" +certs_os_crt="op://sinon/.sinon-talos/Secrets/certs_os_crt" +certs_os_key="op://sinon/.sinon-talos/Secrets/certs_os_key" \ No newline at end of file