feat(taskfiles): delete-all-reset-node-pvcs, cleanup

.taskfiles/bootstrap/Taskfile.dist.yaml

@@ -27,8 +27,10 @@ tasks:
       PRE_ARGS: '{{.PRE_ARGS}}'
       POST_ARGS: '{{.POST_ARGS}}'
     cmds:
+      #- |
+      #  {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + (.value | @sh)) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}}
       - |
-        {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + (.value | @sh)) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}}
+        {{.PRE_ARGS}}op item get --vault {{.C}} ".{{.C}}-vars" --format=json | yq --input-format json '.fields | with(.[]; (.label | key) = "key") | (.[] | select(has("value") | not) | .value) |= "" | map(.key + "=" + .value) | .[]'{{.POST_ARGS}} {{.CLI_ARGS}}
 
   1p-vars-env-run:
     dir: '{{.USER_WORKING_DIR}}'
@@ -70,8 +72,8 @@ tasks:
       - task: 1p-vars-env-shell
         vars:
           C: *c
-          POST_ARGS: " | kubectl create secret generic -n flux-system sinon-vars --from-env-file=/dev/stdin"
-      - kubectl create secret generic -n flux-system sinon-secrets # currently nothing hopefully maybe?
+          POST_ARGS: " | kubectl create secret generic -n flux-system {{.C}}-vars --from-env-file=/dev/stdin"
+      - kubectl create secret generic -n flux-system {{.C}}-secrets # currently nothing hopefully maybe?
       - op read "op://{{.C}}/Flux/agekey" | kubectl create secret generic -n flux-system agekey --from-file=age.agekey=/dev/stdin
       - yq '.spec.values | explode(.)' ./kube/deploy/core/secrets/external-secrets/app/hr.yaml | helm install -n external-secrets external-secrets oci://ghcr.io/external-secrets/charts/external-secrets --version $(yq '.spec.chart.spec.version' ./kube/deploy/core/secrets/external-secrets/app/hr.yaml) -f -
-      - kubectl apply --server-side -f /{{.ROOT_DIR}}/kube/clusters/sinon/flux/flux-repo.yaml
+      - kubectl apply --server-side -f /{{.ROOT_DIR}}/kube/clusters/{{.C}}/flux/flux-repo.yaml

.taskfiles/k8s/Taskfile.dist.yaml

@@ -132,6 +132,7 @@ tasks:
       - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/${APPNAME}/{{.APP}}/g' %
       # uppercase, for variable substitution references e.g. ${APP_DNS_AUTHENTIK}
       - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME}/_{{.APP}}}/g;s/\(_{{.APP}}}\)/\U\1/g' %
+      - grep -lR 'APPNAME' ./kube/deploy/apps/{{.APP}}/ | xargs -I% sed -i 's/_APPNAME:=/_{{.APP}}:=/g' %
 
   shortnames:
     desc: List all installed CRDs and their short names.
@@ -146,23 +147,38 @@ tasks:
       - |-
         while true; do
           kubectl delete pod -A --field-selector=status.phase==Failed || true;
+          kubectl delete pod -A --field-selector=status.phase==Pending || true;
           kubectl delete pod -A --field-selector=status.phase==Succeeded || true;
         done
 
   delete-stuck-pvc:
     aliases: [delpvc]
     desc: Delete PVC which is stuck, e.g. if it's a local-path PVC and the node has been wiped and reset.
     vars:
+      #NS: '{{.NS | default "default"}}'
       NS: '{{ or .NS (fail "Missing `NS` environment variable!") }}'
       PVC: '{{ or .PVC (fail "Missing `PVC` environment variable!") }}'
     cmds:
+      #- |
+      #  kubectl get pvc -n {{.NS}} {{.PVC}} -o jsonpath='{.spec.volumeName}'
       - |
         kubectl delete pvc -n {{.NS}} {{.PVC}} --wait=false
       - |
         kubectl patch pvc -n {{.NS}} {{.PVC}} --type='json' -p '[{"op": "remove", "path": "/metadata/finalizers"}]'
       - |
         until kubectl wait --for=delete pvc -n {{.NS}} {{.PVC}} --timeout=3600s; do sleep 1; done
 
+  delete-all-reset-node-pvcs:
+    desc: Delete all PVCs that are stuck because they're tied to a node that has been reset.
+    vars:
+      NODE: '{{ or .NODE (fail "Missing `NODE` environment variable!") }}'
+      FAILCONTINUE: '{{ .FAILCONTINUE | default "false" }}'
+    cmds:
+      - |
+        kubectl get pv -o jsonpath='{range .items[?(@.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]=="{{.NODE}}")]}{"NS="}{.spec.claimRef.namespace}{" "}{"PVC="}{.spec.claimRef.name}{"\n"}{end}' | while read -r i; do
+          task k8s:delete-stuck-pvc ${i} || {{.FAILCONTINUE}}
+        done
+
   iperf2:
     desc: Start a iperf2 server on one node, and iperf2 client on another node, to benchmark network performance.
     dir: "/{{.ROOT_DIR}}/.taskfiles/k8s/template/iperf2"

.taskfiles/rook/Taskfile.dist.yaml

@@ -148,6 +148,19 @@ tasks:
           NODE: "humming"
           CEPH_DISK: "/dev/disk/by-id/ata-INTEL_SSDSC2BB016T4_BTWD709202L91P6HGN"
 
+  wipe-nodes-sinon:
+    desc: Wipe all nodes in cluster "sinon"
+    dir: '/{{.ROOT_DIR}}/.taskfiles/rook'
+    cmds:
+      - task: wipe-node
+        vars:
+          NODE: "sinon"
+          CEPH_DISK: "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_5QKKKY0B"
+      - task: wipe-node
+        vars:
+          NODE: "sinon"
+          CEPH_DISK: "/dev/disk/by-id/ata-WDC_WD120EFBX-68B0EN0_5QKLS2WB"
+
   force-delete-cluster:
     desc: |-
       Sometimes Rook seems to fail applying the cluster and want to delete it before it even gets anything set up, this Task will force delete all finalizers to delete all unready Ceph resources.
@@ -174,9 +187,9 @@ tasks:
     vars:
       C: '{{ or .C (fail "Missing `C` environment variable for cluster!") }}'
     cmds:
-      - task: cluster:cluster-switch
-        vars:
-          C: '{{.C}}'
+      #- task: cluster:cluster-switch
+      #  vars:
+      #    C: '{{.C}}'
       - flux suspend ks 0-{{.C}}-config
       - flux suspend ks 1-core-storage-rook-ceph-app
       - flux suspend ks 1-core-storage-rook-ceph-cluster-{{.C}}

.taskfiles/talos/Taskfile.dist.yaml

@@ -41,7 +41,7 @@ tasks:
     cmds:
       - pwd
       - op user get --me # check signin status, fail if not signed in
-      - op run -- task --taskfile /{{.ROOT_DIR}}/.taskfiles/bootstrap/Taskfile.dist.yaml 1p-vars-env-run C={{.C}} -- talhelper genconfig
+      - op --cache=false run -- task --taskfile /{{.ROOT_DIR}}/.taskfiles/bootstrap/Taskfile.dist.yaml 1p-vars-env-run C={{.C}} -- talhelper genconfig
 
   bootstrap:
     aliases: [bs]
@@ -74,3 +74,12 @@ tasks:
       NODE: '{{ or .NODE (fail "Missing `NODE` environment variable!") }}'
     cmds:
       - talosctl reboot --nodes $(kubectl get nodes {{.NODE}} -o yaml | yq '.status.addresses | map(select(.type == "InternalIP")) | .[].address')
+
+  gensecret:
+    desc: Generate secrets and push to 1Password
+    dir: "/{{.ROOT_DIR}}/kube/clusters/{{.C}}/talos"
+    vars:
+      C: '{{ or .C (fail "Missing C environment variable for cluster!") }}'
+    cmds:
+      - op user get --me # check signin status, fail if not signed in
+      - export $(talhelper gensecret | yq --input-format yaml --output-format shell | sed -e 's/\'$//g' -e 's/=\'/=/g'); op item create --category=PASSWORD --title='.{{.C}}-talos' --vault='{{.C}}' --generate-password='64,letters,digits,symbols' Secrets.cluster_id[password]="$cluster_id" Secrets.cluster_secret[password]="$cluster_secret" Secrets.secrets_bootstraptoken[password]="$secrets_bootstraptoken" Secrets.secrets_secretboxencryptionsecret[password]="$secrets_secretboxencryptionsecret" Secrets.trustdinfo_token[password]="$trustdinfo_token" Secrets.certs_etcd_crt[password]="$certs_etcd_crt" Secrets.certs_etcd_key[password]="$certs_etcd_key" Secrets.certs_k8s_crt[password]="$certs_k8s_crt" Secrets.certs_k8s_key[password]="$certs_k8s_key" Secrets.certs_k8saggregator_crt[password]="$certs_k8saggregator_crt" Secrets.certs_k8saggregator_key[password]="$certs_k8saggregator_key" Secrets.certs_k8sserviceaccount_key[password]="$certs_k8sserviceaccount_key" Secrets.certs_os_crt[password]="$certs_os_crt" Secrets.certs_os_key[password]="$certs_os_key" # TODO: this is a fish command, check if it works on Task's sh interpreter

.taskfiles/talos/talhelper-secrets-1p.env

@@ -0,0 +1,14 @@
+cluster_id="op://sinon/.sinon-talos/Secrets/cluster_id"
+cluster_secret="op://sinon/.sinon-talos/Secrets/cluster_secret"
+secrets_bootstraptoken="op://sinon/.sinon-talos/Secrets/secrets_bootstraptoken"
+secrets_secretboxencryptionsecret="op://sinon/.sinon-talos/Secrets/secrets_secretboxencryptionsecret"
+trustdinfo_token="op://sinon/.sinon-talos/Secrets/trustdinfo_token"
+certs_etcd_crt="op://sinon/.sinon-talos/Secrets/certs_etcd_crt"
+certs_etcd_key="op://sinon/.sinon-talos/Secrets/certs_etcd_key"
+certs_k8s_crt="op://sinon/.sinon-talos/Secrets/certs_k8s_crt"
+certs_k8s_key="op://sinon/.sinon-talos/Secrets/certs_k8s_key"
+certs_k8saggregator_crt="op://sinon/.sinon-talos/Secrets/certs_k8saggregator_crt"
+certs_k8saggregator_key="op://sinon/.sinon-talos/Secrets/certs_k8saggregator_key"
+certs_k8sserviceaccount_key="op://sinon/.sinon-talos/Secrets/certs_k8sserviceaccount_key"
+certs_os_crt="op://sinon/.sinon-talos/Secrets/certs_os_crt"
+certs_os_key="op://sinon/.sinon-talos/Secrets/certs_os_key"