From 37c1dcebd4489f88d6e60de10ea89cac1caec26b Mon Sep 17 00:00:00 2001
From: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
Date: Mon, 13 Jan 2025 09:29:47 +0000
Subject: [PATCH 1/4] Fix nightly cleanup to deal with duplicate server names

---
 .github/workflows/nightly-cleanup.yml | 40 +++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
index f76bd51a9..577a20775 100644
--- a/.github/workflows/nightly-cleanup.yml
+++ b/.github/workflows/nightly-cleanup.yml
@@ -63,15 +63,43 @@ jobs:
             echo "No clusters to delete."
             exit 0
           fi
-          
+      
           for cluster_prefix in ${ci_clusters}
           do
             echo "Processing cluster: $cluster_prefix"
-            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
-            if [[ $TAGS =~ "keep" ]]; then
-              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
-            else
-              ./dev/delete-cluster.py ${cluster_prefix} --force
+      
+            # Retrieve all servers matching the cluster prefix
+            SERVERS=$(openstack server list --name "${cluster_prefix}-.*" -f value -c ID -c Name)
+      
+            if [[ -z "$SERVERS" ]]; then
+              echo "No servers found for cluster ${cluster_prefix}"
+              continue
+            fi
+      
+            KEEP_FLAG=false
+            while IFS= read -r line; do
+              SERVER_ID=$(echo "$line" | awk '{print $1}')
+              SERVER_NAME=$(echo "$line" | awk '{print $2}')
+      
+              # Check tags only on control nodes
+              if [[ "$SERVER_NAME" == "${cluster_prefix}-control" ]]; then
+                TAGS=$(openstack server show $SERVER_ID --column tags --format value)
+                if [[ $TAGS =~ "keep" ]]; then
+                  echo "Skipping cluster ${cluster_prefix} - control instance is tagged as keep"
+                  KEEP_FLAG=true
+                  break
+                fi
+              fi
+            done <<< "$SERVERS"
+      
+            # Delete all servers if control node is not tagged with keep
+            if [[ "$KEEP_FLAG" == false ]]; then
+              echo "Deleting all servers in cluster ${cluster_prefix}"
+              while IFS= read -r line; do
+                SERVER_ID=$(echo "$line" | awk '{print $1}')
+                echo "Deleting server $SERVER_ID"
+                openstack server delete $SERVER_ID || true
+              done <<< "$SERVERS"
             fi
           done
         shell: bash

From 9b1bf122847f8345ac70e764fd81300829de73d0 Mon Sep 17 00:00:00 2001
From: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
Date: Mon, 13 Jan 2025 10:03:32 +0000
Subject: [PATCH 2/4] Update nightly-cleanup.yml

---
 .github/workflows/nightly-cleanup.yml | 40 ++++-----------------------
 1 file changed, 6 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
index 577a20775..8ea3ca74d 100644
--- a/.github/workflows/nightly-cleanup.yml
+++ b/.github/workflows/nightly-cleanup.yml
@@ -63,43 +63,15 @@ jobs:
             echo "No clusters to delete."
             exit 0
           fi
-      
+           
           for cluster_prefix in ${ci_clusters}
           do
             echo "Processing cluster: $cluster_prefix"
-      
-            # Retrieve all servers matching the cluster prefix
-            SERVERS=$(openstack server list --name "${cluster_prefix}-.*" -f value -c ID -c Name)
-      
-            if [[ -z "$SERVERS" ]]; then
-              echo "No servers found for cluster ${cluster_prefix}"
-              continue
-            fi
-      
-            KEEP_FLAG=false
-            while IFS= read -r line; do
-              SERVER_ID=$(echo "$line" | awk '{print $1}')
-              SERVER_NAME=$(echo "$line" | awk '{print $2}')
-      
-              # Check tags only on control nodes
-              if [[ "$SERVER_NAME" == "${cluster_prefix}-control" ]]; then
-                TAGS=$(openstack server show $SERVER_ID --column tags --format value)
-                if [[ $TAGS =~ "keep" ]]; then
-                  echo "Skipping cluster ${cluster_prefix} - control instance is tagged as keep"
-                  KEEP_FLAG=true
-                  break
-                fi
-              fi
-            done <<< "$SERVERS"
-      
-            # Delete all servers if control node is not tagged with keep
-            if [[ "$KEEP_FLAG" == false ]]; then
-              echo "Deleting all servers in cluster ${cluster_prefix}"
-              while IFS= read -r line; do
-                SERVER_ID=$(echo "$line" | awk '{print $1}')
-                echo "Deleting server $SERVER_ID"
-                openstack server delete $SERVER_ID || true
-              done <<< "$SERVERS"
+            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
+            if [[ $TAGS =~ "keep" ]]; then
+              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+            else
+              ./dev/delete-cluster.py ${cluster_prefix} --force
             fi
           done
         shell: bash

From f1fd75e772d4c9122bb7fcb79279c7a26b2f5f5b Mon Sep 17 00:00:00 2001
From: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
Date: Mon, 13 Jan 2025 10:21:42 +0000
Subject: [PATCH 3/4] Update nightly-cleanup.yml

---
 .github/workflows/nightly-cleanup.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
index 8ea3ca74d..e15049f08 100644
--- a/.github/workflows/nightly-cleanup.yml
+++ b/.github/workflows/nightly-cleanup.yml
@@ -63,11 +63,20 @@ jobs:
             echo "No clusters to delete."
             exit 0
           fi
-           
+          
           for cluster_prefix in ${ci_clusters}
           do
             echo "Processing cluster: $cluster_prefix"
-            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+            SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length)
+            
+            if [[ $SERVER_COUNT -gt 1 ]]; then
+              echo "Warning: More than one server found for control node '${cluster_prefix}-control'."
+              continue
+            fi
+            TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags' )
+            
             if [[ $TAGS =~ "keep" ]]; then
               echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
             else

From edbcebc09b1321c86bcea1a7f3f181ae70b7ac14 Mon Sep 17 00:00:00 2001
From: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
Date: Mon, 13 Jan 2025 13:39:06 +0000
Subject: [PATCH 4/4] Fix tag determination

---
 .github/workflows/nightly-cleanup.yml | 29 ++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
index e15049f08..0f7156fad 100644
--- a/.github/workflows/nightly-cleanup.yml
+++ b/.github/workflows/nightly-cleanup.yml
@@ -70,17 +70,28 @@ jobs:
             # Get all servers with the matching name for control node
             CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
             SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length)
-            
+
             if [[ $SERVER_COUNT -gt 1 ]]; then
-              echo "Warning: More than one server found for control node '${cluster_prefix}-control'."
-              continue
-            fi
-            TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags' )
-            
-            if [[ $TAGS =~ "keep" ]]; then
-              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+              echo "Multiple servers found for control node '${cluster_prefix}-control'. Checking tags for each..."
+              
+              for server in $(echo "$CONTROL_SERVERS" | jq -r '.[].ID'); do
+                # Get tags for each control node
+                TAGS=$(openstack server show "$server" --column tags --format value)
+                
+                if [[ $TAGS =~ "keep" ]]; then
+                  echo "Skipping ${cluster_prefix} (server ${server}) - control instance is tagged as keep"
+                else
+                  ./dev/delete-cluster.py ${cluster_prefix} --force
+                fi
+              done
             else
-              ./dev/delete-cluster.py ${cluster_prefix} --force
+              # If only one server, extract its tags and proceed
+              TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags')
+              if [[ $TAGS =~ "keep" ]]; then
+                echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+              else
+                ./dev/delete-cluster.py ${cluster_prefix} --force
+              fi
             fi
           done
         shell: bash