update airgap create functions to support multi-pool configurations

.github/workflows/stale-prs.yml

@@ -12,11 +12,11 @@ jobs:
 
     steps:      
       - name: Checkout code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
 
       - name: Check PRs and compile report
         id: compile_report
-        uses: actions/github-script@211cb3fefb35a799baa5156f9321bb774fe56294
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
         with:
           script: |
             const fiveDaysAgo = new Date(new Date().setDate(new Date().getDate() - 5)); 
@@ -51,8 +51,11 @@ jobs:
                     per_page: 100,
                   });
                   
+                  if (pr.draft) {
+                    continue;
+                  }
                   if ( reviews.data.length == 0 && createdDate < fiveDaysAgo ) {
-                    noReviewsReport += `${pr.title} #${pr.number} ${pr.html_url} ${pr.user.login}\n`;
+                    noReviewsReport += `${pr.title} | #${pr.number} ${pr.html_url} by ${pr.user.login}\n`;
                   } 
                   else {
                       for (const review of reviews.data) {
@@ -68,37 +71,37 @@ jobs:
                     }
                   }
                 } catch( error ){
-                  hasErrors += `${pr.title} (#${pr.number}) ${pr.html_url} - Error: ${error} \n`;
+                  hasErrors += `${pr.title} | (#${pr.number}) ${pr.html_url} - Error: ${error} \n`;
                 }
                 if ( lastReviewDate && (lastReviewDate < threeDaysAgo && updatedDate <= lastReviewDate) && statesCompare.includes(lastReviewStateOne) && statesCompare.includes(lastReviewStateTwo) ) {
-                  needsChanges += `${pr.title} (#${pr.number}) ${pr.html_url} ${pr.user.login} \n`;
+                  needsChanges += `${pr.title} | (#${pr.number}) ${pr.html_url} by ${pr.user.login} \n`;
                   
                 } else if ( lastReviewDate && (updatedDate > lastReviewDate) && (updatedDate < threeDaysAgo) ){
-                  needsReviews += `${pr.title} (#${pr.number}) ${pr.html_url} ${pr.user.login} \n`;
+                  needsReviews += `${pr.title} |  (#${pr.number}) ${pr.html_url} by ${pr.user.login} \n`;
 
                 }  else if ( lastReviewDate && numberOfApprovedState >=2 && (lastReviewDate < threeDaysAgo) && !statesCompare.includes(lastReviewStateOne) ) {
-                  needsMerge += `${pr.title} (#${pr.number}) ${pr.html_url} ${pr.user.login} \n`;
+                  needsMerge += `${pr.title} | (#${pr.number}) ${pr.html_url} by ${pr.user.login} \n`;
                 }
               }
               if ( noReviewsReport ) {
-                report += `PRs with no reviews yet: \n` + noReviewsReport +`\n\n`
+                report += `*PRs with no reviews yet:* \n` + noReviewsReport +`\n\n`
               }
               if ( needsReviews ) {
-                report += `PRs that need reviews: \n`+ needsReviews +`\n\n` 
+                report += `*PRs that need reviews:* \n`+ needsReviews +`\n\n` 
               }
               if ( needsChanges ) {
-                report += `PRs with changes requested: \n`+ needsChanges +`\n\n`
+                report += `*PRs with changes requested:* \n`+ needsChanges +`\n\n`
               }
               if ( needsMerge ) {
-                report += `PRs that need to be merged: \n` + needsMerge
+                report += `*PRs that need to be merged:* \n` + needsMerge
               }
               if ( hasErrors ) {
-                report += `PRs that have errors when fetching: \n` + hasErrors
+                report += `*PRs that have errors when fetching:* \n` + hasErrors
               }
               return report;
 
             } catch( error ){
-              report += `There is an error getting PR Status report - Error: ${error} \n`;
+              report += `*There is an error getting PR Status report - Error:* ${error} \n`;
               return report;
             }
            
@@ -108,8 +111,8 @@ jobs:
         run: |
           result="${{ steps.compile_report.outputs.result }}"
           if [[ -n "$result" ]]; then 
-            curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"PR Status Report:\n$result\"}" ${{ secrets.SLACK_WEBHOOK_URL }}
+            curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"*PR Status Report:*\n\n$result\"}" ${{ secrets.SLACK_WEBHOOK_URL }}
           else
-            curl -X POST -H 'Content-type: application/json' --data '{"text":"PR Status Report: There are errors please check the GH actions logs"}' ${{ secrets.SLACK_WEBHOOK_URL }}
+            curl -X POST -H 'Content-type: application/json' --data '{"text":"*PR Status Report:* There are errors please check the GH actions logs \n\n"}' ${{ secrets.SLACK_WEBHOOK_URL }}
           fi
-        shell: bash
+        shell: bash

clients/corral/config.go

@@ -25,6 +25,14 @@ type Packages struct {
 	HasCustomRepo       string            `json:"hasCustomRepo" yaml:"hasCustomRepo"`
 }
 
+// Args is a struct that contains arguments to a corral create command, and any updates to the config
+// that should be applied before creating the corral
+type Args struct {
+	Name        string
+	PackageName string
+	Updates     map[string]string
+}
+
 // PackagesConfig is a function that reads in the corral package object from the config file
 func PackagesConfig() *Packages {
 	var corralPackages Packages

clients/corral/corral.go

@@ -4,22 +4,24 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/shepherd/pkg/session"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
-	debugFlag               = "--trace"
-	skipCleanupFlag         = "--skip-cleanup"
-	corralPrivateSSHKey     = "corral_private_key"
-	corralPublicSSHKey      = "corral_public_key"
-	corralRegistryIP        = "registry_ip"
-	corralRegistryPrivateIP = "registry_private_ip"
+	debugFlag           = "--trace"
+	skipCleanupFlag     = "--skip-cleanup"
+	corralPrivateSSHKey = "corral_private_key"
+	corralPublicSSHKey  = "corral_public_key"
 )
 
 // GetCorralEnvVar gets corral environment variables
@@ -72,29 +74,153 @@ func SetCustomRepo(repo string) error {
 
 // CreateCorral creates a corral taking the corral name, the package path, and a debug set so if someone wants to view the
 // corral create log
-func CreateCorral(ts *session.Session, corralName, packageName string, debug bool, cleanup bool) ([]byte, error) {
+func CreateCorral(ts *session.Session, corralName, packageName string, debug, cleanup bool) ([]byte, error) {
+	command, err := startCorral(ts, corralName, packageName, debug, cleanup)
+	if err != nil {
+		return nil, err
+	}
+
+	return runAndWaitOnCommand(command)
+}
+
+func runAndWaitOnCommand(command *exec.Cmd) ([]byte, error) {
+	err := command.Wait()
+	var msg []byte
+	if command.Stdout != nil {
+		msg = command.Stdout.(*bytes.Buffer).Bytes()
+	}
+
+	if msg != nil {
+		logrus.Infof("Stdout: %s", string(msg))
+	}
+
+	return msg, errors.Wrap(err, "Debug: "+string(msg))
+}
+
+func startCorral(ts *session.Session, corralName, packageName string, debug, cleanup bool) (*exec.Cmd, error) {
 	ts.RegisterCleanupFunc(func() error {
 		return DeleteCorral(corralName)
 	})
 
 	args := []string{"create"}
+
 	if !cleanup {
 		args = append(args, skipCleanupFlag)
 	}
 	if debug {
 		args = append(args, debugFlag)
 	}
+
 	args = append(args, corralName, packageName)
 	logrus.Infof("Creating corral with the following parameters: %v", args)
-	// complicated, but running the command in a way that allows us to
-	// capture the output and error(s) and print it to the console
-	msg, err := exec.Command("corral", args...).CombinedOutput()
-	logrus.Infof("Corral create output: %s", string(msg))
+
+	cmdToRun := exec.Command("corral", args...)
+
+	// create a buffer for stdout/stderr so we can read from it later. commands initiate this to nil by default.
+	var b bytes.Buffer
+	cmdToRun.Stdout = &b
+	cmdToRun.Stderr = &b
+	err := cmdToRun.Start()
 	if err != nil {
-		return nil, errors.Wrap(err, "Unable to create corral: "+string(msg))
+		return nil, err
+	}
+
+	// this ensures corral is completely initiated. Otherwise, race conditions occur.
+	err = waitForCorralConfig(corralName)
+	if err != nil {
+		return nil, err
+	}
+
+	return cmdToRun, err
+}
+
+func waitForCorralConfig(corralName string) error {
+	backoff := wait.Backoff{
+		Duration: 1 * time.Second,
+		Factor:   1.1,
+		Jitter:   0.1,
+		Steps:    10,
+	}
+
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+
+	corralOSPath := homeDir + "/.corral/corrals/" + corralName + "/corral.yaml"
+
+	return wait.ExponentialBackoff(backoff, func() (finished bool, err error) {
+		_, err = os.Stat(corralOSPath)
+		if err != nil {
+			return false, nil
+		}
+
+		fileContents, err := os.ReadFile(corralOSPath)
+		if err != nil {
+			return false, nil
+		}
+
+		if len(string(fileContents)) <= 0 {
+			return false, nil
+		}
+
+		return true, err
+	})
+}
+
+// CreateMultipleCorrals creates corrals taking the corral name, the package path, and a debug set so if someone wants to view the
+// corral create log. Using this function implies calling WaitOnCorralWithCombinedOutput to get the output once finished.
+func CreateMultipleCorrals(ts *session.Session, commands []Args, debug, cleanup bool) ([][]byte, error) {
+	var waitGroup sync.WaitGroup
+
+	var msgs [][]byte
+	var errStrings []string
+
+	for _, currentCommand := range commands {
+		// break out of any error that comes up before we run the waitGroup, to avoid running if we're already in an error state.
+		for key, value := range currentCommand.Updates {
+			logrus.Info(key, ": ", value)
+			err := UpdateCorralConfig(key, value)
+			if err != nil {
+				errStrings = append(errStrings, fmt.Sprint(err.Error(), "Unable to update corral: "+currentCommand.Name+" for "+key+": "+value))
+				break
+			}
+		}
+
+		cmdToRun, err := startCorral(ts, currentCommand.Name, currentCommand.PackageName, debug, cleanup)
+		if err != nil {
+			errStrings = append(errStrings, err.Error())
+			break
+		}
+
+		waitGroup.Add(1)
+
+		go func() {
+			defer waitGroup.Done()
+
+			msg, err := runAndWaitOnCommand(cmdToRun)
+			if err != nil {
+				errStrings = append(errStrings, err.Error())
+			}
+
+			msgs = append(msgs, msg)
+		}()
+
 	}
 
-	return msg, nil
+	waitGroup.Wait()
+
+	var formattedError error
+	var longString string
+	if len(errStrings) > 0 {
+		for _, err := range errStrings {
+			longString += err
+		}
+		formattedError = fmt.Errorf(longString)
+	}
+
+	logrus.Info("done with registration")
+	return msgs, formattedError
 }
 
 // DeleteCorral deletes a corral based on the corral name
@@ -219,23 +345,3 @@ func SetCorralSSHKeys(corralName string) error {
 
 	return UpdateCorralConfig(corralPublicSSHKey, publicSSHkey)
 }
-
-// SetCorralBastion is a helper function that will set the corral bastion private and pulic addresses previously generated by `corralName`
-func SetCorralBastion(corralName string) error {
-	bastion_ip, err := GetCorralEnvVar(corralName, corralRegistryIP)
-	if err != nil {
-		return err
-	}
-
-	err = UpdateCorralConfig(corralRegistryIP, bastion_ip)
-	if err != nil {
-		return err
-	}
-
-	bastion_internal_ip, err := GetCorralEnvVar(corralName, corralRegistryPrivateIP)
-	if err != nil {
-		return err
-	}
-
-	return UpdateCorralConfig(corralRegistryPrivateIP, bastion_internal_ip)
-}