Skip to content

Commit

Permalink
extend boskosctl (especially for other repos / older branches)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sbueringer
sbueringer committed Jun 7, 2024
1 parent 82b5a32 commit 02a536f
Show file tree
Hide file tree
Showing 4 changed files with 153 additions and 49 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505
github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20240404200847-de75746a9505
github.com/vmware/govmomi v0.37.2
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/mod v0.17.0
golang.org/x/tools v0.21.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
2 changes: 1 addition & 1 deletion hack/e2e.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ on_exit() {
[[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"

# If Boskos is being used then release the vsphere project.
[ -z "${BOSKOS_HOST:-}" ] || docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --resource-folder="${BOSKOS_RESOURCE_FOLDER}" --resource-pool="${BOSKOS_RESOURCE_POOL}"
[ -z "${BOSKOS_HOST:-}" ] || docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-username="${VSPHERE_USERNAME}" --vsphere-password="${VSPHERE_PASSWORD}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
fi

# kill the VPN only when we started it (not vcsim)
Expand Down
197 changes: 149 additions & 48 deletions hack/tools/boskosctl/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,17 @@ package main

import (
"context"
"encoding/json"
"fmt"
"net/netip"
"os"
"strconv"
"strings"
"time"

"github.com/pkg/errors"
"github.com/spf13/cobra"
"go4.org/netipx"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -34,12 +39,16 @@ import (
)

var (
boskosHost string
resourceOwner string
resourceType string
resourceName string
resourceFolder string
resourcePool string
boskosHost string
resourceOwner string
resourceType string
resourceName string
vSphereUsername string
vSpherePassword string
vSphereServer string
vSphereTLSThumbprint string
vSphereFolder string
vSphereResourcePool string
)

func main() {
Expand All @@ -61,7 +70,7 @@ func setupCommands(ctx context.Context) *cobra.Command {
rootCmd := &cobra.Command{
Use: "boskosctl",
SilenceUsage: true,
Short: "boskosctl can be used to consume Boskos vsphere resources",
Short: "boskosctl can be used to consume Boskos vSphere resources",
}
// Note: http://boskos.test-pods.svc.cluster.local is the URL of the service usually used in k8s.io clusters.
rootCmd.PersistentFlags().StringVar(&boskosHost, "boskos-host", getOrDefault(os.Getenv("BOSKOS_HOST"), "http://boskos.test-pods.svc.cluster.local"), "Boskos server URL.")
Expand Down Expand Up @@ -92,8 +101,12 @@ func setupCommands(ctx context.Context) *cobra.Command {
RunE: runCmd(ctx),
}
releaseCmd.PersistentFlags().StringVar(&resourceName, "resource-name", "", "Name of the resource.")
releaseCmd.PersistentFlags().StringVar(&resourceFolder, "resource-folder", "", "vSphere folder of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&resourcePool, "resource-pool", "", "vSphere resource pool of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSphereUsername, "vsphere-username", "", "vSphere username of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSpherePassword, "vsphere-password", "", "vSphere password of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSphereServer, "vsphere-server", "", "vSphere server of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSphereTLSThumbprint, "vsphere-tls-thumbprint", "", "vSphere TLS thumbprint of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSphereFolder, "vsphere-folder", "", "vSphere folder of the resource, required for cleanup before release")
releaseCmd.PersistentFlags().StringVar(&vSphereResourcePool, "vsphere-resource-pool", "", "vSphere resource pool of the resource, required for cleanup before release")
rootCmd.AddCommand(releaseCmd)

return rootCmd
Expand Down Expand Up @@ -146,16 +159,29 @@ func runCmd(ctx context.Context) func(cmd *cobra.Command, _ []string) error {
if resourceName == "" {
return fmt.Errorf("--resource-name must be set")
}
if resourceFolder == "" {
return fmt.Errorf("--resource-folder must be set")
if vSphereUsername == "" {
return fmt.Errorf("--vsphere-username must be set")
}
if resourcePool == "" {
return fmt.Errorf("--resource-pool must be set")
if vSpherePassword == "" {
return fmt.Errorf("--vsphere-password must be set")
}
log := log.WithValues("resourceName", resourceName, "resourceFolder", resourceFolder, "resourcePool", resourcePool)
if vSphereServer == "" {
return fmt.Errorf("--vsphere-server must be set")
}
if vSphereTLSThumbprint == "" {
return fmt.Errorf("--vsphere-tls-thumbprint must be set")
}
if vSphereFolder == "" {
return fmt.Errorf("--vsphere-folder must be set")
}
if vSphereResourcePool == "" {
return fmt.Errorf("--vsphere-resource-pool must be set")
}

log := log.WithValues("resourceName", resourceName, "vSphereServer", vSphereServer, "vSphereFolder", vSphereFolder, "vSphereResourcePool", vSphereResourcePool)
ctx := ctrl.LoggerInto(ctx, log)

return release(ctx, client, resourceName, resourceFolder, resourcePool)
return release(ctx, client, resourceName, vSphereUsername, vSpherePassword, vSphereServer, vSphereTLSThumbprint, vSphereFolder, vSphereResourcePool)
}

return nil
Expand Down Expand Up @@ -183,24 +209,114 @@ func acquire(ctx context.Context, client *boskos.Client, resourceType string) er
return errors.Errorf("failed to get user data, resource %q is missing \"resourcePool\" key", res.Name)
}
ipPool, hasIPPool := res.UserData.Load("ipPool")
if !hasIPPool {
fmt.Printf(`
export BOSKOS_RESOURCE_NAME=%s
export BOSKOS_RESOURCE_FOLDER=%s
export BOSKOS_RESOURCE_POOL=%s
`, res.Name, folder, resourcePool)

var sb strings.Builder
sb.WriteString(fmt.Sprintf("export BOSKOS_RESOURCE_NAME=%s\n", res.Name))
sb.WriteString(fmt.Sprintf("export BOSKOS_RESOURCE_FOLDER=%s\n", folder))
sb.WriteString(fmt.Sprintf("export BOSKOS_RESOURCE_POOL=%s\n", resourcePool))

if hasIPPool {
envVars, err := getIPPoolEnvVars(ipPool.(string))
if err != nil {
return errors.Wrapf(err, "failed to calculate IP pool env vars")
}
for k, v := range envVars {
sb.WriteString(fmt.Sprintf("export %s=%s\n", k, v))
}
}

fmt.Printf(`
export BOSKOS_RESOURCE_NAME=%s
export BOSKOS_RESOURCE_FOLDER=%s
export BOSKOS_RESOURCE_POOL=%s
export BOSKOS_RESOURCE_IP_POOL='%s'
`, res.Name, folder, resourcePool, ipPool)
fmt.Println(sb.String())

return nil
}

// inClusterIPPoolSpec defines the desired state of InClusterIPPool.
// Note: This is a copy of the relevant fields from: https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/blob/main/api/v1alpha2/inclusterippool_types.go
// This was copied to avoid a go dependency on this provider.
type inClusterIPPoolSpec struct {
// Addresses is a list of IP addresses that can be assigned. This set of
// addresses can be non-contiguous.
Addresses []string `json:"addresses"`

// Prefix is the network prefix to use.
// +kubebuilder:validation:Maximum=128
Prefix int `json:"prefix"`

// Gateway
// +optional
Gateway string `json:"gateway,omitempty"`
}

// getIPPoolEnvVars calculates env vars based on the ipPool string.
// Note: It's easier to calculate these env vars here in Go compared to if consumers of boskosctl have to do it in bash.
func getIPPoolEnvVars(ipPool string) (map[string]string, error) {
ipPoolSpec := inClusterIPPoolSpec{}
if err := json.Unmarshal([]byte(ipPool), &ipPoolSpec); err != nil {
return nil, fmt.Errorf("failed to unmarshal IP pool configuration")
}

ipSet, err := allIPs(ipPoolSpec.Addresses)
if err != nil {
return nil, errors.Wrapf(err, "failed to calculate IP addresses")
}

envVars := map[string]string{
// We need surrounding '' so the JSON string is preserved correctly.
"BOSKOS_RESOURCE_IP_POOL": fmt.Sprintf("'%s'", ipPool),
"BOSKOS_RESOURCE_IP_POOL_PREFIX": strconv.Itoa(ipPoolSpec.Prefix),
"BOSKOS_RESOURCE_IP_POOL_GATEWAY": ipPoolSpec.Gateway,
}
for i, ip := range ipSet {
envVars[fmt.Sprintf("BOSKOS_RESOURCE_IP_POOL_IP_%d", i)] = ip.String()
}
return envVars, nil
}

// allIPs gets all IPs from addresses.
// Note: Based on https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/blob/f656d1d169aea5063dd2e0563f94ef1cc384371e/internal/poolutil/pool.go#L160
func allIPs(addressesArray []string) ([]netip.Addr, error) {
builder := &netipx.IPSetBuilder{}

for _, addresses := range addressesArray {
if strings.Contains(addresses, "-") {
addrRange, err := netipx.ParseIPRange(addresses)
if err != nil {
return nil, err
}
builder.AddRange(addrRange)
} else if strings.Contains(addresses, "/") {
prefix, err := netip.ParsePrefix(addresses)
if err != nil {
return nil, err
}
builder.AddPrefix(prefix)
} else {
addr, err := netip.ParseAddr(addresses)
if err != nil {
return nil, err
}
builder.Add(addr)
}
}
ipSet, err := builder.IPSet()
if err != nil {
return nil, errors.Wrapf(err, "failed to calculate IP set from addresses")
}

var allIPs []netip.Addr
for _, ipRange := range ipSet.Ranges() {
ip := ipRange.From()
for {
allIPs = append(allIPs, ip)
if ip == ipRange.To() {
break
}
ip = ip.Next()
}
}
return allIPs, nil
}

func heartbeat(ctx context.Context, client *boskos.Client, resourceName string) error {
log := ctrl.LoggerFrom(ctx)
for {
Expand All @@ -216,33 +332,18 @@ func heartbeat(ctx context.Context, client *boskos.Client, resourceName string)
}
}

func release(ctx context.Context, client *boskos.Client, resourceName, resourceFolder, resourcePool string) error {
func release(ctx context.Context, client *boskos.Client, resourceName, vSphereUsername, vSpherePassword, vSphereServer, vSphereTLSThumbprint, vSphereFolder, vSphereResourcePool string) error {
log := ctrl.LoggerFrom(ctx)

var username, password, server, thumbprint string
if username = os.Getenv("VSPHERE_USERNAME"); username == "" {
return fmt.Errorf("env var VSPHERE_USERNAME must be set")
}
if password = os.Getenv("VSPHERE_PASSWORD"); password == "" {
return fmt.Errorf("env var VSPHERE_PASSWORD must be set")
}
if server = os.Getenv("VSPHERE_SERVER"); server == "" {
return fmt.Errorf("env var VSPHERE_SERVER must be set")
}
if thumbprint = os.Getenv("VSPHERE_TLS_THUMBPRINT"); thumbprint == "" {
return fmt.Errorf("env var VSPHERE_TLS_THUMBPRINT must be set")
}
log = log.WithValues("vSphereServer", server)
ctx = ctrl.LoggerInto(ctx, log)

log.Info("Releasing resource")

// Create clients for vSphere.
vSphereClients, err := janitor.NewVSphereClients(ctx, janitor.NewVSphereClientsInput{
Username: username,
Password: password,
Server: server,
Thumbprint: thumbprint,
Username: vSphereUsername,
Password: vSpherePassword,
Server: vSphereServer,
Thumbprint: vSphereTLSThumbprint,
UserAgent: "boskosctl",
})
if err != nil {
Expand All @@ -257,7 +358,7 @@ func release(ctx context.Context, client *boskos.Client, resourceName, resourceF
log.Info("Cleaning up vSphere")
// Note: We intentionally want to skip clusterModule cleanup. If we run this too often we might hit race conditions
// when other tests are creating cluster modules in parallel.
if err := j.CleanupVSphere(ctx, []string{resourceFolder}, []string{resourcePool}, []string{resourceFolder}, true); err != nil {
if err := j.CleanupVSphere(ctx, []string{vSphereFolder}, []string{vSphereResourcePool}, []string{vSphereFolder}, true); err != nil {
log.Info("Cleaning up vSphere failed")

// Try to release resource as dirty.
Expand Down

0 comments on commit 02a536f

Please sign in to comment.