test: fix collector for machines not having an IP in status

test/e2e/e2e_suite_test.go

@@ -266,7 +266,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	// vspherelog.MachineLogCollector tries to ssh to the machines to collect logs.
 	// This does not work when using vcsim because there are no real machines running ssh.
 	if testTarget != VCSimTestTarget {
-		clusterProxyOptions = append(clusterProxyOptions, framework.WithMachineLogCollector(vspherelog.MachineLogCollector{}))
+		clusterProxyOptions = append(clusterProxyOptions, framework.WithMachineLogCollector(&vspherelog.MachineLogCollector{
+			Client: vsphereClient,
+			Finder: vsphereFinder,
+		}))
 	}
 	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), clusterProxyOptions...)
 

test/framework/log/collector.go

@@ -27,7 +27,11 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/vmware/govmomi"
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/vim25/mo"
 	"golang.org/x/crypto/ssh"
+	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -39,20 +43,19 @@ const (
 	VSpherePrivateKeyFilePath = "VSPHERE_SSH_PRIVATE_KEY"
 )
 
-type MachineLogCollector struct{}
+type MachineLogCollector struct {
+	Client *govmomi.Client
+	Finder *find.Finder
+}
 
-func (collector MachineLogCollector) CollectMachinePoolLog(_ context.Context, _ client.Client, _ *expv1.MachinePool, _ string) error {
+func (c *MachineLogCollector) CollectMachinePoolLog(_ context.Context, _ client.Client, _ *expv1.MachinePool, _ string) error {
 	return nil
 }
 
-func (collector MachineLogCollector) CollectMachineLog(_ context.Context, _ client.Client, m *clusterv1.Machine, outputPath string) error {
-	var hostIPAddr string
-	for _, address := range m.Status.Addresses {
-		if address.Type != clusterv1.MachineExternalIP {
-			continue
-		}
-		hostIPAddr = address.Address
-		break
+func (c *MachineLogCollector) CollectMachineLog(ctx context.Context, _ client.Client, m *clusterv1.Machine, outputPath string) error {
+	machineIPAddresses, err := c.machineIPAddresses(ctx, m)
+	if err != nil {
+		return err
 	}
 
 	captureLogs := func(hostFileName, command string, args ...string) func() error {
@@ -62,7 +65,15 @@ func (collector MachineLogCollector) CollectMachineLog(_ context.Context, _ clie
 				return err
 			}
 			defer f.Close()
-			return executeRemoteCommand(f, hostIPAddr, command, args...)
+			var lastErr error
+			// Try with all available IPs unless it succeeded.
+			for _, machineIPAddress := range machineIPAddresses {
+				lastErr = executeRemoteCommand(f, machineIPAddress, command, args...)
+				if lastErr == nil {
+					break
+				}
+			}
+			return lastErr
 		}
 	}
 
@@ -78,10 +89,49 @@ func (collector MachineLogCollector) CollectMachineLog(_ context.Context, _ clie
 	})
 }
 
-func (collector MachineLogCollector) CollectInfrastructureLogs(_ context.Context, _ client.Client, _ *clusterv1.Cluster, _ string) error {
+func (c *MachineLogCollector) CollectInfrastructureLogs(_ context.Context, _ client.Client, _ *clusterv1.Cluster, _ string) error {
 	return nil
 }
 
+func (c *MachineLogCollector) machineIPAddresses(ctx context.Context, m *clusterv1.Machine) ([]string, error) {
+	for _, address := range m.Status.Addresses {
+		if address.Type != clusterv1.MachineExternalIP {
+			continue
+		}
+		return []string{address.Address}, nil
+	}
+
+	vmObj, err := c.Finder.VirtualMachine(ctx, m.GetName())
+	if err != nil {
+		return nil, err
+	}
+
+	var vm mo.VirtualMachine
+
+	if err := c.Client.RetrieveOne(ctx, vmObj.Reference(), []string{"guest.net"}, &vm); err != nil {
+		// We cannot get the properties e.g. when the machine already got deleted or is getting deleted.
+		return nil, errors.Errorf("Error retrieving properties for machine %s", klog.KObj(m))
+	}
+
+	addresses := []string{}
+
+	// Return all IPs so we can try each of them until one succeeded.
+	for _, nic := range vm.Guest.Net {
+		if nic.IpConfig == nil {
+			continue
+		}
+		for _, ip := range nic.IpConfig.IpAddress {
+			addresses = append(addresses, ip.IpAddress)
+		}
+	}
+
+	if len(addresses) == 0 {
+		return nil, errors.Errorf("Unable to find IP Addresses for Machine %s", klog.KObj(m))
+	}
+
+	return addresses, nil
+}
+
 func createOutputFile(path string) (*os.File, error) {
 	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
 		return nil, err