Merge branch 'main' into update-du-dd-progress-when-terminal-event-is…

…-missing

.golangci.yaml

@@ -227,6 +227,12 @@ linters-settings:
     require-explanation: true
     # Enable to require nolint directives to mention the specific linter being suppressed. Default is false.
     require-specific: true
+  perfsprint:
+    strconcat: false
+    sprintf1: false
+    errorf: false
+    int-conversion: true
+    fiximports: true
   revive:
     rules:
       - name: unexported-return
@@ -310,6 +316,7 @@ linters:
     - nilerr
     - noctx
     - nolintlint
+    - perfsprint
     - revive
     - staticcheck
     - stylecheck
@@ -331,7 +338,6 @@ issues:
       text: "DefaultVolumesToRestic" # No need to report deprecate for DefaultVolumesToRestic.
     - path: ".*_test.go$"
       linters:
-        - dupword
         - errcheck
         - goconst
         - gosec
@@ -342,7 +348,6 @@ issues:
         - unused
     - path: test/
       linters:
-        - dupword
         - errcheck
         - goconst
         - gosec
@@ -351,6 +356,14 @@ issues:
         - stylecheck
         - unparam
         - unused
+    - path: ".*data_upload_controller_test.go$"
+      linters:
+        - dupword
+      text: "type"
+    - path: ".*config_test.go$"
+      linters:
+        - dupword
+      text: "bucket"
 
   # The list of ids of default excludes to include or disable. By default it's empty.
   include:

changelogs/unreleased/8550-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8044, allow users to ignore delay binding the restorePVC of data mover when it is in WaitForFirstConsumer mode

changelogs/unreleased/8590-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8579, set event burst to block event broadcaster from filtering events

changelogs/unreleased/8591-reasonerjt

@@ -0,0 +1 @@
+Skip patching the PV in finalization for failed operation

changelogs/unreleased/8594-Lyndon-Li

@@ -0,0 +1 @@
+Data mover restore for Windows

changelogs/unreleased/8602-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8067, add tmp folder (/tmp for linux, C:\Windows\Temp for Windows) as an alternative of udmrepo's config file location

changelogs/unreleased/8603-ywk253100

@@ -0,0 +1 @@
+Check the PVB status via podvolume Backupper rather than calling API server to avoid API server issue

changelogs/unreleased/8606-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8418, add Windows toleration to data mover pods

config/crd/v2alpha1/bases/velero.io_datadownloads.yaml

@@ -92,6 +92,13 @@ spec:
                   DataMover specifies the data mover to be used by the backup.
                   If DataMover is "" or "velero", the built-in data mover will be used.
                 type: string
+              nodeOS:
+                description: NodeOS is OS of the node where the DataDownload is processed.
+                enum:
+                - auto
+                - linux
+                - windows
+                type: string
               operationTimeout:
                 description: |-
                   OperationTimeout specifies the time used to wait internal operations,

config/crd/v2alpha1/bases/velero.io_datauploads.yaml

@@ -144,7 +144,8 @@ spec:
             description: DataUploadStatus is the current status of a DataUpload.
             properties:
               acceptedByNode:
-                description: Node is name of the node where the DataUpload is prepared.
+                description: AcceptedByNode is name of the node where the DataUpload
+                  is prepared.
                 type: string
               acceptedTimestamp:
                 description: |-
@@ -175,6 +176,13 @@ spec:
               node:
                 description: Node is name of the node where the DataUpload is processed.
                 type: string
+              nodeOS:
+                description: NodeOS is OS of the node where the DataUpload is processed.
+                enum:
+                - auto
+                - linux
+                - windows
+                type: string
               path:
                 description: Path is the full path of the snapshot volume being backed
                   up.

internal/delete/delete_item_action_handler_test.go

@@ -230,7 +230,7 @@ func (h *harness) addResource(t *testing.T, resource *test.APIResource) {
 }
 
 // recordResourcesAction is a delete item action that can be configured to run
-// for specific resources/namespaces and simply record the items that is is
+// for specific resources/namespaces and simply record the items that is
 // executed for.
 type recordResourcesAction struct {
 	selector velero.ResourceSelector

pkg/apis/velero/v2alpha1/data_download_types.go

@@ -54,6 +54,10 @@ type DataDownloadSpec struct {
 	// OperationTimeout specifies the time used to wait internal operations,
 	// before returning error as timeout.
 	OperationTimeout metav1.Duration `json:"operationTimeout"`
+
+	// NodeOS is OS of the node where the DataDownload is processed.
+	// +optional
+	NodeOS NodeOS `json:"nodeOS,omitempty"`
 }
 
 // TargetVolumeSpec is the specification for a target PVC.

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -96,6 +96,16 @@ const (
 	DataUploadPhaseFailed     DataUploadPhase = "Failed"
 )
 
+// NodeOS represents OS of a node.
+// +kubebuilder:validation:Enum=auto;linux;windows
+type NodeOS string
+
+const (
+	NodeOSLinux   NodeOS = "linux"
+	NodeOSWindows NodeOS = "windows"
+	NodeOSAuto    NodeOS = "auto"
+)
+
 // DataUploadStatus is the current status of a DataUpload.
 type DataUploadStatus struct {
 	// Phase is the current state of the DataUpload.
@@ -144,7 +154,12 @@ type DataUploadStatus struct {
 	// Node is name of the node where the DataUpload is processed.
 	// +optional
 	Node string `json:"node,omitempty"`
-	// Node is name of the node where the DataUpload is prepared.
+
+	// NodeOS is OS of the node where the DataUpload is processed.
+	// +optional
+	NodeOS NodeOS `json:"nodeOS,omitempty"`
+
+	// AcceptedByNode is name of the node where the DataUpload is prepared.
 	// +optional
 	AcceptedByNode string `json:"acceptedByNode,omitempty"`
 
@@ -221,4 +236,8 @@ type DataUploadResult struct {
 	// +optional
 	// +nullable
 	DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"`
+
+	// NodeOS is OS of the node where the DataUpload is processed.
+	// +optional
+	NodeOS NodeOS `json:"nodeOS,omitempty"`
 }

pkg/backup/actions/csi/pvc_action.go

@@ -19,6 +19,7 @@ package csi
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
@@ -484,9 +485,7 @@ func newDataUpload(
 	if backup.Spec.UploaderConfig != nil &&
 		backup.Spec.UploaderConfig.ParallelFilesUpload > 0 {
 		dataUpload.Spec.DataMoverConfig = make(map[string]string)
-		dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = fmt.Sprintf(
-			"%d", backup.Spec.UploaderConfig.ParallelFilesUpload,
-		)
+		dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = strconv.Itoa(backup.Spec.UploaderConfig.ParallelFilesUpload)
 	}
 
 	return dataUpload

pkg/backup/backup.go

@@ -35,10 +35,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/selection"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -315,7 +313,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(
 
 	var podVolumeBackupper podvolume.Backupper
 	if kb.podVolumeBackupperFactory != nil {
-		podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(ctx, backupRequest.Backup, kb.uploaderType)
+		podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(ctx, log, backupRequest.Backup, kb.uploaderType)
 		if err != nil {
 			log.WithError(errors.WithStack(err)).Debugf("Error from NewBackupper")
 			return errors.WithStack(err)
@@ -745,6 +743,7 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(ctx context.Context, ite
 	log := itemBlock.Log
 	defer itemBlock.itemBackupper.hookTracker.AsyncItemBlocks.Done()
 
+	// the post hooks will not execute until all PVBs of the item block pods are processed
 	if err := kb.waitUntilPVBsProcessed(ctx, log, itemBlock, hookPods); err != nil {
 		log.WithError(err).Error("failed to wait PVBs processed for the ItemBlock")
 		return
@@ -758,36 +757,19 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(ctx context.Context, ite
 	}
 }
 
+// wait all PVBs of the item block pods to be processed
 func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log logrus.FieldLogger, itemBlock BackupItemBlock, pods []itemblock.ItemBlockItem) error {
-	requirement, err := labels.NewRequirement(velerov1api.BackupUIDLabel, selection.Equals, []string{string(itemBlock.itemBackupper.backupRequest.UID)})
-	if err != nil {
-		return errors.Wrapf(err, "failed to create label requirement")
-	}
-	options := &kbclient.ListOptions{
-		LabelSelector: labels.NewSelector().Add(*requirement),
-	}
-	pvbList := &velerov1api.PodVolumeBackupList{}
-	if err := kb.kbClient.List(context.Background(), pvbList, options); err != nil {
-		return errors.Wrap(err, "failed to list PVBs")
-	}
-
-	podMap := map[string]struct{}{}
-	for _, pod := range pods {
-		podMap[string(pod.Item.GetUID())] = struct{}{}
-	}
-
 	pvbMap := map[*velerov1api.PodVolumeBackup]bool{}
-	for i, pvb := range pvbList.Items {
-		if _, exist := podMap[string(pvb.Spec.Pod.UID)]; !exist {
-			continue
+	for _, pod := range pods {
+		namespace, name := pod.Item.GetNamespace(), pod.Item.GetName()
+		pvbs, err := itemBlock.itemBackupper.podVolumeBackupper.ListPodVolumeBackupsByPod(namespace, name)
+		if err != nil {
+			return errors.Wrapf(err, "failed to list PodVolumeBackups for pod %s/%s", namespace, name)
 		}
-
-		processed := false
-		if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted ||
-			pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed {
-			processed = true
+		for _, pvb := range pvbs {
+			pvbMap[pvb] = pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted ||
+				pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed
 		}
-		pvbMap[&pvbList.Items[i]] = processed
 	}
 
 	checkFunc := func(context.Context) (done bool, err error) {
@@ -796,8 +778,8 @@ func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log l
 			if processed {
 				continue
 			}
-			updatedPVB := &velerov1api.PodVolumeBackup{}
-			if err := kb.kbClient.Get(ctx, kbclient.ObjectKeyFromObject(pvb), updatedPVB); err != nil {
+			updatedPVB, err := itemBlock.itemBackupper.podVolumeBackupper.GetPodVolumeBackup(pvb.Namespace, pvb.Name)
+			if err != nil {
 				allProcessed = false
 				log.Infof("failed to get PVB: %v", err)
 				continue

pkg/backup/backup_test.go

@@ -3945,7 +3945,7 @@ func TestBackupWithHooks(t *testing.T) {
 
 type fakePodVolumeBackupperFactory struct{}
 
-func (f *fakePodVolumeBackupperFactory) NewBackupper(context.Context, *velerov1.Backup, string) (podvolume.Backupper, error) {
+func (f *fakePodVolumeBackupperFactory) NewBackupper(context.Context, logrus.FieldLogger, *velerov1.Backup, string) (podvolume.Backupper, error) {
 	return &fakePodVolumeBackupper{}, nil
 }
 
@@ -3978,6 +3978,24 @@ func (b *fakePodVolumeBackupper) WaitAllPodVolumesProcessed(log logrus.FieldLogg
 	return b.pvbs
 }
 
+func (b *fakePodVolumeBackupper) GetPodVolumeBackup(namespace, name string) (*velerov1.PodVolumeBackup, error) {
+	for _, pvb := range b.pvbs {
+		if pvb.Namespace == namespace && pvb.Name == name {
+			return pvb, nil
+		}
+	}
+	return nil, nil
+}
+func (b *fakePodVolumeBackupper) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1.PodVolumeBackup, error) {
+	var pvbs []*velerov1.PodVolumeBackup
+	for _, pvb := range b.pvbs {
+		if pvb.Spec.Pod.Namespace == podNamespace && pvb.Spec.Pod.Name == podName {
+			pvbs = append(pvbs, pvb)
+		}
+	}
+	return pvbs, nil
+}
+
 // TestBackupWithPodVolume runs backups of pods that are annotated for PodVolume backup,
 // and ensures that the pod volume backupper is called, that the returned PodVolumeBackups
 // are added to the Request object, and that when PVCs are backed up with PodVolume, the

pkg/builder/data_download_builder.go

@@ -148,6 +148,12 @@ func (d *DataDownloadBuilder) Node(node string) *DataDownloadBuilder {
 	return d
 }
 
+// NodeOS sets the DataDownload's Node OS.
+func (d *DataDownloadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataDownloadBuilder {
+	d.object.Spec.NodeOS = nodeOS
+	return d
+}
+
 // AcceptedByNode sets the DataDownload's AcceptedByNode.
 func (d *DataDownloadBuilder) AcceptedByNode(node string) *DataDownloadBuilder {
 	d.object.Status.AcceptedByNode = node

pkg/builder/data_upload_builder.go

@@ -151,6 +151,12 @@ func (d *DataUploadBuilder) Node(node string) *DataUploadBuilder {
 	return d
 }
 
+// NodeOS sets the DataUpload's Node OS.
+func (d *DataUploadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataUploadBuilder {
+	d.object.Status.NodeOS = nodeOS
+	return d
+}
+
 // AcceptedByNode sets the DataUpload's AcceptedByNode.
 func (d *DataUploadBuilder) AcceptedByNode(node string) *DataUploadBuilder {
 	d.object.Status.AcceptedByNode = node

pkg/cmd/cli/backup/create_test.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -222,7 +223,7 @@ func TestCreateCommand(t *testing.T) {
 		flags.Parse([]string{"--default-volumes-to-fs-backup", defaultVolumesToFsBackup})
 		flags.Parse([]string{"--resource-policies-configmap", resPoliciesConfigmap})
 		flags.Parse([]string{"--data-mover", dataMover})
-		flags.Parse([]string{"--parallel-files-upload", fmt.Sprintf("%d", parallelFilesUpload)})
+		flags.Parse([]string{"--parallel-files-upload", strconv.Itoa(parallelFilesUpload)})
 		//flags.Parse([]string{"--wait"})
 
 		client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch)

pkg/cmd/cli/datamover/restore.go

@@ -160,7 +160,24 @@ func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, conf
 		return nil, errors.Wrap(err, "error to create client")
 	}
 
-	cache, err := ctlcache.New(clientConfig, cacheOption)
+	var cache ctlcache.Cache
+	retry := 10
+	for {
+		cache, err = ctlcache.New(clientConfig, cacheOption)
+		if err == nil {
+			break
+		}
+
+		retry--
+		if retry == 0 {
+			break
+		}
+
+		logger.WithError(err).Warn("Failed to create client cache, need retry")
+
+		time.Sleep(time.Second)
+	}
+
 	if err != nil {
 		cancelFunc()
 		return nil, errors.Wrap(err, "error to create client cache")

pkg/cmd/cli/nodeagent/server.go

@@ -353,7 +353,13 @@ func (s *nodeAgentServer) run() {
 		s.logger.WithError(err).Fatal("Unable to create the data upload controller")
 	}
 
-	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	var restorePVCConfig nodeagent.RestorePVC
+	if s.dataPathConfigs != nil && s.dataPathConfigs.RestorePVCConfig != nil {
+		restorePVCConfig = *s.dataPathConfigs.RestorePVCConfig
+		s.logger.Infof("Using customized restorePVC config %v", restorePVCConfig)
+	}
+
+	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, restorePVCConfig, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data download controller")
 	}