fix: add delay before unattach device

We will wait 3 seconds before return success.

Signed-off-by: Serge Logvinov <[email protected]>

cmd/controller/main.go

@@ -34,21 +34,22 @@ import (
 )
 
 var (
+	version string
+	commit  string
+
 	showVersion = flag.Bool("version", false, "Print the version and exit.")
 	csiEndpoint = flag.String("csi-address", "unix:///csi/csi.sock", "CSI Endpoint")
 
 	cloudconfig = flag.String("cloud-config", "", "The path to the CSI driver cloud config.")
 	kubeconfig  = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.")
-
-	version string
 )
 
 func main() {
 	klog.InitFlags(nil)
 	flag.Set("logtostderr", "true") //nolint: errcheck
 	flag.Parse()
 
-	klog.V(2).Infof("Driver version %v, GitVersion %s", csi.DriverVersion, version)
+	klog.V(2).Infof("Driver version %v, GitVersion %s, GitCommit %s", csi.DriverVersion, version, commit)
 	klog.V(2).Info("Driver CSI Spec version: ", csi.DriverSpecVersion)
 
 	if *showVersion {

cmd/node/main.go

@@ -35,22 +35,23 @@ import (
 )
 
 var (
+	version string
+	commit  string
+
 	showVersion = flag.Bool("version", false, "Print the version and exit.")
 	csiEndpoint = flag.String("csi-address", "unix:///csi/csi.sock", "CSI Endpoint")
 	nodeID      = flag.String("node-id", "", "Node name")
 
 	master     = flag.String("master", "", "Master URL to build a client config from. Either this or kubeconfig needs to be set if the provisioner is being run out of cluster.")
 	kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.")
-
-	version string
 )
 
 func main() {
 	klog.InitFlags(nil)
 	flag.Set("logtostderr", "true") //nolint: errcheck
 	flag.Parse()
 
-	klog.V(2).Infof("Driver version %v, GitVersion %s", csi.DriverVersion, version)
+	klog.V(2).Infof("Driver version %v, GitVersion %s, GitCommit %s", csi.DriverVersion, version, commit)
 	klog.V(2).Info("Driver CSI Spec version: ", csi.DriverSpecVersion)
 
 	if *showVersion {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/kubernetes-csi/csi-lib-utils v0.17.0
-	github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.1-0.20240215135711-7b73b5f8a2b1
+	github.com/sergelogvinov/proxmox-cloud-controller-manager v0.4.0
 	github.com/siderolabs/go-blockdevice v0.4.7
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0

go.sum

@@ -83,8 +83,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.1-0.20240215135711-7b73b5f8a2b1 h1:h9rKPmPAzJggV5oliX2akNAeQu5JmZsfobgN/eT/M4I=
-github.com/sergelogvinov/proxmox-cloud-controller-manager v0.3.1-0.20240215135711-7b73b5f8a2b1/go.mod h1:SILDj23jkQGVPtWCWdadd3E6VDJlWnN+P4rgKGrg1j8=
+github.com/sergelogvinov/proxmox-cloud-controller-manager v0.4.0 h1:i90eGKvr1Brj3ys3DS92OFPGNpU3Q6zXwZry7DA0mMk=
+github.com/sergelogvinov/proxmox-cloud-controller-manager v0.4.0/go.mod h1:J2urerxhi+OZY3Pjq5AH+4y6hBGr6BpFUL9gYhUtdBw=
 github.com/siderolabs/go-blockdevice v0.4.7 h1:2bk4WpEEflGxjrNwp57ye24Pr+cYgAiAeNMWiQOuWbQ=
 github.com/siderolabs/go-blockdevice v0.4.7/go.mod h1:4PeOuk71pReJj1JQEXDE7kIIQJPVe8a+HZQa+qjxSEA=
 github.com/siderolabs/go-cmd v0.1.1 h1:nTouZUSxLeiiEe7hFexSVvaTsY/3O8k1s08BxPRrsps=

pkg/csi/driver.go

@@ -44,7 +44,7 @@ const (
 	StorageInodeSizeKey = "inodeSize"
 
 	// MaxVolumesPerNode is the maximum number of volumes that can be attached to a node
-	MaxVolumesPerNode = 16
+	MaxVolumesPerNode = 24
 	// MinVolumeSize is the minimum size of a volume
 	MinVolumeSize = 1 // GB
 	// DefaultVolumeSize is the default size of a volume

pkg/csi/node.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/siderolabs/go-blockdevice/blockdevice/encryption"
@@ -67,7 +69,8 @@ type NodeService struct {
 	nodeID  string
 	kclient kubernetes.Interface
 
-	Mount mount.IMount
+	Mount       mount.IMount
+	volumeLocks sync.Mutex
 }
 
 // NewNodeService returns a new NodeService
@@ -112,12 +115,15 @@ func (n *NodeService) NodeStageVolume(_ context.Context, request *csi.NodeStageV
 		return nil, status.Error(codes.InvalidArgument, "DevicePath must be provided")
 	}
 
-	m := n.Mount
-
 	if blk := volumeCapability.GetBlock(); blk != nil {
 		return &csi.NodeStageVolumeResponse{}, nil
 	}
 
+	n.volumeLocks.Lock()
+	defer n.volumeLocks.Unlock()
+
+	m := n.Mount
+
 	notMnt, err := m.IsLikelyNotMountPointAttach(stagingTarget)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -221,6 +227,9 @@ func (n *NodeService) NodeUnstageVolume(_ context.Context, request *csi.NodeUnst
 		return &csi.NodeUnstageVolumeResponse{}, nil
 	}
 
+	n.volumeLocks.Lock()
+	defer n.volumeLocks.Unlock()
+
 	cmd := exec.New().Command("fstrim", "-v", stagingTargetPath)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		klog.Errorf("NodeUnstageVolume: failed to trim filesystem %s: %v\n", stagingTargetPath, err)
@@ -233,20 +242,29 @@ func (n *NodeService) NodeUnstageVolume(_ context.Context, request *csi.NodeUnst
 		klog.Errorf("NodeUnstageVolume: failed to find mount file system %s: %v", stagingTargetPath, err)
 	}
 
-	if err := n.Mount.UnmountPath(stagingTargetPath); err != nil {
+	if err = n.Mount.UnmountPath(stagingTargetPath); err != nil {
 		klog.Errorf("NodeUnstageVolume: failed to unmount targetPath %s, error: %v", stagingTargetPath, err)
 
 		return nil, status.Errorf(codes.Internal, "Unmount of targetPath %s failed with error %v", stagingTargetPath, err)
 	}
 
+	// wait fsync to complete
+	time.Sleep(3 * time.Second)
+
 	devicePath := strings.TrimSpace(string(sourcePath))
 	if strings.HasPrefix(devicePath, "/dev/mapper/") {
 		l := luks.New(luks.AESXTSPlain64Cipher)
-		if err := l.Close(devicePath); err != nil {
+		if err = l.Close(devicePath); err != nil {
 			klog.Errorf("NodeUnstageVolume: failed to close encrypted device %s, error: %v", devicePath, err)
 
 			return nil, status.Errorf(codes.Internal, "Close encrypted device %s failed with error %v", devicePath, err)
 		}
+	} else {
+		deviceName := filepath.Base(devicePath)
+
+		if err = os.WriteFile(fmt.Sprintf("/sys/block/%s/device/state", deviceName), []byte("offline"), 0644); err != nil { //nolint:gofumpt
+			klog.Warningf("NodeUnstageVolume: failed to offline device %s, error: %v, ignored", devicePath, err)
+		}
 	}
 
 	return &csi.NodeUnstageVolumeResponse{}, nil

tools/deps.sh

@@ -79,6 +79,11 @@ copy_deps /sbin/cryptsetup
 ARCH=$(uname -m)
 mkdir -p ${DEST}/lib/${ARCH}-linux-gnu && cp /lib/${ARCH}-linux-gnu/libgcc_s.so.* ${DEST}/lib/${ARCH}-linux-gnu/
 
+# hack for fsck https://github.com/sergelogvinov/proxmox-csi-plugin/issues/59
+copy_deps /bin/true
+rm -f ${DEST}/sbin/fsck.xfs
+ln -s /bin/true ${DEST}/sbin/fsck.xfs
+
 # This utils are using by
 # go mod k8s.io/cloud-provider-openstack/pkg/util/mount
 copy_deps /bin/udevadm