Add fsFreeze field to VolumeSnapshot

Longhorn 2187

Signed-off-by: Eric Weber <[email protected]>

app/cmd/start.go

@@ -24,11 +24,13 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/reflection"
+	"k8s.io/mount-utils"
 
 	commonTypes "github.com/longhorn/go-common-libs/types"
 	helpernvme "github.com/longhorn/go-spdk-helper/pkg/nvme"
 	helpertypes "github.com/longhorn/go-spdk-helper/pkg/types"
 	helperutil "github.com/longhorn/go-spdk-helper/pkg/util"
+	engineutil "github.com/longhorn/longhorn-engine/pkg/util"
 	spdk "github.com/longhorn/longhorn-spdk-engine/pkg/spdk"
 	spdkutil "github.com/longhorn/longhorn-spdk-engine/pkg/util"
 	rpc "github.com/longhorn/types/pkg/generated/imrpc"
@@ -159,6 +161,29 @@ func cleanupStaledNvmeAndDmDevices() error {
 	return nil
 }
 
+func unfreezeFileSystems() error {
+	// We do not need to switch to the host mount namespace to get mount points here. Usually, longhorn-engine runs in a
+	// container that has / bind mounted to /host with at least HostToContainer (rslave) propagation.
+	// - If it does not, we likely can't do a namespace swap anyway, since we don't have access to /host/proc.
+	// - If it does, we just need to know where in the container we can access the mount points to unfreeze the file
+	//   system.
+	mounter := mount.New("")
+	sourcePoints, err := mounter.List()
+	if err != nil {
+		return errors.Wrap(err, "failed to list mount points while starting up")
+	}
+
+	for _, sourcePoint := range sourcePoints {
+		if strings.Contains(sourcePoint.Device, "/dev/longhorn") { // TODO: Find this as a const somewhere.
+			// We do not actually expect any filesystems to be frozen. This is a best effort attempt to unfreeze them
+			// if somehow instance manager crashed at the wrong moment during a snapshot. Log on success, but not on
+			// failure.
+			engineutil.AttemptUnfreezeFileSystem(sourcePoint.Path, nil, false, logrus.New())
+		}
+	}
+	return nil
+}
+
 func start(c *cli.Context) (err error) {
 	listen := c.String("listen")
 	logsDir := c.String("logs-dir")
@@ -183,6 +208,10 @@ func start(c *cli.Context) (err error) {
 		if err := cleanupStaledNvmeAndDmDevices(); err != nil {
 			return err
 		}
+	} else {
+		if err := unfreezeFileSystems(); err != nil {
+			return err
+		}
 	}
 
 	// setup tls config

go.mod

@@ -1,8 +1,6 @@
 module github.com/longhorn/longhorn-instance-manager
 
-go 1.22.0
-
-toolchain go1.22.2
+go 1.22.2
 
 require (
 	github.com/RoaringBitmap/roaring v1.9.3
@@ -47,7 +45,7 @@ require (
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003 // indirect
-	github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca // indirect
+	github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
@@ -72,3 +70,7 @@ require (
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 )
+
+replace github.com/longhorn/types => github.com/ejweber/types v0.0.0-20240429162331-ac822e968757
+
+replace github.com/longhorn/longhorn-engine => github.com/ejweber/longhorn-engine v0.0.0-20240429163402-e0e56573c639

go.sum

@@ -20,8 +20,6 @@ github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK
 github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -32,6 +30,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
 github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/ejweber/longhorn-engine v0.0.0-20240429163402-e0e56573c639 h1:jlc/KaMKCwDg+KTDs+vJo8jqQq1BzDCD8i/DgXu01TQ=
+github.com/ejweber/longhorn-engine v0.0.0-20240429163402-e0e56573c639/go.mod h1:zxLL7yQCuB9dWPFygeWxFeA27bpG2xJ4sPDtcxcL7IU=
+github.com/ejweber/types v0.0.0-20240429162331-ac822e968757 h1:thzRDLmEq2LFQ33CQhQKHup903tsGq8tKO55+Txy2E4=
+github.com/ejweber/types v0.0.0-20240429162331-ac822e968757/go.mod h1:1oEh1cnDDqNSuFh/dH/lvJ3Ssq83SOweTAAPLRY4PMI=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
@@ -67,42 +69,18 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/longhorn/backupstore v0.0.0-20240417071544-3bd377eeefeb h1:GkAG0P7QI32PYgjLBV6VvaEAM/Z9dKyPohccU1js+Uk=
-github.com/longhorn/backupstore v0.0.0-20240417071544-3bd377eeefeb/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k=
-github.com/longhorn/backupstore v0.0.0-20240426093637-ac3867f121c0 h1:sb9HK8uWApqrBbpjnxlNSmIB5f4oi3UqfeRHVniPcqo=
-github.com/longhorn/backupstore v0.0.0-20240426093637-ac3867f121c0/go.mod h1:NlOHCyQiGjn9TS1HUfQlBESttDPzi9x1Vs38geRV9SU=
 github.com/longhorn/backupstore v0.0.0-20240427164602-c8721e8ea31a h1:cwHDzWi0zkgg/SHtvEhWIYLvZHJvCp5twS8qBOUt/Nc=
 github.com/longhorn/backupstore v0.0.0-20240427164602-c8721e8ea31a/go.mod h1:NlOHCyQiGjn9TS1HUfQlBESttDPzi9x1Vs38geRV9SU=
-github.com/longhorn/go-common-libs v0.0.0-20240422073843-1578e42a50bd h1:Jjg8nogQ04/DaofTXsHWCRrBEf4g2ufwfzrsAP4SgrQ=
-github.com/longhorn/go-common-libs v0.0.0-20240422073843-1578e42a50bd/go.mod h1:qppGSy9WsxH1C9T6yhSMvG53ynpasyKOH/n9e1jr8mw=
-github.com/longhorn/go-common-libs v0.0.0-20240426033718-77710d2242df h1:q2a/IL7zoPkYxIMsz46E6uWDamu9z4k+h/4u4pHsdqE=
-github.com/longhorn/go-common-libs v0.0.0-20240426033718-77710d2242df/go.mod h1:qppGSy9WsxH1C9T6yhSMvG53ynpasyKOH/n9e1jr8mw=
 github.com/longhorn/go-common-libs v0.0.0-20240427164621-70d1933bfa90 h1:uBVee6AulPsgSvsvaxdj4xIdgkIvjIqA0j7ZD+Hg/Mc=
 github.com/longhorn/go-common-libs v0.0.0-20240427164621-70d1933bfa90/go.mod h1:qppGSy9WsxH1C9T6yhSMvG53ynpasyKOH/n9e1jr8mw=
-github.com/longhorn/go-spdk-helper v0.0.0-20240422131908-185e91a748ad h1:DhZYpqJp5t5RYyNctTBlz2pgfRxV3tJ/qLshXD4HY3Q=
-github.com/longhorn/go-spdk-helper v0.0.0-20240422131908-185e91a748ad/go.mod h1:CiKgQ6CKWCE4CrAUgQx7inzRT7Xzbo6JuDhdRTpaVkY=
-github.com/longhorn/go-spdk-helper v0.0.0-20240426112700-a797602bf7fe h1:RWugaxgHq79c9o62bSdBBlvoE90eIGb7YkJFu9NZDR8=
-github.com/longhorn/go-spdk-helper v0.0.0-20240426112700-a797602bf7fe/go.mod h1:CiKgQ6CKWCE4CrAUgQx7inzRT7Xzbo6JuDhdRTpaVkY=
 github.com/longhorn/go-spdk-helper v0.0.0-20240427164640-8cb919fb8e53 h1:7JOAKtVr0zOBvBcKJw76IJjbiyJZLI7x5O6GFAifUm0=
 github.com/longhorn/go-spdk-helper v0.0.0-20240427164640-8cb919fb8e53/go.mod h1:CiKgQ6CKWCE4CrAUgQx7inzRT7Xzbo6JuDhdRTpaVkY=
-github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240418025706-519598108463 h1:KxddgUYC9InOhe8MxfbxzOL5v9q7f5DyThxQGGKvqVw=
-github.com/longhorn/longhorn-engine v1.6.0-dev-20231217.0.20240418025706-519598108463/go.mod h1:WNiZl2l51I36/c8dewxkyWd0yBA5Anznzki0knKO88U=
-github.com/longhorn/longhorn-spdk-engine v0.0.0-20240423045523-73fd7e6f9745 h1:pAFMPOPdeU0rqSyMbrrKnwcXUri7dLNFGnEO9ghSqUQ=
-github.com/longhorn/longhorn-spdk-engine v0.0.0-20240423045523-73fd7e6f9745/go.mod h1:21mmiI8dBDXzBiEUbCDoTXimCSbdFDn0MneZzINIqXA=
-github.com/longhorn/longhorn-spdk-engine v0.0.0-20240426084525-210492ed1d43 h1:3raFCK8xb7IHZ8MinyE+SJr/sGVjKzExQxGMhkMuPDQ=
-github.com/longhorn/longhorn-spdk-engine v0.0.0-20240426084525-210492ed1d43/go.mod h1:21mmiI8dBDXzBiEUbCDoTXimCSbdFDn0MneZzINIqXA=
 github.com/longhorn/longhorn-spdk-engine v0.0.0-20240427164543-3425d69b6acb h1:Ze4pc/BNokW++sTFNe7DY22sKUo6S8iPiWYcioA/Ulc=
 github.com/longhorn/longhorn-spdk-engine v0.0.0-20240427164543-3425d69b6acb/go.mod h1:21mmiI8dBDXzBiEUbCDoTXimCSbdFDn0MneZzINIqXA=
 github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003 h1:Jw9uANsGcHTxp6HcC++/vN17LfeuDmozHI2j6DoZf5E=
 github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04=
-github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca h1:dECamLpXIlL7GRmiruGseb5xO6hGAWyu2xYm9D46mb8=
-github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0=
-github.com/longhorn/types v0.0.0-20240417112740-a0d8514936b8 h1:M9TQLN379VNN3JsuLuKHDRIy/11jYeIuxYLFuekjcPw=
-github.com/longhorn/types v0.0.0-20240417112740-a0d8514936b8/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
-github.com/longhorn/types v0.0.0-20240424162824-4995e1e42438 h1:nSuJa62e6F9YsP06Js4jnKr0AsvEMGwBRLm9fXTlmns=
-github.com/longhorn/types v0.0.0-20240424162824-4995e1e42438/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
-github.com/longhorn/types v0.0.0-20240427164854-38dbed8528d3 h1:7YDGJTwro/kCcMmnCWM1UIL4JgervR6MT++71PvcbGY=
-github.com/longhorn/types v0.0.0-20240427164854-38dbed8528d3/go.mod h1:pqT+7B8T+nkyUZYe8tL3CwPDCHjkbe3mHUDY5ntJFyQ=
+github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8 h1:lwtmZEomiv8uchwo9JIyoo+lK8J3cLCm7/qzpn6wmzo=
+github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
@@ -168,8 +146,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
-github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
-github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
 github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
 github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

pkg/client/proxy_snapshot.go

@@ -12,7 +12,7 @@ import (
 )
 
 func (c *ProxyClient) VolumeSnapshot(dataEngine, engineName, volumeName, serviceAddress,
-	volumeSnapshotName string, labels map[string]string) (snapshotName string, err error) {
+	volumeSnapshotName string, labels map[string]string, freezeFS bool) (snapshotName string, err error) {
 	input := map[string]string{
 		"engineName":     engineName,
 		"volumeName":     volumeName,
@@ -55,8 +55,9 @@ func (c *ProxyClient) VolumeSnapshot(dataEngine, engineName, volumeName, service
 			VolumeName:         volumeName,
 		},
 		SnapshotVolume: &enginerpc.VolumeSnapshotRequest{
-			Name:   volumeSnapshotName,
-			Labels: labels,
+			Name:     volumeSnapshotName,
+			Labels:   labels,
+			FreezeFS: freezeFS,
 		},
 	}
 	recv, err := c.service.VolumeSnapshot(getContextWithGRPCTimeout(c.ctx), req)

pkg/proxy/snapshot.go

@@ -41,7 +41,7 @@ func (ops V1DataEngineProxyOps) VolumeSnapshot(ctx context.Context, req *rpc.Eng
 	}
 	defer c.Close()
 
-	recv, err := c.VolumeSnapshot(req.SnapshotVolume.Name, req.SnapshotVolume.Labels)
+	recv, err := c.VolumeSnapshot(req.SnapshotVolume.Name, req.SnapshotVolume.Labels, req.SnapshotVolume.FreezeFS)
 	if err != nil {
 		return nil, err
 	}