db: add test for virtual sstable checkpointing

Virtual sstable checkpointing will remove the backing files which
won't be required by the checkpoint. Need to make sure that the
virtual sstables which are present in the checkpoint manifest are
still readable, and that the backing files not required are deleted.

checkpoint.go

@@ -135,9 +135,6 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
 // space overhead for a checkpoint if hard links are disabled. Also beware that
 // even if hard links are used, the space overhead for the checkpoint will
 // increase over time as the DB performs compactions.
-//
-// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
-// sstables is running e2e.
 func (d *DB) Checkpoint(
 	destDir string, opts ...CheckpointOption,
 ) (

checkpoint_test.go

@@ -5,6 +5,7 @@
 package pebble
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math/rand"
@@ -23,17 +24,21 @@ func TestCheckpoint(t *testing.T) {
 	dbs := make(map[string]*DB)
 	defer func() {
 		for _, db := range dbs {
-			require.NoError(t, db.Close())
+			if db.closed.Load() == nil {
+				require.NoError(t, db.Close())
+			}
 		}
 	}()
 
 	mem := vfs.NewMem()
 	var memLog base.InMemLogger
 	opts := &Options{
-		FS:                    vfs.WithLogging(mem, memLog.Infof),
-		FormatMajorVersion:    internalFormatNewest,
-		L0CompactionThreshold: 10,
+		FS:                          vfs.WithLogging(mem, memLog.Infof),
+		FormatMajorVersion:          internalFormatNewest,
+		L0CompactionThreshold:       10,
+		DisableAutomaticCompactions: true,
 	}
+	opts.private.disableTableStats = true
 
 	datadriven.RunTest(t, "testdata/checkpoint", func(t *testing.T, td *datadriven.TestData) string {
 		switch td.Cmd {
@@ -78,6 +83,33 @@ func TestCheckpoint(t *testing.T) {
 			}
 			return memLog.String()
 
+		case "ingest-and-excise":
+			d := dbs[td.CmdArgs[0].String()]
+
+			// Hacky but the command doesn't expect a db string. Get rid of it.
+			td.CmdArgs = td.CmdArgs[1:]
+			if err := runIngestAndExciseCmd(td, d, mem); err != nil {
+				return err.Error()
+			}
+			return ""
+
+		case "build":
+			d := dbs[td.CmdArgs[0].String()]
+
+			// Hacky but the command doesn't expect a db string. Get rid of it.
+			td.CmdArgs = td.CmdArgs[1:]
+			if err := runBuildCmd(td, d, mem); err != nil {
+				return err.Error()
+			}
+			return ""
+
+		case "lsm":
+			d := dbs[td.CmdArgs[0].String()]
+
+			// Hacky but the command doesn't expect a db string. Get rid of it.
+			td.CmdArgs = td.CmdArgs[1:]
+			return runLSMCmd(td, d)
+
 		case "compact":
 			if len(td.CmdArgs) != 1 {
 				return "compact <db>"
@@ -90,6 +122,39 @@ func TestCheckpoint(t *testing.T) {
 			d.TestOnlyWaitForCleaning()
 			return memLog.String()
 
+		case "print-backing":
+			// prints contents of the file backing map in the version. Used to
+			// test whether the checkpoint removed the filebackings correctly.
+			if len(td.CmdArgs) != 1 {
+				return "print-backing <db>"
+			}
+			d := dbs[td.CmdArgs[0].String()]
+			d.mu.Lock()
+			d.mu.versions.logLock()
+			var fileNums []base.DiskFileNum
+			for _, b := range d.mu.versions.backingState.fileBackingMap {
+				fileNums = append(fileNums, b.DiskFileNum)
+			}
+			d.mu.versions.logUnlock()
+			d.mu.Unlock()
+
+			sort.Slice(fileNums, func(i, j int) bool {
+				return uint64(fileNums[i].FileNum()) < uint64(fileNums[j].FileNum())
+			})
+			var buf bytes.Buffer
+			for _, f := range fileNums {
+				buf.WriteString(fmt.Sprintf("%s\n", f.String()))
+			}
+			return buf.String()
+
+		case "close":
+			if len(td.CmdArgs) != 1 {
+				return "close <db>"
+			}
+			d := dbs[td.CmdArgs[0].String()]
+			require.NoError(t, d.Close())
+			return ""
+
 		case "flush":
 			if len(td.CmdArgs) != 1 {
 				return "flush <db>"