From 86593692e09f904f4ea739e065074f44f40ec9ba Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Fri, 22 Sep 2023 09:45:55 -0400 Subject: [PATCH] db: remove 'Experimental' prefix from recent format major versions These versions are now stable. --- batch.go | 8 +++---- compaction_iter_test.go | 2 +- compaction_test.go | 4 ++-- format_major_version.go | 20 ++++++++--------- format_major_version_test.go | 42 ++++++++++++++++++------------------ ingest.go | 2 +- ingest_test.go | 12 +++++------ metamorphic/generator.go | 4 +--- metamorphic/ops.go | 2 +- metamorphic/options.go | 10 ++++----- version_set.go | 2 +- version_set_test.go | 4 ++-- 12 files changed, 55 insertions(+), 57 deletions(-) diff --git a/batch.go b/batch.go index fa69a2382f..2899cd1a82 100644 --- a/batch.go +++ b/batch.go @@ -452,8 +452,8 @@ func (b *Batch) refreshMemTableSize() { case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: b.countRangeKeys++ case InternalKeyKindDeleteSized: - if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete { - b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete + if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete { + b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete } case InternalKeyKindIngestSST: if b.minimumFormatMajorVersion < FormatFlushableIngest { @@ -784,8 +784,8 @@ func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions // complete key slice, letting the caller encode into the DeferredBatchOp.Key // slice and then call Finish() on the returned object. func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp { - if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete { - b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete + if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete { + b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete } // Encode the sum of the key length and the value in the value. diff --git a/compaction_iter_test.go b/compaction_iter_test.go index 115b816cc4..02b1d3541a 100644 --- a/compaction_iter_test.go +++ b/compaction_iter_test.go @@ -93,7 +93,7 @@ func TestCompactionIter(t *testing.T) { if formatVersion < FormatSetWithDelete { return "testdata/compaction_iter" } - if formatVersion < ExperimentalFormatDeleteSizedAndObsolete { + if formatVersion < FormatDeleteSizedAndObsolete { return "testdata/compaction_iter_set_with_del" } return "testdata/compaction_iter_delete_sized" diff --git a/compaction_test.go b/compaction_test.go index 4081ad91c2..150ebfdc5f 100644 --- a/compaction_test.go +++ b/compaction_test.go @@ -1610,12 +1610,12 @@ func TestManualCompaction(t *testing.T) { }, { testData: "testdata/manual_compaction_file_boundaries_delsized", - minVersion: ExperimentalFormatDeleteSizedAndObsolete, + minVersion: FormatDeleteSizedAndObsolete, maxVersion: internalFormatNewest, }, { testData: "testdata/manual_compaction_set_with_del_sstable_Pebblev4", - minVersion: ExperimentalFormatDeleteSizedAndObsolete, + minVersion: FormatDeleteSizedAndObsolete, maxVersion: internalFormatNewest, }, { diff --git a/format_major_version.go b/format_major_version.go index 7e78fa3d56..c418578f37 100644 --- a/format_major_version.go +++ b/format_major_version.go @@ -151,20 +151,20 @@ const ( // compactions for files marked for compaction are complete. FormatPrePebblev1MarkedCompacted - // ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support + // FormatDeleteSizedAndObsolete is a format major version that adds support // for deletion tombstones that encode the size of the value they're // expected to delete. This format major version is required before the // associated key kind may be committed through batch applications or // ingests. It also adds support for keys that are marked obsolete (see // sstable/format.go for details). - ExperimentalFormatDeleteSizedAndObsolete + FormatDeleteSizedAndObsolete - // ExperimentalFormatVirtualSSTables is a format major version that adds support for + // FormatVirtualSSTables is a format major version that adds support for // virtual sstables that can reference a sub-range of keys in an underlying // physical sstable. This information is persisted through new, // backward-incompatible fields in the Manifest, and therefore requires // a format major version. - ExperimentalFormatVirtualSSTables + FormatVirtualSSTables // internalFormatNewest holds the newest format major version, including // experimental ones excluded from the exported FormatNewest constant until @@ -190,7 +190,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { return sstable.TableFormatPebblev2 case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev3 - case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables: + case FormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev4 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) @@ -209,7 +209,7 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted, - ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables: + FormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev1 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) @@ -342,11 +342,11 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ } return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted) }, - ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error { - return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete) + FormatDeleteSizedAndObsolete: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete) }, - ExperimentalFormatVirtualSSTables: func(d *DB) error { - return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables) + FormatVirtualSSTables: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatVirtualSSTables) }, } diff --git a/format_major_version_test.go b/format_major_version_test.go index 640eb5d295..7345934a68 100644 --- a/format_major_version_test.go +++ b/format_major_version_test.go @@ -62,10 +62,10 @@ func TestRatchetFormat(t *testing.T) { require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion()) require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted)) require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(ExperimentalFormatDeleteSizedAndObsolete)) - require.Equal(t, ExperimentalFormatDeleteSizedAndObsolete, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(ExperimentalFormatVirtualSSTables)) - require.Equal(t, ExperimentalFormatVirtualSSTables, d.FormatMajorVersion()) + require.NoError(t, d.RatchetFormatMajorVersion(FormatDeleteSizedAndObsolete)) + require.Equal(t, FormatDeleteSizedAndObsolete, d.FormatMajorVersion()) + require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables)) + require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion()) require.NoError(t, d.Close()) @@ -214,23 +214,23 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) { // fixture is intentionally verbose. m := map[FormatMajorVersion][2]sstable.TableFormat{ - FormatDefault: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatMostCompatible: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - formatVersionedManifestMarker: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatVersioned: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatSetWithDelete: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatBlockPropertyCollector: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatSplitUserKeysMarked: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatRangeKeys: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2}, - FormatMinTableFormatPebblev1: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - FormatPrePebblev1Marked: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - FormatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - FormatSSTableValueBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - ExperimentalFormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, - ExperimentalFormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, + FormatDefault: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, + FormatMostCompatible: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, + formatVersionedManifestMarker: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, + FormatVersioned: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, + FormatSetWithDelete: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, + FormatBlockPropertyCollector: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, + FormatSplitUserKeysMarked: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, + FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, + FormatRangeKeys: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2}, + FormatMinTableFormatPebblev1: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, + FormatPrePebblev1Marked: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, + FormatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, + FormatSSTableValueBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, + FormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, } // Valid versions. diff --git a/ingest.go b/ingest.go index bfdb0998d0..e650ac7702 100644 --- a/ingest.go +++ b/ingest.go @@ -1243,7 +1243,7 @@ func (d *DB) ingest( if len(shared) > 0 && d.opts.Experimental.RemoteStorage == nil { panic("cannot ingest shared sstables with nil SharedStorage") } - if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < ExperimentalFormatVirtualSSTables { + if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < FormatVirtualSSTables { return IngestOperationStats{}, errors.New("pebble: format major version too old for excise, shared or external sstable ingestion") } // Allocate file numbers for all of the files being ingested and mark them as diff --git a/ingest_test.go b/ingest_test.go index ce97df9d19..38c063e590 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -601,7 +601,7 @@ func TestExcise(t *testing.T) { EventListener: &EventListener{FlushEnd: func(info FlushInfo) { flushed = true }}, - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, } // Disable automatic compactions because otherwise we'll race with // delete-only compactions triggered by ingesting range tombstones. @@ -788,7 +788,7 @@ func TestIngestShared(t *testing.T) { L0CompactionThreshold: 100, L0StopWritesThreshold: 100, DebugCheck: DebugCheckLevels, - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, } // lel. lel := MakeLoggingEventListener(DefaultLogger) @@ -1072,7 +1072,7 @@ func TestSimpleIngestShared(t *testing.T) { mem := vfs.NewMem() var d *DB var provider2 objstorage.Provider - opts2 := Options{FS: vfs.NewMem(), FormatMajorVersion: ExperimentalFormatVirtualSSTables} + opts2 := Options{FS: vfs.NewMem(), FormatMajorVersion: FormatVirtualSSTables} opts2.EnsureDefaults() // Create an objProvider where we will fake-create some sstables that can @@ -1110,7 +1110,7 @@ func TestSimpleIngestShared(t *testing.T) { mem = vfs.NewMem() require.NoError(t, mem.MkdirAll("ext", 0755)) opts := &Options{ - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, FS: mem, L0CompactionThreshold: 100, L0StopWritesThreshold: 100, @@ -1259,7 +1259,7 @@ func TestConcurrentExcise(t *testing.T) { L0CompactionThreshold: 100, L0StopWritesThreshold: 100, DebugCheck: DebugCheckLevels, - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, } // lel. lel := MakeLoggingEventListener(DefaultLogger) @@ -1609,7 +1609,7 @@ func TestIngestExternal(t *testing.T) { EventListener: &EventListener{FlushEnd: func(info FlushInfo) { flushed = true }}, - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, } opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{ "external-locator": remoteStorage, diff --git a/metamorphic/generator.go b/metamorphic/generator.go index 2bb8cc5ea7..4767fe7f42 100644 --- a/metamorphic/generator.go +++ b/metamorphic/generator.go @@ -564,9 +564,7 @@ func (g *generator) dbRatchetFormatMajorVersion() { // version may be behind the database's format major version, in which case // RatchetFormatMajorVersion should deterministically error. - // TODO(jackson): When the latest format major versions ares stabilized, - // return this to just using `FormatNewest`. - n := int(newestFormatMajorVersionTODO - minimumFormatMajorVersion) + n := int(newestFormatMajorVersionToTest - minimumFormatMajorVersion) vers := pebble.FormatMajorVersion(g.rng.Intn(n+1)) + minimumFormatMajorVersion g.add(&dbRatchetFormatMajorVersionOp{vers: vers}) } diff --git a/metamorphic/ops.go b/metamorphic/ops.go index 45c73c9ed9..eb79c8527b 100644 --- a/metamorphic/ops.go +++ b/metamorphic/ops.go @@ -201,7 +201,7 @@ type deleteOp struct { func (o *deleteOp) run(t *test, h historyRecorder) { w := t.getWriter(o.writerID) var err error - if t.testOpts.deleteSized && t.isFMV(pebble.ExperimentalFormatDeleteSizedAndObsolete) { + if t.testOpts.deleteSized && t.isFMV(pebble.FormatDeleteSizedAndObsolete) { // Call DeleteSized with a deterministic size derived from the index. // The size does not need to be accurate for correctness. err = w.DeleteSized(o.key, hashSize(t.idx), t.writeOpts) diff --git a/metamorphic/options.go b/metamorphic/options.go index cd73289629..79236dec58 100644 --- a/metamorphic/options.go +++ b/metamorphic/options.go @@ -36,11 +36,11 @@ const ( // that are less than defaultFormatMajorVersion but are at least // minimumFormatMajorVersion. defaultFormatMajorVersion = pebble.FormatPrePebblev1Marked - // newestFormatMajorVersionTODO is the most recent format major version the - // metamorphic tests should use. This may be greater than + // newestFormatMajorVersionToTest is the most recent format major version + // the metamorphic tests should use. This may be greater than // pebble.FormatNewest when some format major versions are marked as // experimental. - newestFormatMajorVersionTODO = pebble.ExperimentalFormatVirtualSSTables + newestFormatMajorVersionToTest = pebble.FormatNewest ) func parseOptions( @@ -386,7 +386,7 @@ func standardOptions() []*TestOptions { 26: fmt.Sprintf(` [Options] format_major_version=%s -`, newestFormatMajorVersionTODO), +`, newestFormatMajorVersionToTest), 27: ` [TestOptions] shared_storage_enabled=true @@ -440,7 +440,7 @@ func randomOptions( opts.FlushDelayRangeKey = time.Millisecond * time.Duration(5*rng.Intn(245)) // 5-250ms opts.FlushSplitBytes = 1 << rng.Intn(20) // 1B - 1MB opts.FormatMajorVersion = minimumFormatMajorVersion - n := int(newestFormatMajorVersionTODO - opts.FormatMajorVersion) + n := int(newestFormatMajorVersionToTest - opts.FormatMajorVersion) opts.FormatMajorVersion += pebble.FormatMajorVersion(rng.Intn(n + 1)) opts.Experimental.L0CompactionConcurrency = 1 + rng.Intn(4) // 1-4 opts.Experimental.LevelMultiplier = 5 << rng.Intn(7) // 5 - 320 diff --git a/version_set.go b/version_set.go index 0ed0cd08d2..2a466b1f92 100644 --- a/version_set.go +++ b/version_set.go @@ -484,7 +484,7 @@ func (vs *versionSet) logAndApply( defer vs.mu.Lock() var err error - if vs.opts.FormatMajorVersion < ExperimentalFormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 { + if vs.opts.FormatMajorVersion < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 { return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version") } newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE( diff --git a/version_set_test.go b/version_set_test.go index beedc1b7fe..b3b17bfbf4 100644 --- a/version_set_test.go +++ b/version_set_test.go @@ -41,7 +41,7 @@ func TestLatestRefCounting(t *testing.T) { FS: mem, MaxManifestFileSize: 1, DisableAutomaticCompactions: true, - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, } d, err := Open("", opts) require.NoError(t, err) @@ -220,7 +220,7 @@ func TestVirtualSSTableManifestReplay(t *testing.T) { require.NoError(t, mem.MkdirAll("ext", 0755)) opts := &Options{ - FormatMajorVersion: ExperimentalFormatVirtualSSTables, + FormatMajorVersion: FormatVirtualSSTables, FS: mem, MaxManifestFileSize: 1, DisableAutomaticCompactions: true,