Skip to content

Commit

Permalink
db: remove 'Experimental' prefix from recent format major versions
Browse files Browse the repository at this point in the history 
These versions are now stable.
  • Loading branch information
@jbowens
jbowens committed Sep 22, 2023
1 parent d038189 commit 8659369
Show file tree
Hide file tree
Showing 12 changed files with 55 additions and 57 deletions.
8 changes: 4 additions & 4 deletions batch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,8 @@ func (b *Batch) refreshMemTableSize() {
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
b.countRangeKeys++
case InternalKeyKindDeleteSized:
if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}
case InternalKeyKindIngestSST:
if b.minimumFormatMajorVersion < FormatFlushableIngest {
Expand Down Expand Up @@ -784,8 +784,8 @@ func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions
// complete key slice, letting the caller encode into the DeferredBatchOp.Key
// slice and then call Finish() on the returned object.
func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}

// Encode the sum of the key length and the value in the value.
Expand Down
2 changes: 1 addition & 1 deletion compaction_iter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func TestCompactionIter(t *testing.T) {
if formatVersion < FormatSetWithDelete {
return "testdata/compaction_iter"
}
if formatVersion < ExperimentalFormatDeleteSizedAndObsolete {
if formatVersion < FormatDeleteSizedAndObsolete {
return "testdata/compaction_iter_set_with_del"
}
return "testdata/compaction_iter_delete_sized"
Expand Down
4 changes: 2 additions & 2 deletions compaction_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1610,12 +1610,12 @@ func TestManualCompaction(t *testing.T) {
},
{
testData: "testdata/manual_compaction_file_boundaries_delsized",
minVersion: ExperimentalFormatDeleteSizedAndObsolete,
minVersion: FormatDeleteSizedAndObsolete,
maxVersion: internalFormatNewest,
},
{
testData: "testdata/manual_compaction_set_with_del_sstable_Pebblev4",
minVersion: ExperimentalFormatDeleteSizedAndObsolete,
minVersion: FormatDeleteSizedAndObsolete,
maxVersion: internalFormatNewest,
},
{
Expand Down
20 changes: 10 additions & 10 deletions format_major_version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,20 +151,20 @@ const (
// compactions for files marked for compaction are complete.
FormatPrePebblev1MarkedCompacted

// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
// FormatDeleteSizedAndObsolete is a format major version that adds support
// for deletion tombstones that encode the size of the value they're
// expected to delete. This format major version is required before the
// associated key kind may be committed through batch applications or
// ingests. It also adds support for keys that are marked obsolete (see
// sstable/format.go for details).
ExperimentalFormatDeleteSizedAndObsolete
FormatDeleteSizedAndObsolete

// ExperimentalFormatVirtualSSTables is a format major version that adds support for
// FormatVirtualSSTables is a format major version that adds support for
// virtual sstables that can reference a sub-range of keys in an underlying
// physical sstable. This information is persisted through new,
// backward-incompatible fields in the Manifest, and therefore requires
// a format major version.
ExperimentalFormatVirtualSSTables
FormatVirtualSSTables

// internalFormatNewest holds the newest format major version, including
// experimental ones excluded from the exported FormatNewest constant until
Expand All @@ -190,7 +190,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
return sstable.TableFormatPebblev2
case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev3
case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev4
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
Expand All @@ -209,7 +209,7 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev1
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
Expand Down Expand Up @@ -342,11 +342,11 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
},
ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
FormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
},
ExperimentalFormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
FormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
},
}

Expand Down
42 changes: 21 additions & 21 deletions format_major_version_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ func TestRatchetFormat(t *testing.T) {
require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion())
require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted))
require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion())
require.NoError(t, d.RatchetFormatMajorVersion(ExperimentalFormatDeleteSizedAndObsolete))
require.Equal(t, ExperimentalFormatDeleteSizedAndObsolete, d.FormatMajorVersion())
require.NoError(t, d.RatchetFormatMajorVersion(ExperimentalFormatVirtualSSTables))
require.Equal(t, ExperimentalFormatVirtualSSTables, d.FormatMajorVersion())
require.NoError(t, d.RatchetFormatMajorVersion(FormatDeleteSizedAndObsolete))
require.Equal(t, FormatDeleteSizedAndObsolete, d.FormatMajorVersion())
require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables))
require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion())

require.NoError(t, d.Close())

Expand Down Expand Up @@ -214,23 +214,23 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
// fixture is intentionally verbose.

m := map[FormatMajorVersion][2]sstable.TableFormat{
FormatDefault: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatMostCompatible: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
formatVersionedManifestMarker: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatVersioned: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatSetWithDelete: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatBlockPropertyCollector: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatSplitUserKeysMarked: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatRangeKeys: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2},
FormatMinTableFormatPebblev1: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatPrePebblev1Marked: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatSSTableValueBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
ExperimentalFormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
ExperimentalFormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
FormatDefault: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatMostCompatible: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
formatVersionedManifestMarker: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatVersioned: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatSetWithDelete: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
FormatBlockPropertyCollector: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatSplitUserKeysMarked: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
FormatRangeKeys: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2},
FormatMinTableFormatPebblev1: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatPrePebblev1Marked: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
FormatSSTableValueBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
FormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
FormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
}

// Valid versions.
Expand Down
2 changes: 1 addition & 1 deletion ingest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1243,7 +1243,7 @@ func (d *DB) ingest(
if len(shared) > 0 && d.opts.Experimental.RemoteStorage == nil {
panic("cannot ingest shared sstables with nil SharedStorage")
}
if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < ExperimentalFormatVirtualSSTables {
if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < FormatVirtualSSTables {
return IngestOperationStats{}, errors.New("pebble: format major version too old for excise, shared or external sstable ingestion")
}
// Allocate file numbers for all of the files being ingested and mark them as
Expand Down
12 changes: 6 additions & 6 deletions ingest_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,7 @@ func TestExcise(t *testing.T) {
EventListener: &EventListener{FlushEnd: func(info FlushInfo) {
flushed = true
}},
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
}
// Disable automatic compactions because otherwise we'll race with
// delete-only compactions triggered by ingesting range tombstones.
Expand Down Expand Up @@ -788,7 +788,7 @@ func TestIngestShared(t *testing.T) {
L0CompactionThreshold: 100,
L0StopWritesThreshold: 100,
DebugCheck: DebugCheckLevels,
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
}
// lel.
lel := MakeLoggingEventListener(DefaultLogger)
Expand Down Expand Up @@ -1072,7 +1072,7 @@ func TestSimpleIngestShared(t *testing.T) {
mem := vfs.NewMem()
var d *DB
var provider2 objstorage.Provider
opts2 := Options{FS: vfs.NewMem(), FormatMajorVersion: ExperimentalFormatVirtualSSTables}
opts2 := Options{FS: vfs.NewMem(), FormatMajorVersion: FormatVirtualSSTables}
opts2.EnsureDefaults()

// Create an objProvider where we will fake-create some sstables that can
Expand Down Expand Up @@ -1110,7 +1110,7 @@ func TestSimpleIngestShared(t *testing.T) {
mem = vfs.NewMem()
require.NoError(t, mem.MkdirAll("ext", 0755))
opts := &Options{
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
FS: mem,
L0CompactionThreshold: 100,
L0StopWritesThreshold: 100,
Expand Down Expand Up @@ -1259,7 +1259,7 @@ func TestConcurrentExcise(t *testing.T) {
L0CompactionThreshold: 100,
L0StopWritesThreshold: 100,
DebugCheck: DebugCheckLevels,
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
}
// lel.
lel := MakeLoggingEventListener(DefaultLogger)
Expand Down Expand Up @@ -1609,7 +1609,7 @@ func TestIngestExternal(t *testing.T) {
EventListener: &EventListener{FlushEnd: func(info FlushInfo) {
flushed = true
}},
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
}
opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
"external-locator": remoteStorage,
Expand Down
4 changes: 1 addition & 3 deletions metamorphic/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,9 +564,7 @@ func (g *generator) dbRatchetFormatMajorVersion() {
// version may be behind the database's format major version, in which case
// RatchetFormatMajorVersion should deterministically error.

// TODO(jackson): When the latest format major versions ares stabilized,
// return this to just using `FormatNewest`.
n := int(newestFormatMajorVersionTODO - minimumFormatMajorVersion)
n := int(newestFormatMajorVersionToTest - minimumFormatMajorVersion)
vers := pebble.FormatMajorVersion(g.rng.Intn(n+1)) + minimumFormatMajorVersion
g.add(&dbRatchetFormatMajorVersionOp{vers: vers})
}
Expand Down
2 changes: 1 addition & 1 deletion metamorphic/ops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ type deleteOp struct {
func (o *deleteOp) run(t *test, h historyRecorder) {
w := t.getWriter(o.writerID)
var err error
if t.testOpts.deleteSized && t.isFMV(pebble.ExperimentalFormatDeleteSizedAndObsolete) {
if t.testOpts.deleteSized && t.isFMV(pebble.FormatDeleteSizedAndObsolete) {
// Call DeleteSized with a deterministic size derived from the index.
// The size does not need to be accurate for correctness.
err = w.DeleteSized(o.key, hashSize(t.idx), t.writeOpts)
Expand Down
10 changes: 5 additions & 5 deletions metamorphic/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ const (
// that are less than defaultFormatMajorVersion but are at least
// minimumFormatMajorVersion.
defaultFormatMajorVersion = pebble.FormatPrePebblev1Marked
// newestFormatMajorVersionTODO is the most recent format major version the
// metamorphic tests should use. This may be greater than
// newestFormatMajorVersionToTest is the most recent format major version
// the metamorphic tests should use. This may be greater than
// pebble.FormatNewest when some format major versions are marked as
// experimental.
newestFormatMajorVersionTODO = pebble.ExperimentalFormatVirtualSSTables
newestFormatMajorVersionToTest = pebble.FormatNewest
)

func parseOptions(
Expand Down Expand Up @@ -386,7 +386,7 @@ func standardOptions() []*TestOptions {
26: fmt.Sprintf(`
[Options]
format_major_version=%s
`, newestFormatMajorVersionTODO),
`, newestFormatMajorVersionToTest),
27: `
[TestOptions]
shared_storage_enabled=true
Expand Down Expand Up @@ -440,7 +440,7 @@ func randomOptions(
opts.FlushDelayRangeKey = time.Millisecond * time.Duration(5*rng.Intn(245)) // 5-250ms
opts.FlushSplitBytes = 1 << rng.Intn(20) // 1B - 1MB
opts.FormatMajorVersion = minimumFormatMajorVersion
n := int(newestFormatMajorVersionTODO - opts.FormatMajorVersion)
n := int(newestFormatMajorVersionToTest - opts.FormatMajorVersion)
opts.FormatMajorVersion += pebble.FormatMajorVersion(rng.Intn(n + 1))
opts.Experimental.L0CompactionConcurrency = 1 + rng.Intn(4) // 1-4
opts.Experimental.LevelMultiplier = 5 << rng.Intn(7) // 5 - 320
Expand Down
2 changes: 1 addition & 1 deletion version_set.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ func (vs *versionSet) logAndApply(
defer vs.mu.Lock()

var err error
if vs.opts.FormatMajorVersion < ExperimentalFormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
if vs.opts.FormatMajorVersion < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version")
}
newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE(
Expand Down
4 changes: 2 additions & 2 deletions version_set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestLatestRefCounting(t *testing.T) {
FS: mem,
MaxManifestFileSize: 1,
DisableAutomaticCompactions: true,
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
}
d, err := Open("", opts)
require.NoError(t, err)
Expand Down Expand Up @@ -220,7 +220,7 @@ func TestVirtualSSTableManifestReplay(t *testing.T) {
require.NoError(t, mem.MkdirAll("ext", 0755))

opts := &Options{
FormatMajorVersion: ExperimentalFormatVirtualSSTables,
FormatMajorVersion: FormatVirtualSSTables,
FS: mem,
MaxManifestFileSize: 1,
DisableAutomaticCompactions: true,
Expand Down

0 comments on commit 8659369

Please sign in to comment.