diff --git a/objstorage/objstorageprovider/provider.go b/objstorage/objstorageprovider/provider.go index a16c63bb3c..7ac35631a8 100644 --- a/objstorage/objstorageprovider/provider.go +++ b/objstorage/objstorageprovider/provider.go @@ -17,7 +17,7 @@ import ( "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" - "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" "github.com/cockroachdb/pebble/objstorage/remote" "github.com/cockroachdb/pebble/vfs" ) @@ -38,7 +38,7 @@ type provider struct { shared struct { // catalogBatch accumulates shared object creations and deletions until // Sync is called. - catalogBatch sharedobjcat.Batch + catalogBatch remoteobjcat.Batch storageObjects map[remote.Locator]remote.Storage } @@ -447,7 +447,7 @@ func (p *provider) addMetadata(meta objstorage.ObjectMetadata) { defer p.mu.Unlock() p.mu.knownObjects[meta.DiskFileNum] = meta if meta.IsRemote() { - p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ + p.mu.shared.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{ FileNum: meta.DiskFileNum, FileType: meta.FileType, CreatorID: meta.Remote.CreatorID, diff --git a/objstorage/objstorageprovider/sharedobjcat/catalog.go b/objstorage/objstorageprovider/remoteobjcat/catalog.go similarity index 90% rename from objstorage/objstorageprovider/sharedobjcat/catalog.go rename to objstorage/objstorageprovider/remoteobjcat/catalog.go index ed946d33a2..f9d22572ad 100644 --- a/objstorage/objstorageprovider/sharedobjcat/catalog.go +++ b/objstorage/objstorageprovider/remoteobjcat/catalog.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package sharedobjcat +package remoteobjcat import ( "fmt" @@ -19,7 +19,7 @@ import ( "github.com/cockroachdb/pebble/vfs/atomicfs" ) -// Catalog is used to manage the on-disk shared object catalog. +// Catalog is used to manage the on-disk remote object catalog. // // The catalog file is a log of records, where each record is an encoded // versionEdit. @@ -30,7 +30,7 @@ type Catalog struct { sync.Mutex creatorID objstorage.CreatorID - objects map[base.DiskFileNum]SharedObjectMetadata + objects map[base.DiskFileNum]RemoteObjectMetadata marker *atomicfs.Marker @@ -45,8 +45,8 @@ type Catalog struct { } } -// SharedObjectMetadata encapsulates the data stored in the catalog file for each object. -type SharedObjectMetadata struct { +// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object. +type RemoteObjectMetadata struct { // FileNum is the identifier for the object within the context of a single DB // instance. FileNum base.DiskFileNum @@ -67,19 +67,19 @@ type SharedObjectMetadata struct { } const ( - catalogFilenameBase = "SHARED-CATALOG" - catalogMarkerName = "shared-catalog" + catalogFilenameBase = "REMOTE-OBJ-CATALOG" + catalogMarkerName = "remote-obj-catalog" // We create a new file when the size exceeds 1MB (and some other conditions // hold; see record.RotationHelper). rotateFileSize = 1024 * 1024 // 1MB ) -// CatalogContents contains the shared objects in the catalog. +// CatalogContents contains the remote objects in the catalog. type CatalogContents struct { // CreatorID, if it is set. CreatorID objstorage.CreatorID - Objects []SharedObjectMetadata + Objects []RemoteObjectMetadata } // Open creates a Catalog and loads any existing catalog file, returning the @@ -89,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) { fs: fs, dirname: dirname, } - c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata) + c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata) var err error c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName) @@ -108,7 +108,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) { } res := CatalogContents{ CreatorID: c.mu.creatorID, - Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)), + Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)), } for _, meta := range c.mu.objects { res.Objects = append(res.Objects, meta) @@ -138,7 +138,7 @@ func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error { ve := versionEdit{CreatorID: id} if err := c.writeToCatalogFileLocked(&ve); err != nil { - return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err) } c.mu.creatorID = id return nil @@ -172,7 +172,7 @@ type Batch struct { // // The given FileNum must be new - it must not match that of any object that was // ever in the catalog. -func (b *Batch) AddObject(meta SharedObjectMetadata) { +func (b *Batch) AddObject(meta RemoteObjectMetadata) { b.ve.NewObjects = append(b.ve.NewObjects, meta) } @@ -196,7 +196,7 @@ func (b *Batch) IsEmpty() bool { func (b *Batch) Copy() Batch { var res Batch if len(b.ve.NewObjects) > 0 { - res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects)) + res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects)) copy(res.ve.NewObjects, b.ve.NewObjects) } if len(b.ve.DeletedObjects) > 0 { @@ -245,7 +245,7 @@ func (c *Catalog) ApplyBatch(b Batch) error { } if err := c.writeToCatalogFileLocked(&b.ve); err != nil { - return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err) } b.Reset() @@ -257,7 +257,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error { f, err := c.fs.Open(catalogPath) if err != nil { return errors.Wrapf( - err, "pebble: could not open shared object catalog file %q for DB %q", + err, "pebble: could not open remote object catalog file %q for DB %q", errors.Safe(filename), c.dirname, ) } @@ -269,13 +269,13 @@ func (c *Catalog) loadFromCatalogFile(filename string) error { break } if err != nil { - return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q", errors.Safe(filename)) } var ve versionEdit err = ve.Decode(r) if err != nil { - return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q", errors.Safe(filename)) } // Apply the version edit to the current state. @@ -340,7 +340,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) { // Create a versionEdit that gets us from an empty catalog to the current state. var ve versionEdit ve.CreatorID = c.mu.creatorID - ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects)) + ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects)) for _, meta := range c.mu.objects { ve.NewObjects = append(ve.NewObjects, meta) } diff --git a/objstorage/objstorageprovider/sharedobjcat/catalog_test.go b/objstorage/objstorageprovider/remoteobjcat/catalog_test.go similarity index 90% rename from objstorage/objstorageprovider/sharedobjcat/catalog_test.go rename to objstorage/objstorageprovider/remoteobjcat/catalog_test.go index 7b4b1f6a88..37a0c3c7dc 100644 --- a/objstorage/objstorageprovider/sharedobjcat/catalog_test.go +++ b/objstorage/objstorageprovider/remoteobjcat/catalog_test.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package sharedobjcat_test +package remoteobjcat_test import ( "fmt" @@ -15,7 +15,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/objstorage" - "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" "github.com/cockroachdb/pebble/vfs" ) @@ -23,7 +23,7 @@ func TestCatalog(t *testing.T) { mem := vfs.NewMem() var memLog base.InMemLogger - var cat *sharedobjcat.Catalog + var cat *remoteobjcat.Catalog datadriven.RunTest(t, "testdata/catalog", func(t *testing.T, td *datadriven.TestData) string { toUInt64 := func(args ...string) []uint64 { t.Helper() @@ -38,13 +38,13 @@ func TestCatalog(t *testing.T) { return res } - parseAdd := func(args []string) sharedobjcat.SharedObjectMetadata { + parseAdd := func(args []string) remoteobjcat.RemoteObjectMetadata { t.Helper() if len(args) != 3 { td.Fatalf(t, "add ") } vals := toUInt64(args...) - return sharedobjcat.SharedObjectMetadata{ + return remoteobjcat.RemoteObjectMetadata{ FileNum: base.FileNum(vals[0]).DiskFileNum(), // When we support other file types, we should let the test determine this. FileType: base.FileTypeTable, @@ -72,8 +72,8 @@ func TestCatalog(t *testing.T) { if err != nil { td.Fatalf(t, "%v", err) } - var contents sharedobjcat.CatalogContents - cat, contents, err = sharedobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname) + var contents remoteobjcat.CatalogContents + cat, contents, err = remoteobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname) if err != nil { return err.Error() } @@ -98,7 +98,7 @@ func TestCatalog(t *testing.T) { return memLog.String() case "batch": - var b sharedobjcat.Batch + var b remoteobjcat.Batch for _, cmd := range strings.Split(td.Input, "\n") { tokens := strings.Split(cmd, " ") if len(tokens) == 0 { @@ -136,10 +136,10 @@ func TestCatalog(t *testing.T) { td.Fatalf(t, "random-batches n= size=") } } - var b sharedobjcat.Batch + var b remoteobjcat.Batch for batchIdx := 0; batchIdx < n; batchIdx++ { for i := 0; i < size; i++ { - b.AddObject(sharedobjcat.SharedObjectMetadata{ + b.AddObject(remoteobjcat.RemoteObjectMetadata{ FileNum: base.FileNum(rand.Uint64()).DiskFileNum(), // When we support other file types, we should let the test determine this. FileType: base.FileTypeTable, diff --git a/objstorage/objstorageprovider/remoteobjcat/testdata/catalog b/objstorage/objstorageprovider/remoteobjcat/testdata/catalog new file mode 100644 index 0000000000..d3e8d37718 --- /dev/null +++ b/objstorage/objstorageprovider/remoteobjcat/testdata/catalog @@ -0,0 +1,267 @@ +open test +---- + +list test +---- + +batch +add 1 10 100 +---- +create: test/REMOTE-OBJ-CATALOG-000001 +sync: test/REMOTE-OBJ-CATALOG-000001 +create: test/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 +close: test/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 +sync: test +sync: test/REMOTE-OBJ-CATALOG-000001 + +list test +---- +REMOTE-OBJ-CATALOG-000001 +marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + +batch +add 2 20 200 +add 3 30 300 +---- +sync: test/REMOTE-OBJ-CATALOG-000001 + +batch +delete 1 +---- +sync: test/REMOTE-OBJ-CATALOG-000001 + +list test +---- +REMOTE-OBJ-CATALOG-000001 +marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + +set-creator-id 5 +---- +sync: test/REMOTE-OBJ-CATALOG-000001 + +set-creator-id 5 +---- + +set-creator-id 6 +---- +error setting creator ID: attempt to change CreatorID from 5 to 6 + +# Bad batches. +batch +add 3 1 1 +---- +error applying batch: adding existing object 000003 + +batch +delete 1000 +---- +error applying batch: deleting non-existent object 001000 + +close +---- +close: test/REMOTE-OBJ-CATALOG-000001 + +open test +---- +creator-id: 5 +000002: 20/000200 +000003: 30/000300 + +set-creator-id 6 +---- +error setting creator ID: attempt to change CreatorID from 5 to 6 + +batch +add 4 40 40 +delete 3 +add 8 80 80 +---- +create: test/REMOTE-OBJ-CATALOG-000002 +sync: test/REMOTE-OBJ-CATALOG-000002 +create: test/marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 +close: test/marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 +remove: test/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000001 +sync: test/REMOTE-OBJ-CATALOG-000002 + +list test +---- +REMOTE-OBJ-CATALOG-000002 +marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 + +close +---- +close: test/REMOTE-OBJ-CATALOG-000002 + +open test +---- +creator-id: 5 +000002: 20/000200 +000004: 40/000040 +000008: 80/000080 + +close +---- + +open other-path +---- + +batch +add 5 50 500 +---- +create: other-path/REMOTE-OBJ-CATALOG-000001 +sync: other-path/REMOTE-OBJ-CATALOG-000001 +create: other-path/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 +close: other-path/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 +sync: other-path +sync: other-path/REMOTE-OBJ-CATALOG-000001 + +# Adding and deleting objects in the same batch is allowed. + +batch +add 9 50 501 +delete 9 +---- +sync: other-path/REMOTE-OBJ-CATALOG-000001 + +list other-path +---- +REMOTE-OBJ-CATALOG-000001 +marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + +list test +---- +REMOTE-OBJ-CATALOG-000002 +marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 + +close +---- +close: other-path/REMOTE-OBJ-CATALOG-000001 + +open test +---- +creator-id: 5 +000002: 20/000200 +000004: 40/000040 +000008: 80/000080 + +# Test rotation. +list test +---- +REMOTE-OBJ-CATALOG-000002 +marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 + +random-batches n=20 size=2000 +---- +create: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +create: test/marker.remote-obj-catalog.000003.REMOTE-OBJ-CATALOG-000003 +close: test/marker.remote-obj-catalog.000003.REMOTE-OBJ-CATALOG-000003 +remove: test/marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000002 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000003 +close: test/REMOTE-OBJ-CATALOG-000003 +create: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +create: test/marker.remote-obj-catalog.000004.REMOTE-OBJ-CATALOG-000004 +close: test/marker.remote-obj-catalog.000004.REMOTE-OBJ-CATALOG-000004 +remove: test/marker.remote-obj-catalog.000003.REMOTE-OBJ-CATALOG-000003 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000003 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 + +list test +---- +REMOTE-OBJ-CATALOG-000004 +marker.remote-obj-catalog.000004.REMOTE-OBJ-CATALOG-000004 + +random-batches n=20 size=2000 +---- +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000004 +close: test/REMOTE-OBJ-CATALOG-000004 +create: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +create: test/marker.remote-obj-catalog.000005.REMOTE-OBJ-CATALOG-000005 +close: test/marker.remote-obj-catalog.000005.REMOTE-OBJ-CATALOG-000005 +remove: test/marker.remote-obj-catalog.000004.REMOTE-OBJ-CATALOG-000004 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000004 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000005 + +list test +---- +REMOTE-OBJ-CATALOG-000005 +marker.remote-obj-catalog.000005.REMOTE-OBJ-CATALOG-000005 + +# Even with huge batches, we don't rotate on each batch. +random-batches n=10 size=50000 +---- +sync: test/REMOTE-OBJ-CATALOG-000005 +close: test/REMOTE-OBJ-CATALOG-000005 +create: test/REMOTE-OBJ-CATALOG-000006 +sync: test/REMOTE-OBJ-CATALOG-000006 +create: test/marker.remote-obj-catalog.000006.REMOTE-OBJ-CATALOG-000006 +close: test/marker.remote-obj-catalog.000006.REMOTE-OBJ-CATALOG-000006 +remove: test/marker.remote-obj-catalog.000005.REMOTE-OBJ-CATALOG-000005 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000005 +sync: test/REMOTE-OBJ-CATALOG-000006 +sync: test/REMOTE-OBJ-CATALOG-000006 +sync: test/REMOTE-OBJ-CATALOG-000006 +close: test/REMOTE-OBJ-CATALOG-000006 +create: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 +create: test/marker.remote-obj-catalog.000007.REMOTE-OBJ-CATALOG-000007 +close: test/marker.remote-obj-catalog.000007.REMOTE-OBJ-CATALOG-000007 +remove: test/marker.remote-obj-catalog.000006.REMOTE-OBJ-CATALOG-000006 +sync: test +remove: test/REMOTE-OBJ-CATALOG-000006 +sync: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 +sync: test/REMOTE-OBJ-CATALOG-000007 + +list test +---- +REMOTE-OBJ-CATALOG-000007 +marker.remote-obj-catalog.000007.REMOTE-OBJ-CATALOG-000007 diff --git a/objstorage/objstorageprovider/sharedobjcat/version_edit.go b/objstorage/objstorageprovider/remoteobjcat/version_edit.go similarity index 95% rename from objstorage/objstorageprovider/sharedobjcat/version_edit.go rename to objstorage/objstorageprovider/remoteobjcat/version_edit.go index 7ec17622b2..a0423dca5b 100644 --- a/objstorage/objstorageprovider/sharedobjcat/version_edit.go +++ b/objstorage/objstorageprovider/remoteobjcat/version_edit.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package sharedobjcat +package remoteobjcat import ( "bufio" @@ -15,12 +15,12 @@ import ( "github.com/cockroachdb/pebble/objstorage/remote" ) -// versionEdit is a modification to the shared object state which can be encoded +// versionEdit is a modification to the remote object state which can be encoded // into a record. // // TODO(radu): consider adding creation and deletion time for debugging purposes. type versionEdit struct { - NewObjects []SharedObjectMetadata + NewObjects []RemoteObjectMetadata DeletedObjects []base.DiskFileNum CreatorID objstorage.CreatorID } @@ -164,7 +164,7 @@ func (v *versionEdit) Decode(r io.Reader) error { } if err == nil { - v.NewObjects = append(v.NewObjects, SharedObjectMetadata{ + v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{ FileNum: base.FileNum(fileNum).DiskFileNum(), FileType: fileType, CreatorID: objstorage.CreatorID(creatorID), @@ -224,4 +224,4 @@ func decodeString(br io.ByteReader) (string, error) { return string(buf), nil } -var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog") +var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog") diff --git a/objstorage/objstorageprovider/sharedobjcat/version_edit_test.go b/objstorage/objstorageprovider/remoteobjcat/version_edit_test.go similarity index 95% rename from objstorage/objstorageprovider/sharedobjcat/version_edit_test.go rename to objstorage/objstorageprovider/remoteobjcat/version_edit_test.go index cac378ec18..14e50253dd 100644 --- a/objstorage/objstorageprovider/sharedobjcat/version_edit_test.go +++ b/objstorage/objstorageprovider/remoteobjcat/version_edit_test.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package sharedobjcat +package remoteobjcat import ( "bytes" @@ -22,7 +22,7 @@ func TestVersionEditRoundTrip(t *testing.T) { CreatorID: 12345, }, { - NewObjects: []SharedObjectMetadata{ + NewObjects: []RemoteObjectMetadata{ { FileNum: base.FileNum(1).DiskFileNum(), FileType: base.FileTypeTable, @@ -39,7 +39,7 @@ func TestVersionEditRoundTrip(t *testing.T) { }, { CreatorID: 12345, - NewObjects: []SharedObjectMetadata{ + NewObjects: []RemoteObjectMetadata{ { FileNum: base.FileNum(1).DiskFileNum(), FileType: base.FileTypeTable, diff --git a/objstorage/objstorageprovider/shared.go b/objstorage/objstorageprovider/shared.go index 820b42fca0..107d83dd45 100644 --- a/objstorage/objstorageprovider/shared.go +++ b/objstorage/objstorageprovider/shared.go @@ -14,15 +14,15 @@ import ( "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" - "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" "github.com/cockroachdb/pebble/objstorage/remote" ) // sharedSubsystem contains the provider fields related to shared storage. // All fields remain unset if shared storage is not configured. type sharedSubsystem struct { - catalog *sharedobjcat.Catalog + catalog *remoteobjcat.Catalog cache *sharedcache.Cache // checkRefsOnOpen controls whether we check the ref marker file when opening @@ -49,7 +49,7 @@ func (p *provider) sharedInit() error { if p.st.Shared.StorageFactory == nil { return nil } - catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName) + catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName) if err != nil { return errors.Wrapf(err, "pebble: could not open shared object catalog") } @@ -163,7 +163,7 @@ func (p *provider) sharedCheckInitialized() error { } func (p *provider) sharedSync() error { - batch := func() sharedobjcat.Batch { + batch := func() remoteobjcat.Batch { p.mu.Lock() defer p.mu.Unlock() res := p.mu.shared.catalogBatch.Copy() diff --git a/objstorage/objstorageprovider/shared_backing.go b/objstorage/objstorageprovider/shared_backing.go index 93e47c9ffd..b988383e60 100644 --- a/objstorage/objstorageprovider/shared_backing.go +++ b/objstorage/objstorageprovider/shared_backing.go @@ -12,7 +12,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/objstorage" - "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" "github.com/cockroachdb/pebble/objstorage/remote" ) @@ -275,7 +275,7 @@ func (p *provider) AttachRemoteObjects( p.mu.Lock() defer p.mu.Unlock() for _, d := range decoded { - p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ + p.mu.shared.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{ FileNum: d.meta.DiskFileNum, FileType: d.meta.FileType, CreatorID: d.meta.Remote.CreatorID, diff --git a/objstorage/objstorageprovider/sharedobjcat/testdata/catalog b/objstorage/objstorageprovider/sharedobjcat/testdata/catalog deleted file mode 100644 index f5a02a306b..0000000000 --- a/objstorage/objstorageprovider/sharedobjcat/testdata/catalog +++ /dev/null @@ -1,267 +0,0 @@ -open test ----- - -list test ----- - -batch -add 1 10 100 ----- -create: test/SHARED-CATALOG-000001 -sync: test/SHARED-CATALOG-000001 -create: test/marker.shared-catalog.000001.SHARED-CATALOG-000001 -close: test/marker.shared-catalog.000001.SHARED-CATALOG-000001 -sync: test -sync: test/SHARED-CATALOG-000001 - -list test ----- -SHARED-CATALOG-000001 -marker.shared-catalog.000001.SHARED-CATALOG-000001 - -batch -add 2 20 200 -add 3 30 300 ----- -sync: test/SHARED-CATALOG-000001 - -batch -delete 1 ----- -sync: test/SHARED-CATALOG-000001 - -list test ----- -SHARED-CATALOG-000001 -marker.shared-catalog.000001.SHARED-CATALOG-000001 - -set-creator-id 5 ----- -sync: test/SHARED-CATALOG-000001 - -set-creator-id 5 ----- - -set-creator-id 6 ----- -error setting creator ID: attempt to change CreatorID from 5 to 6 - -# Bad batches. -batch -add 3 1 1 ----- -error applying batch: adding existing object 000003 - -batch -delete 1000 ----- -error applying batch: deleting non-existent object 001000 - -close ----- -close: test/SHARED-CATALOG-000001 - -open test ----- -creator-id: 5 -000002: 20/000200 -000003: 30/000300 - -set-creator-id 6 ----- -error setting creator ID: attempt to change CreatorID from 5 to 6 - -batch -add 4 40 40 -delete 3 -add 8 80 80 ----- -create: test/SHARED-CATALOG-000002 -sync: test/SHARED-CATALOG-000002 -create: test/marker.shared-catalog.000002.SHARED-CATALOG-000002 -close: test/marker.shared-catalog.000002.SHARED-CATALOG-000002 -remove: test/marker.shared-catalog.000001.SHARED-CATALOG-000001 -sync: test -remove: test/SHARED-CATALOG-000001 -sync: test/SHARED-CATALOG-000002 - -list test ----- -SHARED-CATALOG-000002 -marker.shared-catalog.000002.SHARED-CATALOG-000002 - -close ----- -close: test/SHARED-CATALOG-000002 - -open test ----- -creator-id: 5 -000002: 20/000200 -000004: 40/000040 -000008: 80/000080 - -close ----- - -open other-path ----- - -batch -add 5 50 500 ----- -create: other-path/SHARED-CATALOG-000001 -sync: other-path/SHARED-CATALOG-000001 -create: other-path/marker.shared-catalog.000001.SHARED-CATALOG-000001 -close: other-path/marker.shared-catalog.000001.SHARED-CATALOG-000001 -sync: other-path -sync: other-path/SHARED-CATALOG-000001 - -# Adding and deleting objects in the same batch is allowed. - -batch -add 9 50 501 -delete 9 ----- -sync: other-path/SHARED-CATALOG-000001 - -list other-path ----- -SHARED-CATALOG-000001 -marker.shared-catalog.000001.SHARED-CATALOG-000001 - -list test ----- -SHARED-CATALOG-000002 -marker.shared-catalog.000002.SHARED-CATALOG-000002 - -close ----- -close: other-path/SHARED-CATALOG-000001 - -open test ----- -creator-id: 5 -000002: 20/000200 -000004: 40/000040 -000008: 80/000080 - -# Test rotation. -list test ----- -SHARED-CATALOG-000002 -marker.shared-catalog.000002.SHARED-CATALOG-000002 - -random-batches n=20 size=2000 ----- -create: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -create: test/marker.shared-catalog.000003.SHARED-CATALOG-000003 -close: test/marker.shared-catalog.000003.SHARED-CATALOG-000003 -remove: test/marker.shared-catalog.000002.SHARED-CATALOG-000002 -sync: test -remove: test/SHARED-CATALOG-000002 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000003 -close: test/SHARED-CATALOG-000003 -create: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -create: test/marker.shared-catalog.000004.SHARED-CATALOG-000004 -close: test/marker.shared-catalog.000004.SHARED-CATALOG-000004 -remove: test/marker.shared-catalog.000003.SHARED-CATALOG-000003 -sync: test -remove: test/SHARED-CATALOG-000003 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 - -list test ----- -SHARED-CATALOG-000004 -marker.shared-catalog.000004.SHARED-CATALOG-000004 - -random-batches n=20 size=2000 ----- -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000004 -close: test/SHARED-CATALOG-000004 -create: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -create: test/marker.shared-catalog.000005.SHARED-CATALOG-000005 -close: test/marker.shared-catalog.000005.SHARED-CATALOG-000005 -remove: test/marker.shared-catalog.000004.SHARED-CATALOG-000004 -sync: test -remove: test/SHARED-CATALOG-000004 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000005 - -list test ----- -SHARED-CATALOG-000005 -marker.shared-catalog.000005.SHARED-CATALOG-000005 - -# Even with huge batches, we don't rotate on each batch. -random-batches n=10 size=50000 ----- -sync: test/SHARED-CATALOG-000005 -close: test/SHARED-CATALOG-000005 -create: test/SHARED-CATALOG-000006 -sync: test/SHARED-CATALOG-000006 -create: test/marker.shared-catalog.000006.SHARED-CATALOG-000006 -close: test/marker.shared-catalog.000006.SHARED-CATALOG-000006 -remove: test/marker.shared-catalog.000005.SHARED-CATALOG-000005 -sync: test -remove: test/SHARED-CATALOG-000005 -sync: test/SHARED-CATALOG-000006 -sync: test/SHARED-CATALOG-000006 -sync: test/SHARED-CATALOG-000006 -close: test/SHARED-CATALOG-000006 -create: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 -create: test/marker.shared-catalog.000007.SHARED-CATALOG-000007 -close: test/marker.shared-catalog.000007.SHARED-CATALOG-000007 -remove: test/marker.shared-catalog.000006.SHARED-CATALOG-000006 -sync: test -remove: test/SHARED-CATALOG-000006 -sync: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 -sync: test/SHARED-CATALOG-000007 - -list test ----- -SHARED-CATALOG-000007 -marker.shared-catalog.000007.SHARED-CATALOG-000007 diff --git a/objstorage/objstorageprovider/testdata/provider/local_readahead b/objstorage/objstorageprovider/testdata/provider/local_readahead index 0f9873f0ad..87266cc45d 100644 --- a/objstorage/objstorageprovider/testdata/provider/local_readahead +++ b/objstorage/objstorageprovider/testdata/provider/local_readahead @@ -3,12 +3,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 local 1 2000000 ---- diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach b/objstorage/objstorageprovider/testdata/provider/shared_attach index be60d5ff15..aa121686dc 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach @@ -6,12 +6,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 100 ---- @@ -64,8 +64,8 @@ save-backing b3 3 close ---- sync: p1 - sync: p1/SHARED-CATALOG-000001 - close: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + close: p1/REMOTE-OBJ-CATALOG-000001 close: p1 open p2 2 @@ -73,12 +73,12 @@ open p2 2 mkdir-all: p2 0755 open-dir: p2 open-dir: p2 - create: p2/SHARED-CATALOG-000001 - sync: p2/SHARED-CATALOG-000001 - create: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p2/REMOTE-OBJ-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 + create: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p2 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 create 100 shared 100 15 ---- @@ -101,7 +101,7 @@ b3 103 create object "eaac-1-000003.sst.ref.2.000103" close writer for "eaac-1-000003.sst.ref.2.000103" after 0 bytes size of object "eaac-1-000003.sst.ref.1.000003": 0 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 000101 -> shared://61a6-1-000001.sst 000102 -> shared://a629-1-000002.sst 000103 -> shared://eaac-1-000003.sst diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref b/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref index d13f692fd4..8b4c462841 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref @@ -5,12 +5,12 @@ open p5 5 mkdir-all: p5 0755 open-dir: p5 open-dir: p5 - create: p5/SHARED-CATALOG-000001 - sync: p5/SHARED-CATALOG-000001 - create: p5/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p5/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p5/REMOTE-OBJ-CATALOG-000001 + sync: p5/REMOTE-OBJ-CATALOG-000001 + create: p5/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p5/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p5 - sync: p5/SHARED-CATALOG-000001 + sync: p5/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 100 ---- @@ -31,12 +31,12 @@ open p6 6 mkdir-all: p6 0755 open-dir: p6 open-dir: p6 - create: p6/SHARED-CATALOG-000001 - sync: p6/SHARED-CATALOG-000001 - create: p6/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p6/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p6/REMOTE-OBJ-CATALOG-000001 + sync: p6/REMOTE-OBJ-CATALOG-000001 + create: p6/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p6/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p6 - sync: p6/SHARED-CATALOG-000001 + sync: p6/REMOTE-OBJ-CATALOG-000001 # Attach should succeed. attach @@ -45,7 +45,7 @@ p5b1 101 create object "d632-5-000001.sst.ref.6.000101" close writer for "d632-5-000001.sst.ref.6.000101" after 0 bytes size of object "d632-5-000001.sst.ref.5.000001": 0 - sync: p6/SHARED-CATALOG-000001 + sync: p6/REMOTE-OBJ-CATALOG-000001 000101 -> shared://d632-5-000001.sst switch p5 diff --git a/objstorage/objstorageprovider/testdata/provider/shared_attach_multi b/objstorage/objstorageprovider/testdata/provider/shared_attach_multi index 343cb4b8ab..1167698f0d 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_attach_multi +++ b/objstorage/objstorageprovider/testdata/provider/shared_attach_multi @@ -5,12 +5,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 100 ---- @@ -27,12 +27,12 @@ open p2 2 mkdir-all: p2 0755 open-dir: p2 open-dir: p2 - create: p2/SHARED-CATALOG-000001 - sync: p2/SHARED-CATALOG-000001 - create: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p2/REMOTE-OBJ-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 + create: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p2 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 # We should create three ref markers to allow independent removal. attach @@ -49,7 +49,7 @@ b1 103 create object "61a6-1-000001.sst.ref.2.000103" close writer for "61a6-1-000001.sst.ref.2.000103" after 0 bytes size of object "61a6-1-000001.sst.ref.1.000001": 0 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 000101 -> shared://61a6-1-000001.sst 000102 -> shared://61a6-1-000001.sst 000103 -> shared://61a6-1-000001.sst diff --git a/objstorage/objstorageprovider/testdata/provider/shared_basic b/objstorage/objstorageprovider/testdata/provider/shared_basic index bebc95ccd6..7e1ae39be0 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_basic +++ b/objstorage/objstorageprovider/testdata/provider/shared_basic @@ -6,12 +6,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 local 1 100 ---- @@ -53,8 +53,8 @@ list close ---- sync: p1 - sync: p1/SHARED-CATALOG-000001 - close: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + close: p1/REMOTE-OBJ-CATALOG-000001 close: p1 # Test that the objects are there on re-open. @@ -63,8 +63,8 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - open: p1/SHARED-CATALOG-000001 - close: p1/SHARED-CATALOG-000001 + open: p1/REMOTE-OBJ-CATALOG-000001 + close: p1/REMOTE-OBJ-CATALOG-000001 list ---- @@ -120,13 +120,13 @@ size: 100 close ---- sync: p1 - create: p1/SHARED-CATALOG-000002 - sync: p1/SHARED-CATALOG-000002 - create: p1/marker.shared-catalog.000002.SHARED-CATALOG-000002 - close: p1/marker.shared-catalog.000002.SHARED-CATALOG-000002 - remove: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000002 + sync: p1/REMOTE-OBJ-CATALOG-000002 + create: p1/marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 + close: p1/marker.remote-obj-catalog.000002.REMOTE-OBJ-CATALOG-000002 + remove: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - remove: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000002 - close: p1/SHARED-CATALOG-000002 + remove: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000002 + close: p1/REMOTE-OBJ-CATALOG-000002 close: p1 diff --git a/objstorage/objstorageprovider/testdata/provider/shared_no_ref b/objstorage/objstorageprovider/testdata/provider/shared_no_ref index 8b6d079ca3..f1075c5ea2 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_no_ref +++ b/objstorage/objstorageprovider/testdata/provider/shared_no_ref @@ -6,12 +6,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 100 no-ref-tracking ---- @@ -66,8 +66,8 @@ size: 100 close ---- - sync: p1/SHARED-CATALOG-000001 - close: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + close: p1/REMOTE-OBJ-CATALOG-000001 close: p1 # Test that the objects are there on re-open. @@ -76,8 +76,8 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - open: p1/SHARED-CATALOG-000001 - close: p1/SHARED-CATALOG-000001 + open: p1/REMOTE-OBJ-CATALOG-000001 + close: p1/REMOTE-OBJ-CATALOG-000001 list ---- @@ -123,18 +123,18 @@ open p2 2 mkdir-all: p2 0755 open-dir: p2 open-dir: p2 - create: p2/SHARED-CATALOG-000001 - sync: p2/SHARED-CATALOG-000001 - create: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p2/REMOTE-OBJ-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 + create: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p2 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 attach b1 101 b2 102 ---- - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 000101 -> shared://61a6-1-000001.sst 000102 -> shared://61a6-1-000001.sst diff --git a/objstorage/objstorageprovider/testdata/provider/shared_readahead b/objstorage/objstorageprovider/testdata/provider/shared_readahead index 2088a314bf..aba18cbf9a 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_readahead +++ b/objstorage/objstorageprovider/testdata/provider/shared_readahead @@ -3,12 +3,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 2000000 ---- diff --git a/objstorage/objstorageprovider/testdata/provider/shared_remove b/objstorage/objstorageprovider/testdata/provider/shared_remove index 353cc04c8d..5204ca641a 100644 --- a/objstorage/objstorageprovider/testdata/provider/shared_remove +++ b/objstorage/objstorageprovider/testdata/provider/shared_remove @@ -3,12 +3,12 @@ open p1 1 mkdir-all: p1 0755 open-dir: p1 open-dir: p1 - create: p1/SHARED-CATALOG-000001 - sync: p1/SHARED-CATALOG-000001 - create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p1/REMOTE-OBJ-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 + create: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p1/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p1 - sync: p1/SHARED-CATALOG-000001 + sync: p1/REMOTE-OBJ-CATALOG-000001 create 1 shared 1 100 ---- @@ -42,12 +42,12 @@ open p2 2 mkdir-all: p2 0755 open-dir: p2 open-dir: p2 - create: p2/SHARED-CATALOG-000001 - sync: p2/SHARED-CATALOG-000001 - create: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 - close: p2/marker.shared-catalog.000001.SHARED-CATALOG-000001 + create: p2/REMOTE-OBJ-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 + create: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 + close: p2/marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 sync: p2 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 create 4 shared 4 100 ---- @@ -66,7 +66,7 @@ b2 102 create object "a629-1-000002.sst.ref.2.000102" close writer for "a629-1-000002.sst.ref.2.000102" after 0 bytes size of object "a629-1-000002.sst.ref.1.000002": 0 - sync: p2/SHARED-CATALOG-000001 + sync: p2/REMOTE-OBJ-CATALOG-000001 000101 -> shared://61a6-1-000001.sst 000102 -> shared://a629-1-000002.sst diff --git a/scan_internal.go b/scan_internal.go index 6bd075e8e1..747a981fbf 100644 --- a/scan_internal.go +++ b/scan_internal.go @@ -35,7 +35,7 @@ var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared i // SharedSSTMeta represents an sstable on shared storage that can be ingested // by another pebble instance. This struct must contain all fields that are // required for a Pebble instance to ingest a foreign sstable on shared storage, -// including constructing any relevant objstorage.Provider / sharedobjcat.Catalog +// including constructing any relevant objstorage.Provider / remoteobjcat.Catalog // data structures, as well as creating virtual FileMetadatas. // // Note that the Pebble instance creating and returning a SharedSSTMeta might