Skip to content

Commit

Permalink
objstorage: rename sharedobjcat to remoteobjcat
Browse files Browse the repository at this point in the history 
Also rename the catalog filename to use `REMOTE-OBJ-CATALOG`.
  • Loading branch information
@RaduBerinde
RaduBerinde committed Jul 21, 2023
1 parent 94f9166 commit 5db8494
Show file tree
Hide file tree
Showing 18 changed files with 402 additions and 402 deletions.
6 changes: 3 additions & 3 deletions objstorage/objstorageprovider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
Expand All @@ -38,7 +38,7 @@ type provider struct {
shared struct {
// catalogBatch accumulates shared object creations and deletions until
// Sync is called.
catalogBatch sharedobjcat.Batch
catalogBatch remoteobjcat.Batch

storageObjects map[remote.Locator]remote.Storage
}
Expand Down Expand Up @@ -447,7 +447,7 @@ func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
defer p.mu.Unlock()
p.mu.knownObjects[meta.DiskFileNum] = meta
if meta.IsRemote() {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
p.mu.shared.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: meta.DiskFileNum,
FileType: meta.FileType,
CreatorID: meta.Remote.CreatorID,
Expand Down
38 changes: 19 additions & 19 deletions ...bjstorageprovider/sharedobjcat/catalog.go → ...bjstorageprovider/remoteobjcat/catalog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package sharedobjcat
package remoteobjcat

import (
"fmt"
Expand All @@ -19,7 +19,7 @@ import (
"github.com/cockroachdb/pebble/vfs/atomicfs"
)

// Catalog is used to manage the on-disk shared object catalog.
// Catalog is used to manage the on-disk remote object catalog.
//
// The catalog file is a log of records, where each record is an encoded
// versionEdit.
Expand All @@ -30,7 +30,7 @@ type Catalog struct {
sync.Mutex

creatorID objstorage.CreatorID
objects map[base.DiskFileNum]SharedObjectMetadata
objects map[base.DiskFileNum]RemoteObjectMetadata

marker *atomicfs.Marker

Expand All @@ -45,8 +45,8 @@ type Catalog struct {
}
}

// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
type SharedObjectMetadata struct {
// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
type RemoteObjectMetadata struct {
// FileNum is the identifier for the object within the context of a single DB
// instance.
FileNum base.DiskFileNum
Expand All @@ -67,19 +67,19 @@ type SharedObjectMetadata struct {
}

const (
catalogFilenameBase = "SHARED-CATALOG"
catalogMarkerName = "shared-catalog"
catalogFilenameBase = "REMOTE-OBJ-CATALOG"
catalogMarkerName = "remote-obj-catalog"

// We create a new file when the size exceeds 1MB (and some other conditions
// hold; see record.RotationHelper).
rotateFileSize = 1024 * 1024 // 1MB
)

// CatalogContents contains the shared objects in the catalog.
// CatalogContents contains the remote objects in the catalog.
type CatalogContents struct {
// CreatorID, if it is set.
CreatorID objstorage.CreatorID
Objects []SharedObjectMetadata
Objects []RemoteObjectMetadata
}

// Open creates a Catalog and loads any existing catalog file, returning the
Expand All @@ -89,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
fs: fs,
dirname: dirname,
}
c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)

var err error
c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
Expand All @@ -108,7 +108,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
}
res := CatalogContents{
CreatorID: c.mu.creatorID,
Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)),
Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
}
for _, meta := range c.mu.objects {
res.Objects = append(res.Objects, meta)
Expand Down Expand Up @@ -138,7 +138,7 @@ func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {

ve := versionEdit{CreatorID: id}
if err := c.writeToCatalogFileLocked(&ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
c.mu.creatorID = id
return nil
Expand Down Expand Up @@ -172,7 +172,7 @@ type Batch struct {
//
// The given FileNum must be new - it must not match that of any object that was
// ever in the catalog.
func (b *Batch) AddObject(meta SharedObjectMetadata) {
func (b *Batch) AddObject(meta RemoteObjectMetadata) {
b.ve.NewObjects = append(b.ve.NewObjects, meta)
}

Expand All @@ -196,7 +196,7 @@ func (b *Batch) IsEmpty() bool {
func (b *Batch) Copy() Batch {
var res Batch
if len(b.ve.NewObjects) > 0 {
res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
copy(res.ve.NewObjects, b.ve.NewObjects)
}
if len(b.ve.DeletedObjects) > 0 {
Expand Down Expand Up @@ -245,7 +245,7 @@ func (c *Catalog) ApplyBatch(b Batch) error {
}

if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}

b.Reset()
Expand All @@ -257,7 +257,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
f, err := c.fs.Open(catalogPath)
if err != nil {
return errors.Wrapf(
err, "pebble: could not open shared object catalog file %q for DB %q",
err, "pebble: could not open remote object catalog file %q for DB %q",
errors.Safe(filename), c.dirname,
)
}
Expand All @@ -269,13 +269,13 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
break
}
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
var ve versionEdit
err = ve.Decode(r)
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
// Apply the version edit to the current state.
Expand Down Expand Up @@ -340,7 +340,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
// Create a versionEdit that gets us from an empty catalog to the current state.
var ve versionEdit
ve.CreatorID = c.mu.creatorID
ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
for _, meta := range c.mu.objects {
ve.NewObjects = append(ve.NewObjects, meta)
}
Expand Down
20 changes: 10 additions & 10 deletions ...rageprovider/sharedobjcat/catalog_test.go → ...rageprovider/remoteobjcat/catalog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package sharedobjcat_test
package remoteobjcat_test

import (
"fmt"
Expand All @@ -15,15 +15,15 @@ import (
"github.com/cockroachdb/datadriven"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/vfs"
)

func TestCatalog(t *testing.T) {
mem := vfs.NewMem()
var memLog base.InMemLogger

var cat *sharedobjcat.Catalog
var cat *remoteobjcat.Catalog
datadriven.RunTest(t, "testdata/catalog", func(t *testing.T, td *datadriven.TestData) string {
toUInt64 := func(args ...string) []uint64 {
t.Helper()
Expand All @@ -38,13 +38,13 @@ func TestCatalog(t *testing.T) {
return res
}

parseAdd := func(args []string) sharedobjcat.SharedObjectMetadata {
parseAdd := func(args []string) remoteobjcat.RemoteObjectMetadata {
t.Helper()
if len(args) != 3 {
td.Fatalf(t, "add <file-num> <creator-id> <creator-file-num>")
}
vals := toUInt64(args...)
return sharedobjcat.SharedObjectMetadata{
return remoteobjcat.RemoteObjectMetadata{
FileNum: base.FileNum(vals[0]).DiskFileNum(),
// When we support other file types, we should let the test determine this.
FileType: base.FileTypeTable,
Expand Down Expand Up @@ -72,8 +72,8 @@ func TestCatalog(t *testing.T) {
if err != nil {
td.Fatalf(t, "%v", err)
}
var contents sharedobjcat.CatalogContents
cat, contents, err = sharedobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname)
var contents remoteobjcat.CatalogContents
cat, contents, err = remoteobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname)
if err != nil {
return err.Error()
}
Expand All @@ -98,7 +98,7 @@ func TestCatalog(t *testing.T) {
return memLog.String()

case "batch":
var b sharedobjcat.Batch
var b remoteobjcat.Batch
for _, cmd := range strings.Split(td.Input, "\n") {
tokens := strings.Split(cmd, " ")
if len(tokens) == 0 {
Expand Down Expand Up @@ -136,10 +136,10 @@ func TestCatalog(t *testing.T) {
td.Fatalf(t, "random-batches n=<val> size=<val>")
}
}
var b sharedobjcat.Batch
var b remoteobjcat.Batch
for batchIdx := 0; batchIdx < n; batchIdx++ {
for i := 0; i < size; i++ {
b.AddObject(sharedobjcat.SharedObjectMetadata{
b.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: base.FileNum(rand.Uint64()).DiskFileNum(),
// When we support other file types, we should let the test determine this.
FileType: base.FileTypeTable,
Expand Down
Loading

0 comments on commit 5db8494

Please sign in to comment.