Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

objstorage: rename sharedobjcat to remoteobjcat #2767

Merged
merged 1 commit into from
Jul 24, 2023
Merged

objstorage: rename sharedobjcat to remoteobjcat #2767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions objstorage/objstorageprovider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
Expand All @@ -38,7 +38,7 @@ type provider struct {
shared struct {
// catalogBatch accumulates shared object creations and deletions until
// Sync is called.
catalogBatch sharedobjcat.Batch
catalogBatch remoteobjcat.Batch

storageObjects map[remote.Locator]remote.Storage
}
Expand Down Expand Up @@ -447,7 +447,7 @@ func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
defer p.mu.Unlock()
p.mu.knownObjects[meta.DiskFileNum] = meta
if meta.IsRemote() {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
p.mu.shared.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: meta.DiskFileNum,
FileType: meta.FileType,
CreatorID: meta.Remote.CreatorID,
Expand Down
38 changes: 19 additions & 19 deletions ...bjstorageprovider/sharedobjcat/catalog.go → ...bjstorageprovider/remoteobjcat/catalog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package sharedobjcat
package remoteobjcat

import (
"fmt"
Expand All @@ -19,7 +19,7 @@ import (
"github.com/cockroachdb/pebble/vfs/atomicfs"
)

// Catalog is used to manage the on-disk shared object catalog.
// Catalog is used to manage the on-disk remote object catalog.
//
// The catalog file is a log of records, where each record is an encoded
// versionEdit.
Expand All @@ -30,7 +30,7 @@ type Catalog struct {
sync.Mutex

creatorID objstorage.CreatorID
objects map[base.DiskFileNum]SharedObjectMetadata
objects map[base.DiskFileNum]RemoteObjectMetadata

marker *atomicfs.Marker

Expand All @@ -45,8 +45,8 @@ type Catalog struct {
}
}

// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
type SharedObjectMetadata struct {
// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
type RemoteObjectMetadata struct {
// FileNum is the identifier for the object within the context of a single DB
// instance.
FileNum base.DiskFileNum
Expand All @@ -67,19 +67,19 @@ type SharedObjectMetadata struct {
}

const (
catalogFilenameBase = "SHARED-CATALOG"
catalogMarkerName = "shared-catalog"
catalogFilenameBase = "REMOTE-OBJ-CATALOG"
catalogMarkerName = "remote-obj-catalog"

// We create a new file when the size exceeds 1MB (and some other conditions
// hold; see record.RotationHelper).
rotateFileSize = 1024 * 1024 // 1MB
)

// CatalogContents contains the shared objects in the catalog.
// CatalogContents contains the remote objects in the catalog.
type CatalogContents struct {
// CreatorID, if it is set.
CreatorID objstorage.CreatorID
Objects []SharedObjectMetadata
Objects []RemoteObjectMetadata
}

// Open creates a Catalog and loads any existing catalog file, returning the
Expand All @@ -89,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
fs: fs,
dirname: dirname,
}
c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)

var err error
c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
Expand All @@ -108,7 +108,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
}
res := CatalogContents{
CreatorID: c.mu.creatorID,
Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)),
Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
}
for _, meta := range c.mu.objects {
res.Objects = append(res.Objects, meta)
Expand Down Expand Up @@ -138,7 +138,7 @@ func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {

ve := versionEdit{CreatorID: id}
if err := c.writeToCatalogFileLocked(&ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
c.mu.creatorID = id
return nil
Expand Down Expand Up @@ -172,7 +172,7 @@ type Batch struct {
//
// The given FileNum must be new - it must not match that of any object that was
// ever in the catalog.
func (b *Batch) AddObject(meta SharedObjectMetadata) {
func (b *Batch) AddObject(meta RemoteObjectMetadata) {
b.ve.NewObjects = append(b.ve.NewObjects, meta)
}

Expand All @@ -196,7 +196,7 @@ func (b *Batch) IsEmpty() bool {
func (b *Batch) Copy() Batch {
var res Batch
if len(b.ve.NewObjects) > 0 {
res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
copy(res.ve.NewObjects, b.ve.NewObjects)
}
if len(b.ve.DeletedObjects) > 0 {
Expand Down Expand Up @@ -245,7 +245,7 @@ func (c *Catalog) ApplyBatch(b Batch) error {
}

if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}

b.Reset()
Expand All @@ -257,7 +257,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
f, err := c.fs.Open(catalogPath)
if err != nil {
return errors.Wrapf(
err, "pebble: could not open shared object catalog file %q for DB %q",
err, "pebble: could not open remote object catalog file %q for DB %q",
errors.Safe(filename), c.dirname,
)
}
Expand All @@ -269,13 +269,13 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
break
}
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
var ve versionEdit
err = ve.Decode(r)
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
// Apply the version edit to the current state.
Expand Down Expand Up @@ -340,7 +340,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
// Create a versionEdit that gets us from an empty catalog to the current state.
var ve versionEdit
ve.CreatorID = c.mu.creatorID
ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
for _, meta := range c.mu.objects {
ve.NewObjects = append(ve.NewObjects, meta)
}
Expand Down
20 changes: 10 additions & 10 deletions ...rageprovider/sharedobjcat/catalog_test.go → ...rageprovider/remoteobjcat/catalog_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package sharedobjcat_test
package remoteobjcat_test

import (
"fmt"
Expand All @@ -15,15 +15,15 @@ import (
"github.com/cockroachdb/datadriven"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/vfs"
)

func TestCatalog(t *testing.T) {
mem := vfs.NewMem()
var memLog base.InMemLogger

var cat *sharedobjcat.Catalog
var cat *remoteobjcat.Catalog
datadriven.RunTest(t, "testdata/catalog", func(t *testing.T, td *datadriven.TestData) string {
toUInt64 := func(args ...string) []uint64 {
t.Helper()
Expand All @@ -38,13 +38,13 @@ func TestCatalog(t *testing.T) {
return res
}

parseAdd := func(args []string) sharedobjcat.SharedObjectMetadata {
parseAdd := func(args []string) remoteobjcat.RemoteObjectMetadata {
t.Helper()
if len(args) != 3 {
td.Fatalf(t, "add <file-num> <creator-id> <creator-file-num>")
}
vals := toUInt64(args...)
return sharedobjcat.SharedObjectMetadata{
return remoteobjcat.RemoteObjectMetadata{
FileNum: base.FileNum(vals[0]).DiskFileNum(),
// When we support other file types, we should let the test determine this.
FileType: base.FileTypeTable,
Expand Down Expand Up @@ -72,8 +72,8 @@ func TestCatalog(t *testing.T) {
if err != nil {
td.Fatalf(t, "%v", err)
}
var contents sharedobjcat.CatalogContents
cat, contents, err = sharedobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname)
var contents remoteobjcat.CatalogContents
cat, contents, err = remoteobjcat.Open(vfs.WithLogging(mem, memLog.Infof), dirname)
if err != nil {
return err.Error()
}
Expand All @@ -98,7 +98,7 @@ func TestCatalog(t *testing.T) {
return memLog.String()

case "batch":
var b sharedobjcat.Batch
var b remoteobjcat.Batch
for _, cmd := range strings.Split(td.Input, "\n") {
tokens := strings.Split(cmd, " ")
if len(tokens) == 0 {
Expand Down Expand Up @@ -136,10 +136,10 @@ func TestCatalog(t *testing.T) {
td.Fatalf(t, "random-batches n=<val> size=<val>")
}
}
var b sharedobjcat.Batch
var b remoteobjcat.Batch
for batchIdx := 0; batchIdx < n; batchIdx++ {
for i := 0; i < size; i++ {
b.AddObject(sharedobjcat.SharedObjectMetadata{
b.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: base.FileNum(rand.Uint64()).DiskFileNum(),
// When we support other file types, we should let the test determine this.
FileType: base.FileTypeTable,
Expand Down
Loading
Loading