Skip to content

Commit

Permalink
Split serialization into composition
Browse files Browse the repository at this point in the history
This also moves out copyall, removes the duplicated reload function,
adds a more general list func.

Signed-off-by: Thomas Jungblut <[email protected]>
  • Loading branch information
tjungblu committed Jul 25, 2024
1 parent 00c8969 commit 2f5d584
Show file tree
Hide file tree
Showing 11 changed files with 149 additions and 144 deletions.
8 changes: 5 additions & 3 deletions db.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@ type DB struct {
rwtx *Tx
txs []*Tx

freelist fl.Interface
freelistLoad sync.Once
freelist fl.Interface
freelistLoad sync.Once
freelistReadWriter fl.ReadWriter

pagePool sync.Pool

Expand Down Expand Up @@ -417,12 +418,13 @@ func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
func (db *DB) loadFreelist() {
db.freelistLoad.Do(func() {
db.freelist = newFreelist(db.FreelistType)
db.freelistReadWriter = fl.NewSortedSerializer()
if !db.hasSyncedFreelist() {
// Reconstruct free list by scanning the DB.
db.freelist.Init(db.freepages())
} else {
// Read free list from freelist page.
db.freelist.Read(db.page(db.meta().Freelist()))
db.freelistReadWriter.Read(db.freelist, db.page(db.meta().Freelist()))
}
db.stats.FreePageN = db.freelist.FreeCount()
})
Expand Down
2 changes: 1 addition & 1 deletion internal/freelist/array.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (f *array) FreeCount() int {
return len(f.ids)
}

func (f *array) freePageIds() common.Pgids {
func (f *array) FreePageIds() common.Pgids {
return f.ids
}

Expand Down
8 changes: 4 additions & 4 deletions internal/freelist/array_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ func TestFreelistArray_allocate(t *testing.T) {
if id := int(f.Allocate(1, 0)); id != 0 {
t.Fatalf("exp=0; got=%v", id)
}
if exp := common.Pgids([]common.Pgid{9, 18}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
if exp := common.Pgids([]common.Pgid{9, 18}); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.FreePageIds())
}

if id := int(f.Allocate(1, 1)); id != 9 {
Expand All @@ -46,7 +46,7 @@ func TestFreelistArray_allocate(t *testing.T) {
if id := int(f.Allocate(1, 1)); id != 0 {
t.Fatalf("exp=0; got=%v", id)
}
if exp := common.Pgids([]common.Pgid{}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
if exp := common.Pgids([]common.Pgid{}); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.FreePageIds())
}
}
23 changes: 9 additions & 14 deletions internal/freelist/freelist.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ import (

type ReadWriter interface {
// Read calls Init with the page ids stored in the given page.
Read(page *common.Page)
Read(f Interface, page *common.Page)

// Write writes the freelist into the given page.
Write(page *common.Page)
Write(f Interface, page *common.Page)

// EstimatedWritePageSize returns the size of the freelist after serialization in Write.
// This should never underestimate the size.
EstimatedWritePageSize() int
EstimatedWritePageSize(f Interface) int
}

type allocator interface {
Expand All @@ -23,8 +23,8 @@ type allocator interface {
// FreeCount returns the number of free pages.
FreeCount() int

// freePageIds returns the IDs of all free pages.
freePageIds() common.Pgids
// FreePageIds returns the IDs of all free pages.
FreePageIds() common.Pgids

// mergeSpans is merging the given pages into the freelist
mergeSpans(ids common.Pgids)
Expand Down Expand Up @@ -54,7 +54,6 @@ type txManager interface {
}

type Interface interface {
ReadWriter
allocator
txManager

Expand All @@ -79,13 +78,9 @@ type Interface interface {
// Rollback removes the pages from a given pending tx.
Rollback(txId common.Txid)

// Copyall copies a list of all free ids and all pending ids in one sorted list.
// f.count returns the minimum length required for dst.
Copyall(dst []common.Pgid)
// List returns a list of all free ids and all pending ids in one sorted list.
List() common.Pgids

// Reload reads the freelist from a page and filters out pending items.
Reload(p *common.Page)

// NoSyncReload reads the freelist from Pgids and filters out pending items.
NoSyncReload(pgIds common.Pgids)
// Reload reads the freelist from Pgids and filters out pending items.
Reload(pgIds common.Pgids)
}
32 changes: 17 additions & 15 deletions internal/freelist/freelist_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ func TestFreelist_release(t *testing.T) {
f.Free(102, common.NewPage(39, 0, 0, 0))
f.release(100)
f.release(101)
if exp := common.Pgids([]common.Pgid{9, 12, 13}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
if exp := common.Pgids([]common.Pgid{9, 12, 13}); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.FreePageIds())
}

f.release(102)
if exp := common.Pgids([]common.Pgid{9, 12, 13, 39}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
if exp := common.Pgids([]common.Pgid{9, 12, 13, 39}); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.FreePageIds())
}
}

Expand Down Expand Up @@ -173,8 +173,8 @@ func TestFreelist_releaseRange(t *testing.T) {
f.releaseRange(r.begin, r.end)
}

if exp := common.Pgids(c.wantFree); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Errorf("exp=%v; got=%v for %s", exp, f.freePageIds(), c.title)
if exp := common.Pgids(c.wantFree); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Errorf("exp=%v; got=%v for %s", exp, f.FreePageIds(), c.title)
}
}
}
Expand All @@ -194,11 +194,12 @@ func TestFreelist_read(t *testing.T) {

// Deserialize page into a freelist.
f := newTestFreelist()
f.Read(page)
s := SortedSerializer{}
s.Read(f, page)

// Ensure that there are two page ids in the freelist.
if exp := common.Pgids([]common.Pgid{23, 50}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
if exp := common.Pgids([]common.Pgid{23, 50}); !reflect.DeepEqual(exp, f.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.FreePageIds())
}
}

Expand All @@ -207,21 +208,22 @@ func TestFreelist_write(t *testing.T) {
// Create a freelist and write it to a page.
var buf [4096]byte
f := newTestFreelist()
s := SortedSerializer{}

f.Init([]common.Pgid{12, 39})
f.pendingPageIds()[100] = &txPending{ids: []common.Pgid{28, 11}}
f.pendingPageIds()[101] = &txPending{ids: []common.Pgid{3}}
p := (*common.Page)(unsafe.Pointer(&buf[0]))
f.Write(p)
s.Write(f, p)

// Read the page back out.
f2 := newTestFreelist()
f2.Read(p)
s.Read(f2, p)

// Ensure that the freelist is correct.
// All pages should be present and in reverse order.
if exp := common.Pgids([]common.Pgid{3, 11, 12, 28, 39}); !reflect.DeepEqual(exp, f2.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f2.freePageIds())
if exp := common.Pgids([]common.Pgid{3, 11, 12, 28, 39}); !reflect.DeepEqual(exp, f2.FreePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f2.FreePageIds())
}
}

Expand Down Expand Up @@ -258,15 +260,15 @@ func Test_freelist_ReadIDs_and_getFreePageIDs(t *testing.T) {

f.Init(exp)

if got := f.freePageIds(); !reflect.DeepEqual(exp, got) {
if got := f.FreePageIds(); !reflect.DeepEqual(exp, got) {
t.Fatalf("exp=%v; got=%v", exp, got)
}

f2 := newTestFreelist()
var exp2 []common.Pgid
f2.Init(exp2)

if got2 := f2.freePageIds(); !reflect.DeepEqual(got2, common.Pgids(exp2)) {
if got2 := f2.FreePageIds(); !reflect.DeepEqual(got2, common.Pgids(exp2)) {
t.Fatalf("exp2=%#v; got2=%#v", exp2, got2)
}

Expand Down
2 changes: 1 addition & 1 deletion internal/freelist/hashmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (f *hashMap) FreeCount() int {
return int(f.freePagesCount)
}

func (f *hashMap) freePageIds() common.Pgids {
func (f *hashMap) FreePageIds() common.Pgids {
count := f.FreeCount()
if count == 0 {
return nil
Expand Down
6 changes: 3 additions & 3 deletions internal/freelist/hashmap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func TestFreelistHashmap_mergeWithExist(t *testing.T) {

f.mergeWithExistingSpan(tt.pgid)

if got := f.freePageIds(); !reflect.DeepEqual(tt.want, got) {
if got := f.FreePageIds(); !reflect.DeepEqual(tt.want, got) {
t.Fatalf("name %s; exp=%v; got=%v", tt.name, tt.want, got)
}
if got := f.forwardMap; !reflect.DeepEqual(tt.wantForwardmap, got) {
Expand Down Expand Up @@ -121,7 +121,7 @@ func TestFreelistHashmap_GetFreePageIDs(t *testing.T) {
}

f.forwardMap = fm
res := f.freePageIds()
res := f.FreePageIds()

if !sort.SliceIsSorted(res, func(i, j int) bool { return res[i] < res[j] }) {
t.Fatalf("pgids not sorted")
Expand All @@ -145,6 +145,6 @@ func Benchmark_freelist_hashmapGetFreePageIDs(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
f.freePageIds()
f.FreePageIds()
}
}
82 changes: 82 additions & 0 deletions internal/freelist/read_writer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package freelist

import (
"fmt"
"sort"
"unsafe"

"go.etcd.io/bbolt/internal/common"
)

type SortedSerializer struct {
}

func (s *SortedSerializer) Read(t Interface, p *common.Page) {
if !p.IsFreelistPage() {
panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.Id(), p.Typ()))
}

ids := p.FreelistPageIds()

// Copy the list of page ids from the freelist.
if len(ids) == 0 {
t.Init(nil)
} else {
// copy the ids, so we don't modify on the freelist page directly
idsCopy := make([]common.Pgid, len(ids))
copy(idsCopy, ids)
// Make sure they're sorted.
sort.Sort(common.Pgids(idsCopy))

t.Init(idsCopy)
}
}

func (s *SortedSerializer) EstimatedWritePageSize(t Interface) int {
n := t.Count()
if n >= 0xFFFF {
// The first element will be used to store the count. See freelist.write.
n++
}
return int(common.PageHeaderSize) + (int(unsafe.Sizeof(common.Pgid(0))) * n)
}

func (s *SortedSerializer) Write(t Interface, p *common.Page) {
// Combine the old free pgids and pgids waiting on an open transaction.

// Update the header flag.
p.SetFlags(common.FreelistPageFlag)

// The page.count can only hold up to 64k elements so if we overflow that
// number then we handle it by putting the size in the first element.
l := t.Count()
if l == 0 {
p.SetCount(uint16(l))
} else if l < 0xFFFF {
p.SetCount(uint16(l))
data := common.UnsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p))
ids := unsafe.Slice((*common.Pgid)(data), l)
copyall(t, ids)
} else {
p.SetCount(0xFFFF)
data := common.UnsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p))
ids := unsafe.Slice((*common.Pgid)(data), l+1)
ids[0] = common.Pgid(l)
copyall(t, ids[1:])
}
}

// copyall copies a list of all free ids and all pending ids in one sorted list.
// f.count returns the minimum length required for dst.
func copyall(t Interface, dst []common.Pgid) {
m := make(common.Pgids, 0, t.PendingCount())
for _, txp := range t.pendingPageIds() {
m = append(m, txp.ids...)
}
sort.Sort(m)
common.Mergepgids(dst, t.FreePageIds(), m)
}

func NewSortedSerializer() ReadWriter {
return &SortedSerializer{}
}
Loading

0 comments on commit 2f5d584

Please sign in to comment.