Skip to content

Commit

Permalink
add freelist interface unit tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Thomas Jungblut <[email protected]>
  • Loading branch information
@tjungblu
tjungblu committed Aug 6, 2024
1 parent 978df69 commit e53f4c4
Show file tree
Hide file tree
Showing 3 changed files with 355 additions and 0 deletions.
11 changes: 11 additions & 0 deletions internal/freelist/array_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"reflect"
"testing"

"github.com/stretchr/testify/require"

"go.etcd.io/bbolt/internal/common"
)

Expand Down Expand Up @@ -50,3 +52,12 @@ func TestFreelistArray_allocate(t *testing.T) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
}
}

func TestInvalidArrayAllocation(t *testing.T) {
f := NewArrayFreelist()
ids := []common.Pgid{1}
f.Init(ids)
require.Panics(t, func() {
f.Allocate(common.Txid(1), 1)
})
}
334 changes: 334 additions & 0 deletions internal/freelist/freelist_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package freelist

import (
"fmt"
"math"
"math/rand"
"os"
"reflect"
"slices"
"sort"
"testing"
"testing/quick"
"unsafe"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -34,6 +38,56 @@ func TestFreelist_free_overflow(t *testing.T) {
}
}

// Ensure that double freeing a page is causing a panic
func TestFreelist_free_double_free_panics(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, 0, 0, 3))
require.Panics(t, func() {
f.Free(100, common.NewPage(12, 0, 0, 3))
})
}

// Ensure that attempting to free the meta page panics
func TestFreelist_free_meta_panics(t *testing.T) {
f := newTestFreelist()
require.Panics(t, func() {
f.Free(100, common.NewPage(0, 0, 0, 0))
})
require.Panics(t, func() {
f.Free(100, common.NewPage(1, 0, 0, 0))
})
}

func TestFreelist_free_freelist(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, common.FreelistPageFlag, 0, 0))
pp := f.pendingPageIds()[100]
require.Equal(t, []common.Pgid{12}, pp.ids)
require.Equal(t, []common.Txid{0}, pp.alloctx)
}

func TestFreelist_free_freelist_alloctx(t *testing.T) {
f := newTestFreelist()
f.Free(100, common.NewPage(12, common.FreelistPageFlag, 0, 0))
f.Rollback(100)
require.Empty(t, f.freePageIds())
require.Empty(t, f.pendingPageIds())
require.False(t, f.Freed(12))

// we still hold an allotx reference to page 12 through txid 99 - let's try to free it again
f.Free(101, common.NewPage(12, common.FreelistPageFlag, 0, 0))
require.True(t, f.Freed(12))
if exp := []common.Pgid{12}; !reflect.DeepEqual(exp, f.pendingPageIds()[101].ids) {
t.Fatalf("exp=%v; got=%v", exp, f.pendingPageIds()[101].ids)
}
f.ReleasePendingPages()
require.True(t, f.Freed(12))
require.Empty(t, f.pendingPageIds())
if exp := common.Pgids([]common.Pgid{12}); !reflect.DeepEqual(exp, f.freePageIds()) {
t.Fatalf("exp=%v; got=%v", exp, f.freePageIds())
}
}

// Ensure that a transaction's free pages can be released.
func TestFreelist_release(t *testing.T) {
f := newTestFreelist()
Expand Down Expand Up @@ -220,6 +274,30 @@ func TestFreeList_reload(t *testing.T) {
require.Equal(t, []common.Pgid{10, 11, 12}, f2.pendingPageIds()[5].ids)
}

// Ensure that the txIDx swap, less and len are properly implemented
func TestTxidSorting(t *testing.T) {
require.NoError(t, quick.Check(func(a []uint64) bool {
var txids []common.Txid
for _, txid := range a {
txids = append(txids, common.Txid(txid))
}

sort.Sort(txIDx(txids))

var r []uint64
for _, txid := range txids {
r = append(r, uint64(txid))
}

if !slices.IsSorted(r) {
t.Errorf("txids were not sorted correctly=%v", txids)
return false
}

return true
}, nil))
}

// Ensure that a freelist can deserialize from a freelist page.
func TestFreelist_read(t *testing.T) {
// Create a page.
Expand All @@ -243,6 +321,18 @@ func TestFreelist_read(t *testing.T) {
}
}

// Ensure that we never read a non-freelist page
func TestFreelist_read_panics(t *testing.T) {
buf := make([]byte, 4096)
page := common.LoadPage(buf)
page.SetFlags(common.BranchPageFlag)
page.SetCount(2)
f := newTestFreelist()
require.Panics(t, func() {
f.Read(page)
})
}

// Ensure that a freelist can serialize into a freelist page.
func TestFreelist_write(t *testing.T) {
// Create a freelist and write it to a page.
Expand All @@ -266,6 +356,250 @@ func TestFreelist_write(t *testing.T) {
}
}

func TestFreelist_E2E_HappyPath(t *testing.T) {
f := newTestFreelist()
f.Init([]common.Pgid{})
requirePages(t, f, common.Pgids{}, common.Pgids{})

allocated := f.Allocate(common.Txid(1), 5)
require.Equal(t, common.Pgid(0), allocated)
// tx.go may now allocate more space, and eventually we need to delete a page again
f.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 0))
f.Free(common.Txid(2), common.NewPage(3, common.LeafPageFlag, 0, 0))
f.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
// the above will only mark the pages as pending, so free pages should not return anything
requirePages(t, f, common.Pgids{}, common.Pgids{3, 5, 8})

// someone wants to do a read on top of the next tx id
f.AddReadonlyTXID(common.Txid(3))
// this should free the above pages for tx 2 entirely
f.ReleasePendingPages()
requirePages(t, f, common.Pgids{3, 5, 8}, common.Pgids{})

// no span of two pages available should yield a zero-page result
require.Equal(t, common.Pgid(0), f.Allocate(common.Txid(4), 2))
// we should be able to allocate those pages independently however,
// map and array differ in the order they return the pages
expectedPgids := map[common.Pgid]struct{}{3: {}, 5: {}, 8: {}}
for i := 0; i < 3; i++ {
allocated = f.Allocate(common.Txid(4), 1)
require.Contains(t, expectedPgids, allocated, "expected to find pgid %d", allocated)
require.False(t, f.Freed(allocated))
delete(expectedPgids, allocated)
}
require.Emptyf(t, expectedPgids, "unexpectedly more than one page was still found")
// no more free pages to allocate
require.Equal(t, common.Pgid(0), f.Allocate(common.Txid(4), 1))
}

func TestFreelist_E2E_MultiSpanOverflows(t *testing.T) {
f := newTestFreelist()
f.Init([]common.Pgid{})
f.Free(common.Txid(10), common.NewPage(20, common.LeafPageFlag, 0, 1))
f.Free(common.Txid(10), common.NewPage(25, common.LeafPageFlag, 0, 2))
f.Free(common.Txid(10), common.NewPage(35, common.LeafPageFlag, 0, 3))
f.Free(common.Txid(10), common.NewPage(39, common.LeafPageFlag, 0, 2))
f.Free(common.Txid(10), common.NewPage(45, common.LeafPageFlag, 0, 4))
requirePages(t, f, common.Pgids{}, common.Pgids{20, 21, 25, 26, 27, 35, 36, 37, 38, 39, 40, 41, 45, 46, 47, 48, 49})
f.ReleasePendingPages()
requirePages(t, f, common.Pgids{20, 21, 25, 26, 27, 35, 36, 37, 38, 39, 40, 41, 45, 46, 47, 48, 49}, common.Pgids{})

// that sequence, regardless of implementation, should always yield the same blocks of pages
allocSequence := []int{7, 5, 3, 2}
expectedSpanStarts := []common.Pgid{35, 45, 25, 20}
for i, pageNums := range allocSequence {
allocated := f.Allocate(common.Txid(11), pageNums)
require.Equal(t, expectedSpanStarts[i], allocated)
// ensure all pages in that span are not considered free anymore
for i := 0; i < pageNums; i++ {
require.False(t, f.Freed(allocated+common.Pgid(i)))
}
}
}

func TestFreelist_E2E_Rollbacks(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
requirePages(t, freelist, common.Pgids{}, common.Pgids{5, 6, 8})
freelist.Rollback(common.Txid(2))
requirePages(t, freelist, common.Pgids{}, common.Pgids{})

// unknown transaction should not trigger anything
freelist.Free(common.Txid(4), common.NewPage(13, common.LeafPageFlag, 0, 3))
requirePages(t, freelist, common.Pgids{}, common.Pgids{13, 14, 15, 16})
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{13, 14, 15, 16}, common.Pgids{})
freelist.Rollback(common.Txid(1337))
requirePages(t, freelist, common.Pgids{13, 14, 15, 16}, common.Pgids{})
}

func TestFreelist_E2E_RollbackPanics(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{5})
requirePages(t, freelist, common.Pgids{5}, common.Pgids{})

_ = freelist.Allocate(common.Txid(5), 1)
require.Panics(t, func() {
// depending on the verification level, either should panic
freelist.Free(common.Txid(5), common.NewPage(5, common.LeafPageFlag, 0, 0))
freelist.Rollback(5)
})
}

func TestFreelist_E2E_ReadOnlyTxTracking(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(10), common.NewPage(10, common.LeafPageFlag, 0, 2))
freelist.Free(common.Txid(11), common.NewPage(20, common.LeafPageFlag, 0, 2))
freelist.Free(common.Txid(12), common.NewPage(30, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{}, common.Pgids{10, 11, 12, 20, 21, 22, 30, 31, 32})

freelist.AddReadonlyTXID(11)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12}, common.Pgids{20, 21, 22, 30, 31, 32})

// this should be a no-op, as we still have a read TX open with id 11
freelist.AddReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12}, common.Pgids{20, 21, 22, 30, 31, 32})

// now 12 should be the latest
freelist.RemoveReadonlyTXID(11)
freelist.AddReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22}, common.Pgids{30, 31, 32})

// 12 was registered twice, so we also have to remove it twice to have an effect
freelist.RemoveReadonlyTXID(12)
freelist.AddReadonlyTXID(13)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22}, common.Pgids{30, 31, 32})

freelist.RemoveReadonlyTXID(12)
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{10, 11, 12, 20, 21, 22, 30, 31, 32}, common.Pgids{})
}

// tests the reloading from another physical page
func TestFreelist_E2E_Reload(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{})
buf := make([]byte, 4096)
p := common.LoadPage(buf)
freelist.Write(p)

freelist.Free(common.Txid(3), common.NewPage(3, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(3), common.NewPage(10, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{3, 4, 10, 11, 12})

otherBuf := make([]byte, 4096)
px := common.LoadPage(otherBuf)
freelist.Write(px)

loadFreeList := newTestFreelist()
loadFreeList.Init([]common.Pgid{})
loadFreeList.Read(px)
requirePages(t, loadFreeList, common.Pgids{3, 4, 5, 6, 8, 10, 11, 12}, common.Pgids{})
// restore the original freelist again
loadFreeList.Reload(p)
requirePages(t, loadFreeList, common.Pgids{5, 6, 8}, common.Pgids{})

// reload another page with different free pages to test we are deduplicating the free pages with the pending ones correctly
freelist = newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(5), common.NewPage(5, common.LeafPageFlag, 0, 4))
freelist.Reload(p)
requirePages(t, freelist, common.Pgids{}, common.Pgids{5, 6, 7, 8, 9})
}

// tests the loading and reloading from physical pages
func TestFreelist_E2E_SerDe_HappyPath(t *testing.T) {
freelist := newTestFreelist()
freelist.Init([]common.Pgid{})
freelist.Free(common.Txid(2), common.NewPage(5, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(2), common.NewPage(8, common.LeafPageFlag, 0, 0))
freelist.ReleasePendingPages()
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{})

freelist.Free(common.Txid(3), common.NewPage(3, common.LeafPageFlag, 0, 1))
freelist.Free(common.Txid(3), common.NewPage(10, common.LeafPageFlag, 0, 2))
requirePages(t, freelist, common.Pgids{5, 6, 8}, common.Pgids{3, 4, 10, 11, 12})

buf := make([]byte, 4096)
p := common.LoadPage(buf)
require.Equal(t, 80, freelist.EstimatedWritePageSize())
freelist.Write(p)

loadFreeList := newTestFreelist()
loadFreeList.Init([]common.Pgid{})
loadFreeList.Read(p)
requirePages(t, loadFreeList, common.Pgids{3, 4, 5, 6, 8, 10, 11, 12}, common.Pgids{})
}

// tests the loading of a freelist against other implementations with various sizes
func TestFreelist_E2E_SerDe_AcrossImplementations(t *testing.T) {
testSizes := []int{0, 1, 10, 100, 1000, math.MaxUint16, math.MaxUint16 + 1, math.MaxUint16 * 2}
for _, size := range testSizes {
t.Run(fmt.Sprintf("n=%d", size), func(t *testing.T) {
freelist := newTestFreelist()
expectedFreePgids := common.Pgids{}
for i := 0; i < size; i++ {
pgid := common.Pgid(i + 2)
freelist.Free(common.Txid(1), common.NewPage(pgid, common.LeafPageFlag, 0, 0))
expectedFreePgids = append(expectedFreePgids, pgid)
}
freelist.ReleasePendingPages()
requirePages(t, freelist, expectedFreePgids, common.Pgids{})
buf := make([]byte, freelist.EstimatedWritePageSize())
p := common.LoadPage(buf)
freelist.Write(p)

for n, loadFreeList := range map[string]Interface{
"hashmap": NewHashMapFreelist(),
"array": NewArrayFreelist(),
} {
t.Run(n, func(t *testing.T) {
loadFreeList.Read(p)
requirePages(t, loadFreeList, expectedFreePgids, common.Pgids{})
})
}
})
}
}

func requirePages(t *testing.T, f Interface, freePageIds common.Pgids, pendingPageIds common.Pgids) {
require.Equal(t, f.FreeCount()+f.PendingCount(), f.Count())
require.Equalf(t, freePageIds, f.freePageIds(), "unexpected free pages")
require.Equal(t, len(freePageIds), f.FreeCount())

pp := allPendingPages(f.pendingPageIds())
require.Equalf(t, pendingPageIds, pp, "unexpected pending pages")
require.Equal(t, len(pp), f.PendingCount())

for _, pgid := range f.freePageIds() {
require.Truef(t, f.Freed(pgid), "expected free page to return true on Freed")
}

for _, pgid := range pp {
require.Truef(t, f.Freed(pgid), "expected pending page to return true on Freed")
}
}

func allPendingPages(p map[common.Txid]*txPending) common.Pgids {
pgids := common.Pgids{}
for _, pending := range p {
pgids = append(pgids, pending.ids...)
}
sort.Sort(pgids)
return pgids
}

func Benchmark_FreelistRelease10K(b *testing.B) { benchmark_FreelistRelease(b, 10000) }
func Benchmark_FreelistRelease100K(b *testing.B) { benchmark_FreelistRelease(b, 100000) }
func Benchmark_FreelistRelease1000K(b *testing.B) { benchmark_FreelistRelease(b, 1000000) }
Expand Down
Loading

0 comments on commit e53f4c4

Please sign in to comment.