From 85a116fccf4ccc2ae0c30be306f5a3392d426b94 Mon Sep 17 00:00:00 2001 From: Andy Yang Date: Thu, 3 Oct 2024 15:20:31 -0400 Subject: [PATCH 1/2] changefeedccl: enable vmodule for TestAlterChangefeedSwitchFamily This patch enables verbose logging (vmodule `helpers_test=1`) for `TestAlterChangefeedSwitchFamily` so that we can get more information about what, if any, messages that the changefeed sees before the test times out. Release note: None --- pkg/ccl/changefeedccl/alter_changefeed_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/ccl/changefeedccl/alter_changefeed_test.go b/pkg/ccl/changefeedccl/alter_changefeed_test.go index e1243c9db9eb..3e33dfbf7567 100644 --- a/pkg/ccl/changefeedccl/alter_changefeed_test.go +++ b/pkg/ccl/changefeedccl/alter_changefeed_test.go @@ -278,6 +278,8 @@ func TestAlterChangefeedSwitchFamily(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + require.NoError(t, log.SetVModule("helpers_test=1")) + testFn := func(t *testing.T, s TestServer, f cdctest.TestFeedFactory) { sqlDB := sqlutils.MakeSQLRunner(s.DB) sqlDB.Exec(t, `CREATE TABLE foo (a INT PRIMARY KEY, b STRING, FAMILY onlya (a), FAMILY onlyb (b))`) From abfe3879691e861b2115199076bd0cf5d2490e0e Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Mon, 21 Oct 2024 12:27:07 -0400 Subject: [PATCH 2/2] storage: add TestKeySchema_RandomKeys Add a randomized test that constructs random keys, writes them to a Pebble columnar block, iterates over the block and asserts that the keys are semantically equal to the original generated keys. Epic: none Release note: none --- pkg/storage/engine_key_test.go | 23 +++++++++++ pkg/storage/pebble_key_schema_test.go | 48 ++++++++++++++++++++++ pkg/storage/testdata/key_schema_key_seeker | 22 ++++++++++ 3 files changed, 93 insertions(+) diff --git a/pkg/storage/engine_key_test.go b/pkg/storage/engine_key_test.go index 87f0e018f796..6077e5f08df9 100644 --- a/pkg/storage/engine_key_test.go +++ b/pkg/storage/engine_key_test.go @@ -489,3 +489,26 @@ func engineKey(key string, ts int) EngineKey { Version: encodeMVCCTimestamp(wallTS(ts)), } } + +var possibleVersionLens = []int{ + engineKeyNoVersion, + engineKeyVersionWallTimeLen, + engineKeyVersionWallAndLogicalTimeLen, + engineKeyVersionWallLogicalAndSyntheticTimeLen, + engineKeyVersionLockTableLen, +} + +func randomSerializedEngineKey(r *rand.Rand, maxUserKeyLen int) []byte { + userKeyLen := randutil.RandIntInRange(r, 1, maxUserKeyLen) + versionLen := possibleVersionLens[r.Intn(len(possibleVersionLens))] + serializedLen := userKeyLen + versionLen + 1 + if versionLen > 0 { + serializedLen++ // sentinel + } + k := randutil.RandBytes(r, serializedLen) + k[userKeyLen] = 0x00 + if versionLen > 0 { + k[len(k)-1] = byte(versionLen + 1) + } + return k +} diff --git a/pkg/storage/pebble_key_schema_test.go b/pkg/storage/pebble_key_schema_test.go index 57c917d3a801..7c5800e669af 100644 --- a/pkg/storage/pebble_key_schema_test.go +++ b/pkg/storage/pebble_key_schema_test.go @@ -10,6 +10,7 @@ import ( "encoding/hex" "fmt" "math/rand" + "slices" "strconv" "strings" "testing" @@ -19,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/randutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/crlib/crbytes" "github.com/cockroachdb/crlib/crstrings" @@ -28,6 +30,7 @@ import ( "github.com/cockroachdb/pebble/sstable/block" "github.com/cockroachdb/pebble/sstable/colblk" "github.com/olekukonko/tablewriter" + "github.com/stretchr/testify/require" ) func TestKeySchema_KeyWriter(t *testing.T) { @@ -312,3 +315,48 @@ func parseTestKey(s string) ([]byte, error) { }), nil } } + +func TestKeySchema_RandomKeys(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + rng, _ := randutil.NewTestRand() + maxUserKeyLen := randutil.RandIntInRange(rng, 2, 10) + keys := make([][]byte, randutil.RandIntInRange(rng, 1, 1000)) + for i := range keys { + keys[i] = randomSerializedEngineKey(rng, maxUserKeyLen) + } + slices.SortFunc(keys, EngineKeyCompare) + + var enc colblk.DataBlockEncoder + enc.Init(keySchema) + for i := range keys { + ikey := pebble.InternalKey{ + UserKey: keys[i], + Trailer: pebble.MakeInternalKeyTrailer(0, pebble.InternalKeyKindSet), + } + enc.Add(ikey, keys[i], block.InPlaceValuePrefix(false), enc.KeyWriter.ComparePrev(keys[i]), false /* isObsolete */) + } + blk, _ := enc.Finish(len(keys), enc.Size()) + blk = crbytes.CopyAligned(blk) + + var dec colblk.DataBlockDecoder + dec.Init(keySchema, blk) + var it colblk.DataBlockIter + it.InitOnce(keySchema, EngineKeyCompare, EngineKeySplit, nil) + require.NoError(t, it.Init(&dec, block.NoTransforms)) + for k, kv := 0, it.First(); kv != nil; k, kv = k+1, it.Next() { + require.True(t, EngineKeyEqual(keys[k], kv.K.UserKey)) + require.Zero(t, EngineKeyCompare(keys[k], kv.K.UserKey)) + // Note we allow the key read from the block to be physically different, + // because the above randomization generates point keys with the + // synthetic bit encoding. However the materialized key should not be + // longer than the original key, because we depend on the max key length + // during writing bounding the key length during reading. + if n := len(kv.K.UserKey); n > len(keys[k]) { + t.Fatalf("key %q is longer than original key %q", kv.K.UserKey, keys[k]) + } + checkEngineKey(kv.K.UserKey) + } + require.NoError(t, it.Close()) +} diff --git a/pkg/storage/testdata/key_schema_key_seeker b/pkg/storage/testdata/key_schema_key_seeker index 039eb4557ab0..09ffe8fbc18c 100644 --- a/pkg/storage/testdata/key_schema_key_seeker +++ b/pkg/storage/testdata/key_schema_key_seeker @@ -147,3 +147,25 @@ MaterializeUserKey(-1, 3) = hex:6d6f6f0000000000b2d05e00000000010d MaterializeUserKey(3, 2) = hex:666f6f0000000000b2d05e00000000010d MaterializeUserKey(2, 0) = hex:6261720000000000b2d05e00000000010d MaterializeUserKey(0, 1) = hex:6261780000000000b2d05e00000000010d + +define-block +moo@3.000000001,0 +moo@3.000000000,2 +moo@3.000000000,1 +moo@3.000000000,0 +---- +Parse("moo@3.000000001,0") = hex:6d6f6f0000000000b2d05e0109 +Parse("moo@3.000000000,2") = hex:6d6f6f0000000000b2d05e00000000020d +Parse("moo@3.000000000,1") = hex:6d6f6f0000000000b2d05e00000000010d +Parse("moo@3.000000000,0") = hex:6d6f6f0000000000b2d05e0009 + +materialize-user-key +0 +1 +2 +3 +---- +MaterializeUserKey(-1, 0) = hex:6d6f6f0000000000b2d05e0109 +MaterializeUserKey(0, 1) = hex:6d6f6f0000000000b2d05e00000000020d +MaterializeUserKey(1, 2) = hex:6d6f6f0000000000b2d05e00000000010d +MaterializeUserKey(2, 3) = hex:6d6f6f0000000000b2d05e0009