Skip to content

Commit

Permalink
sstable: clean up datadriven build commands
Browse files Browse the repository at this point in the history 
Clean up the datadriven build and build-raw commands uses by various sstable
unit tests. The build command confusingly had its own fragmenters for range
deletions and range keys sitting between the test instructions and the Writer
being tested.

These commands are updated so that the build instructions mirror what is
supplied to the writer.
  • Loading branch information
@jbowens
jbowens committed Aug 27, 2024
1 parent 6637383 commit 34e929e
Show file tree
Hide file tree
Showing 8 changed files with 100 additions and 148 deletions.
118 changes: 34 additions & 84 deletions sstable/data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,105 +68,55 @@ func optsFromArgs(td *datadriven.TestData, writerOpts *WriterOptions) error {
func runBuildCmd(
td *datadriven.TestData, writerOpts *WriterOptions, cacheSize int,
) (*WriterMetadata, *Reader, error) {

f0 := &objstorage.MemObj{}
if err := optsFromArgs(td, writerOpts); err != nil {
return nil, nil, err
}

w := NewWriter(f0, *writerOpts)
w := NewRawWriter(f0, *writerOpts)
defer func() {
if w != nil {
_ = w.Close()
}
}()
var rangeDels []keyspan.Span
rangeDelFrag := keyspan.Fragmenter{
Cmp: DefaultComparer.Compare,
Format: DefaultComparer.FormatKey,
Emit: func(s keyspan.Span) {
rangeDels = append(rangeDels, s)
},
}
var rangeKeys []keyspan.Span
rangeKeyFrag := keyspan.Fragmenter{
Cmp: DefaultComparer.Compare,
Format: DefaultComparer.FormatKey,
Emit: func(s keyspan.Span) {
rangeKeys = append(rangeKeys, s)
},
}
for _, data := range strings.Split(td.Input, "\n") {
if strings.HasPrefix(data, "rangekey:") {
var err error
func() {
defer func() {
if r := recover(); r != nil {
err = errors.Errorf("%v", r)
}
}()
rangeKeyFrag.Add(keyspan.ParseSpan(strings.TrimPrefix(data, "rangekey:")))
err := func() (err error) {
defer func() {
if r := recover(); r != nil {
err = errors.Errorf("%v", r)
}
}()
if err != nil {
return nil, nil, err
}
continue
}

forceObsolete := false
if strings.HasPrefix(data, "force-obsolete:") {
data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:"))
forceObsolete = true
}
j := strings.Index(data, ":")
key := base.ParseInternalKey(data[:j])
value := []byte(data[j+1:])
switch key.Kind() {
case InternalKeyKindRangeDelete:
if forceObsolete {
return nil, nil, errors.Errorf("force-obsolete is not allowed for RANGEDEL")
}
var err error
func() {
defer func() {
if r := recover(); r != nil {
err = errors.Errorf("%v", r)
switch {
case strings.HasPrefix(data, "EncodeSpan:"):
return w.EncodeSpan(keyspan.ParseSpan(strings.TrimPrefix(data, "EncodeSpan:")))
default:
forceObsolete := strings.HasPrefix(data, "force-obsolete:")
if forceObsolete {
data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:"))
}
j := strings.Index(data, ":")
key := base.ParseInternalKey(data[:j])
value := []byte(data[j+1:])
switch key.Kind() {
case InternalKeyKindRangeDelete:
if forceObsolete {
return errors.Errorf("force-obsolete is not allowed for RANGEDEL")
}
}()
rangeDelFrag.Add(keyspan.Span{
Start: key.UserKey,
End: value,
Keys: []keyspan.Key{{Trailer: key.Trailer}},
})
}()
if err != nil {
return nil, nil, err
}
default:
if err := w.Raw().AddWithForceObsolete(key, value, forceObsolete); err != nil {
return nil, nil, err
}
}
}
rangeDelFrag.Finish()
for _, v := range rangeDels {
for _, k := range v.Keys {
ik := base.InternalKey{UserKey: v.Start, Trailer: k.Trailer}
if err := w.Raw().AddWithForceObsolete(ik, v.End, false); err != nil {
return nil, nil, err
return w.AddWithForceObsolete(key, value, false /* forceObsolete */)
default:
return w.AddWithForceObsolete(key, value, forceObsolete)
}
}
}
}
rangeKeyFrag.Finish()
for _, s := range rangeKeys {
if err := w.addRangeKeySpanToFragmenter(s); err != nil {
}()
if err != nil {
return nil, nil, err
}
}
if err := w.Close(); err != nil {
return nil, nil, err
}
meta, err := w.Raw().Metadata()
meta, err := w.Metadata()
w = nil
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -209,16 +159,16 @@ func runBuildRawCmd(
return nil, nil, err
}

w := NewWriter(f0, *opts)
w := NewRawWriter(f0, *opts)
defer func() {
if w != nil {
_ = w.Close()
}
}()
for _, data := range strings.Split(td.Input, "\n") {
if strings.HasPrefix(data, "rangekey:") {
data = strings.TrimPrefix(data, "rangekey:")
if err := w.addRangeKeySpanToFragmenter(keyspan.ParseSpan(data)); err != nil {
if strings.HasPrefix(data, "EncodeSpan:") {
data = strings.TrimPrefix(data, "EncodeSpan:")
if err := w.EncodeSpan(keyspan.ParseSpan(data)); err != nil {
return nil, nil, err
}
continue
Expand All @@ -227,14 +177,14 @@ func runBuildRawCmd(
j := strings.Index(data, ":")
key := base.ParseInternalKey(data[:j])
value := []byte(data[j+1:])
if err := w.Raw().AddWithForceObsolete(key, value, false); err != nil {
if err := w.AddWithForceObsolete(key, value, false); err != nil {
return nil, nil, err
}
}
if err := w.Close(); err != nil {
return nil, nil, err
}
meta, err := w.Raw().Metadata()
meta, err := w.Metadata()
w = nil
if err != nil {
return nil, nil, err
Expand Down
24 changes: 12 additions & 12 deletions sstable/testdata/block_properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ build collectors=(suffix)
[email protected]:foo
[email protected]:bar
[email protected]:baz
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
----
point: [a@5#1,SET-c@15#3,SET]
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
Expand All @@ -135,9 +135,9 @@ build collectors=(suffix-point-keys-only)
[email protected]:foo
[email protected]:bar
[email protected]:baz
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
----
point: [a@5#1,SET-c@15#3,SET]
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
Expand All @@ -162,9 +162,9 @@ build collectors=(suffix-range-keys-only)
[email protected]:foo
[email protected]:bar
[email protected]:baz
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
----
point: [a@5#1,SET-c@15#3,SET]
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
Expand All @@ -190,9 +190,9 @@ build block-size=1 collectors=(suffix-point-keys-only,suffix-range-keys-only)
[email protected]:foo
[email protected]:bar
[email protected]:baz
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
----
point: [a@5#1,SET-c@15#3,SET]
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
Expand Down
6 changes: 3 additions & 3 deletions sstable/testdata/rewriter
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,9 @@ c
# Rewrite a table that contain only range keys.

build block-size=1 index-block-size=1 filter comparer-split-4b-suffix
rangekey: a-b:{(#1,RANGEKEYSET,_xyz)}
rangekey: b-c:{(#1,RANGEKEYSET,_xyz)}
rangekey: c-d:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)}
----
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
seqnums: [1-1]
Expand Down
6 changes: 3 additions & 3 deletions sstable/testdata/rewriter_v3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,9 @@ c
# Rewrite a table that contain only range keys.

build block-size=1 index-block-size=1 filter comparer-split-4b-suffix
rangekey: a-b:{(#1,RANGEKEYSET,_xyz)}
rangekey: b-c:{(#1,RANGEKEYSET,_xyz)}
rangekey: c-d:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)}
EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)}
----
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
seqnums: [1-1]
Expand Down
8 changes: 4 additions & 4 deletions sstable/testdata/virtual_reader_iter
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ a.SET.1:a
d.SET.2:d
f.SET.3:f
d.RANGEDEL.4:e
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
g.RANGEDEL.5:l
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
----
point: [a#1,SET-f#3,SET]
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]
Expand Down Expand Up @@ -569,9 +569,9 @@ b.SET.5:b
d.SET.2:d
f.SET.3:f
d.RANGEDEL.4:e
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
g.RANGEDEL.5:l
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
----
point: [a#1,SET-f#3,SET]
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]
Expand Down
4 changes: 2 additions & 2 deletions sstable/testdata/virtual_reader_props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,9 @@ a.SET.1:a
d.SET.2:d
f.SET.3:f
d.RANGEDEL.4:e
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
g.RANGEDEL.5:l
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
----
point: [a#1,SET-f#3,SET]
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]
Expand Down
45 changes: 23 additions & 22 deletions sstable/testdata/writer
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ f.SET.5:f
g.DEL.6:
h.MERGE.7:h
i.RANGEDEL.8:j
rangekey: j-k:{(#9,RANGEKEYDEL)}
rangekey: k-l:{(#10,RANGEKEYUNSET,@5)}
rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)}
EncodeSpan: j-k:{(#9,RANGEKEYDEL)}
EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)}
EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)}
----
point: [a#1,SET-h#7,MERGE]
rangedel: [d#4,RANGEDEL-j#inf,RANGEDEL]
Expand All @@ -46,9 +46,9 @@ e.SINGLEDEL.5:
f.SET.6:f
g.DEL.7:
h.SINGLEDEL.8:
rangekey: j-k:{(#9,RANGEKEYDEL)}
rangekey: k-l:{(#10,RANGEKEYUNSET,@5)}
rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)}
EncodeSpan: j-k:{(#9,RANGEKEYDEL)}
EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)}
EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)}
----
point: [a#1,SET-h#8,SINGLEDEL]
rangekey: [j#9,RANGEKEYDEL-m#inf,RANGEKEYSET]
Expand Down Expand Up @@ -91,9 +91,11 @@ i-j:{(#8,RANGEDEL)}
# 1: j---------------z

build
a.RANGEDEL.3:m
f.RANGEDEL.2:s
j.RANGEDEL.1:z
EncodeSpan: a-f:{(#3,RANGEDEL)}
EncodeSpan: f-j:{(#3,RANGEDEL) (#2,RANGEDEL)}
EncodeSpan: j-m:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)}
EncodeSpan: m-s:{(#2,RANGEDEL) (#1,RANGEDEL)}
EncodeSpan: s-z:{(#1,RANGEDEL)}
----
rangedel: [a#3,RANGEDEL-z#inf,RANGEDEL]
seqnums: [1-3]
Expand Down Expand Up @@ -157,7 +159,7 @@ build
b.RANGEDEL.1:c
a.RANGEDEL.2:b
----
pebble: keys must be added in order: b > a
pebble: keys must be added in order: b#1,RANGEDEL, a#2,RANGEDEL

build-raw
.RANGEDEL.1:b
Expand Down Expand Up @@ -190,22 +192,21 @@ c.RANGEDEL.2:d
rangedel: [a#1,RANGEDEL-d#inf,RANGEDEL]
seqnums: [1-2]

build-raw
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)}
rangekey: a-b:{(#2,RANGEKEYSET,@10,foo)}
build
EncodeSpan: a-b:{(#2,RANGEKEYSET,@10,foo) (#1,RANGEKEYSET,@10,foo)}
----
rangekey: [a#2,RANGEKEYSET-b#inf,RANGEKEYSET]
seqnums: [1-2]

build-raw
rangekey: b-c:{(#2,RANGEKEYSET,@10,foo)}
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)}
EncodeSpan: b-c:{(#2,RANGEKEYSET,@10,foo)}
EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo)}
----
pebble: spans must be added in order: b > a
pebble: range keys starts must be added in increasing order: b#2,RANGEKEYSET, a#1,RANGEKEYSET

build-raw
rangekey: a-c:{(#1,RANGEKEYSET,@10,foo)}
rangekey: c-d:{(#2,RANGEKEYSET,@10,foo)}
EncodeSpan: a-c:{(#1,RANGEKEYSET,@10,foo)}
EncodeSpan: c-d:{(#2,RANGEKEYSET,@10,foo)}
----
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
seqnums: [1-2]
Expand All @@ -214,7 +215,7 @@ seqnums: [1-2]
# though the key kinds must be ordered (descending).

build-raw
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)}
EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)}
----
rangekey: [a#1,RANGEKEYSET-b#inf,RANGEKEYDEL]
seqnums: [1-1]
Expand Down Expand Up @@ -274,9 +275,9 @@ layout
# Range keys, if present, are shown in the layout.

build
rangekey: a-b:{(#3,RANGEKEYSET,@3,foo)}
rangekey: b-c:{(#2,RANGEKEYSET,@2,bar)}
rangekey: c-d:{(#1,RANGEKEYSET,@1,baz)}
EncodeSpan: a-b:{(#3,RANGEKEYSET,@3,foo)}
EncodeSpan: b-c:{(#2,RANGEKEYSET,@2,bar)}
EncodeSpan: c-d:{(#1,RANGEKEYSET,@1,baz)}
----
rangekey: [a#3,RANGEKEYSET-d#inf,RANGEKEYSET]
seqnums: [1-3]
Expand Down
Loading

0 comments on commit 34e929e

Please sign in to comment.