diff --git a/sstable/data_test.go b/sstable/data_test.go index 20f57353a7..864751ea4d 100644 --- a/sstable/data_test.go +++ b/sstable/data_test.go @@ -68,105 +68,55 @@ func optsFromArgs(td *datadriven.TestData, writerOpts *WriterOptions) error { func runBuildCmd( td *datadriven.TestData, writerOpts *WriterOptions, cacheSize int, ) (*WriterMetadata, *Reader, error) { - f0 := &objstorage.MemObj{} if err := optsFromArgs(td, writerOpts); err != nil { return nil, nil, err } - w := NewWriter(f0, *writerOpts) + w := NewRawWriter(f0, *writerOpts) defer func() { if w != nil { _ = w.Close() } }() - var rangeDels []keyspan.Span - rangeDelFrag := keyspan.Fragmenter{ - Cmp: DefaultComparer.Compare, - Format: DefaultComparer.FormatKey, - Emit: func(s keyspan.Span) { - rangeDels = append(rangeDels, s) - }, - } - var rangeKeys []keyspan.Span - rangeKeyFrag := keyspan.Fragmenter{ - Cmp: DefaultComparer.Compare, - Format: DefaultComparer.FormatKey, - Emit: func(s keyspan.Span) { - rangeKeys = append(rangeKeys, s) - }, - } for _, data := range strings.Split(td.Input, "\n") { - if strings.HasPrefix(data, "rangekey:") { - var err error - func() { - defer func() { - if r := recover(); r != nil { - err = errors.Errorf("%v", r) - } - }() - rangeKeyFrag.Add(keyspan.ParseSpan(strings.TrimPrefix(data, "rangekey:"))) + err := func() (err error) { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("%v", r) + } }() - if err != nil { - return nil, nil, err - } - continue - } - forceObsolete := false - if strings.HasPrefix(data, "force-obsolete:") { - data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:")) - forceObsolete = true - } - j := strings.Index(data, ":") - key := base.ParseInternalKey(data[:j]) - value := []byte(data[j+1:]) - switch key.Kind() { - case InternalKeyKindRangeDelete: - if forceObsolete { - return nil, nil, errors.Errorf("force-obsolete is not allowed for RANGEDEL") - } - var err error - func() { - defer func() { - if r := recover(); r != nil { - err = errors.Errorf("%v", r) + switch { + case strings.HasPrefix(data, "EncodeSpan:"): + return w.EncodeSpan(keyspan.ParseSpan(strings.TrimPrefix(data, "EncodeSpan:"))) + default: + forceObsolete := strings.HasPrefix(data, "force-obsolete:") + if forceObsolete { + data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:")) + } + j := strings.Index(data, ":") + key := base.ParseInternalKey(data[:j]) + value := []byte(data[j+1:]) + switch key.Kind() { + case InternalKeyKindRangeDelete: + if forceObsolete { + return errors.Errorf("force-obsolete is not allowed for RANGEDEL") } - }() - rangeDelFrag.Add(keyspan.Span{ - Start: key.UserKey, - End: value, - Keys: []keyspan.Key{{Trailer: key.Trailer}}, - }) - }() - if err != nil { - return nil, nil, err - } - default: - if err := w.Raw().AddWithForceObsolete(key, value, forceObsolete); err != nil { - return nil, nil, err - } - } - } - rangeDelFrag.Finish() - for _, v := range rangeDels { - for _, k := range v.Keys { - ik := base.InternalKey{UserKey: v.Start, Trailer: k.Trailer} - if err := w.Raw().AddWithForceObsolete(ik, v.End, false); err != nil { - return nil, nil, err + return w.AddWithForceObsolete(key, value, false /* forceObsolete */) + default: + return w.AddWithForceObsolete(key, value, forceObsolete) + } } - } - } - rangeKeyFrag.Finish() - for _, s := range rangeKeys { - if err := w.addRangeKeySpanToFragmenter(s); err != nil { + }() + if err != nil { return nil, nil, err } } if err := w.Close(); err != nil { return nil, nil, err } - meta, err := w.Raw().Metadata() + meta, err := w.Metadata() w = nil if err != nil { return nil, nil, err @@ -209,16 +159,16 @@ func runBuildRawCmd( return nil, nil, err } - w := NewWriter(f0, *opts) + w := NewRawWriter(f0, *opts) defer func() { if w != nil { _ = w.Close() } }() for _, data := range strings.Split(td.Input, "\n") { - if strings.HasPrefix(data, "rangekey:") { - data = strings.TrimPrefix(data, "rangekey:") - if err := w.addRangeKeySpanToFragmenter(keyspan.ParseSpan(data)); err != nil { + if strings.HasPrefix(data, "EncodeSpan:") { + data = strings.TrimPrefix(data, "EncodeSpan:") + if err := w.EncodeSpan(keyspan.ParseSpan(data)); err != nil { return nil, nil, err } continue @@ -227,14 +177,14 @@ func runBuildRawCmd( j := strings.Index(data, ":") key := base.ParseInternalKey(data[:j]) value := []byte(data[j+1:]) - if err := w.Raw().AddWithForceObsolete(key, value, false); err != nil { + if err := w.AddWithForceObsolete(key, value, false); err != nil { return nil, nil, err } } if err := w.Close(); err != nil { return nil, nil, err } - meta, err := w.Raw().Metadata() + meta, err := w.Metadata() w = nil if err != nil { return nil, nil, err diff --git a/sstable/testdata/block_properties b/sstable/testdata/block_properties index 16040b3860..7173df8b92 100644 --- a/sstable/testdata/block_properties +++ b/sstable/testdata/block_properties @@ -108,9 +108,9 @@ build collectors=(suffix) a@5.SET.1:foo b@10.SET.2:bar c@15.SET.3:baz -rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} -rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} -rangekey: f@20-z@25:{(#6,RANGEKEYDEL)} +EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} +EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} +EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)} ---- point: [a@5#1,SET-c@15#3,SET] rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL] @@ -135,9 +135,9 @@ build collectors=(suffix-point-keys-only) a@5.SET.1:foo b@10.SET.2:bar c@15.SET.3:baz -rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} -rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} -rangekey: f@20-z@25:{(#6,RANGEKEYDEL)} +EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} +EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} +EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)} ---- point: [a@5#1,SET-c@15#3,SET] rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL] @@ -162,9 +162,9 @@ build collectors=(suffix-range-keys-only) a@5.SET.1:foo b@10.SET.2:bar c@15.SET.3:baz -rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} -rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} -rangekey: f@20-z@25:{(#6,RANGEKEYDEL)} +EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} +EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} +EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)} ---- point: [a@5#1,SET-c@15#3,SET] rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL] @@ -190,9 +190,9 @@ build block-size=1 collectors=(suffix-point-keys-only,suffix-range-keys-only) a@5.SET.1:foo b@10.SET.2:bar c@15.SET.3:baz -rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} -rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} -rangekey: f@20-z@25:{(#6,RANGEKEYDEL)} +EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)} +EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)} +EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)} ---- point: [a@5#1,SET-c@15#3,SET] rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL] diff --git a/sstable/testdata/rewriter b/sstable/testdata/rewriter index ddc2c44cbc..22374d2ab8 100644 --- a/sstable/testdata/rewriter +++ b/sstable/testdata/rewriter @@ -212,9 +212,9 @@ c # Rewrite a table that contain only range keys. build block-size=1 index-block-size=1 filter comparer-split-4b-suffix -rangekey: a-b:{(#1,RANGEKEYSET,_xyz)} -rangekey: b-c:{(#1,RANGEKEYSET,_xyz)} -rangekey: c-d:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)} ---- rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-1] diff --git a/sstable/testdata/rewriter_v3 b/sstable/testdata/rewriter_v3 index fc8b8a48c3..3c1dfbbb13 100644 --- a/sstable/testdata/rewriter_v3 +++ b/sstable/testdata/rewriter_v3 @@ -212,9 +212,9 @@ c # Rewrite a table that contain only range keys. build block-size=1 index-block-size=1 filter comparer-split-4b-suffix -rangekey: a-b:{(#1,RANGEKEYSET,_xyz)} -rangekey: b-c:{(#1,RANGEKEYSET,_xyz)} -rangekey: c-d:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)} +EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)} ---- rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-1] diff --git a/sstable/testdata/virtual_reader_iter b/sstable/testdata/virtual_reader_iter index 1c23a66bbb..8916ab140c 100644 --- a/sstable/testdata/virtual_reader_iter +++ b/sstable/testdata/virtual_reader_iter @@ -32,9 +32,9 @@ a.SET.1:a d.SET.2:d f.SET.3:f d.RANGEDEL.4:e -rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)} g.RANGEDEL.5:l -rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)} +EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)} ---- point: [a#1,SET-f#3,SET] rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL] @@ -569,9 +569,9 @@ b.SET.5:b d.SET.2:d f.SET.3:f d.RANGEDEL.4:e -rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)} g.RANGEDEL.5:l -rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)} +EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)} ---- point: [a#1,SET-f#3,SET] rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL] diff --git a/sstable/testdata/virtual_reader_props b/sstable/testdata/virtual_reader_props index 7d041960fa..10b1b9b753 100644 --- a/sstable/testdata/virtual_reader_props +++ b/sstable/testdata/virtual_reader_props @@ -101,9 +101,9 @@ a.SET.1:a d.SET.2:d f.SET.3:f d.RANGEDEL.4:e -rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)} g.RANGEDEL.5:l -rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)} +EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)} ---- point: [a#1,SET-f#3,SET] rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL] diff --git a/sstable/testdata/writer b/sstable/testdata/writer index 24417dd1ce..93b1099ef6 100644 --- a/sstable/testdata/writer +++ b/sstable/testdata/writer @@ -23,9 +23,9 @@ f.SET.5:f g.DEL.6: h.MERGE.7:h i.RANGEDEL.8:j -rangekey: j-k:{(#9,RANGEKEYDEL)} -rangekey: k-l:{(#10,RANGEKEYUNSET,@5)} -rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: j-k:{(#9,RANGEKEYDEL)} +EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)} +EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)} ---- point: [a#1,SET-h#7,MERGE] rangedel: [d#4,RANGEDEL-j#inf,RANGEDEL] @@ -46,9 +46,9 @@ e.SINGLEDEL.5: f.SET.6:f g.DEL.7: h.SINGLEDEL.8: -rangekey: j-k:{(#9,RANGEKEYDEL)} -rangekey: k-l:{(#10,RANGEKEYUNSET,@5)} -rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: j-k:{(#9,RANGEKEYDEL)} +EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)} +EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)} ---- point: [a#1,SET-h#8,SINGLEDEL] rangekey: [j#9,RANGEKEYDEL-m#inf,RANGEKEYSET] @@ -91,9 +91,11 @@ i-j:{(#8,RANGEDEL)} # 1: j---------------z build -a.RANGEDEL.3:m -f.RANGEDEL.2:s -j.RANGEDEL.1:z +EncodeSpan: a-f:{(#3,RANGEDEL)} +EncodeSpan: f-j:{(#3,RANGEDEL) (#2,RANGEDEL)} +EncodeSpan: j-m:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)} +EncodeSpan: m-s:{(#2,RANGEDEL) (#1,RANGEDEL)} +EncodeSpan: s-z:{(#1,RANGEDEL)} ---- rangedel: [a#3,RANGEDEL-z#inf,RANGEDEL] seqnums: [1-3] @@ -157,7 +159,7 @@ build b.RANGEDEL.1:c a.RANGEDEL.2:b ---- -pebble: keys must be added in order: b > a +pebble: keys must be added in order: b#1,RANGEDEL, a#2,RANGEDEL build-raw .RANGEDEL.1:b @@ -190,22 +192,21 @@ c.RANGEDEL.2:d rangedel: [a#1,RANGEDEL-d#inf,RANGEDEL] seqnums: [1-2] -build-raw -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)} -rangekey: a-b:{(#2,RANGEKEYSET,@10,foo)} +build +EncodeSpan: a-b:{(#2,RANGEKEYSET,@10,foo) (#1,RANGEKEYSET,@10,foo)} ---- rangekey: [a#2,RANGEKEYSET-b#inf,RANGEKEYSET] seqnums: [1-2] build-raw -rangekey: b-c:{(#2,RANGEKEYSET,@10,foo)} -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)} +EncodeSpan: b-c:{(#2,RANGEKEYSET,@10,foo)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo)} ---- -pebble: spans must be added in order: b > a +pebble: range keys starts must be added in increasing order: b#2,RANGEKEYSET, a#1,RANGEKEYSET build-raw -rangekey: a-c:{(#1,RANGEKEYSET,@10,foo)} -rangekey: c-d:{(#2,RANGEKEYSET,@10,foo)} +EncodeSpan: a-c:{(#1,RANGEKEYSET,@10,foo)} +EncodeSpan: c-d:{(#2,RANGEKEYSET,@10,foo)} ---- rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-2] @@ -214,7 +215,7 @@ seqnums: [1-2] # though the key kinds must be ordered (descending). build-raw -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)} ---- rangekey: [a#1,RANGEKEYSET-b#inf,RANGEKEYDEL] seqnums: [1-1] @@ -274,9 +275,9 @@ layout # Range keys, if present, are shown in the layout. build -rangekey: a-b:{(#3,RANGEKEYSET,@3,foo)} -rangekey: b-c:{(#2,RANGEKEYSET,@2,bar)} -rangekey: c-d:{(#1,RANGEKEYSET,@1,baz)} +EncodeSpan: a-b:{(#3,RANGEKEYSET,@3,foo)} +EncodeSpan: b-c:{(#2,RANGEKEYSET,@2,bar)} +EncodeSpan: c-d:{(#1,RANGEKEYSET,@1,baz)} ---- rangekey: [a#3,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-3] diff --git a/sstable/testdata/writer_v3 b/sstable/testdata/writer_v3 index d7216b39a2..48cd1920f1 100644 --- a/sstable/testdata/writer_v3 +++ b/sstable/testdata/writer_v3 @@ -23,9 +23,9 @@ f.SET.5:f g.DEL.6: h.MERGE.7:h i.RANGEDEL.8:j -rangekey: j-k:{(#9,RANGEKEYDEL)} -rangekey: k-l:{(#10,RANGEKEYUNSET,@5)} -rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)} +EncodeSpan: j-k:{(#9,RANGEKEYDEL)} +EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)} +EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)} ---- point: [a#1,SET-h#7,MERGE] rangedel: [d#4,RANGEDEL-j#inf,RANGEDEL] @@ -65,9 +65,11 @@ i-j:{(#8,RANGEDEL)} # 1: j---------------z build -a.RANGEDEL.3:m -f.RANGEDEL.2:s -j.RANGEDEL.1:z +EncodeSpan: a-f:{(#3,RANGEDEL)} +EncodeSpan: f-j:{(#3,RANGEDEL) (#2,RANGEDEL)} +EncodeSpan: j-m:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)} +EncodeSpan: m-s:{(#2,RANGEDEL) (#1,RANGEDEL)} +EncodeSpan: s-z:{(#1,RANGEDEL)} ---- rangedel: [a#3,RANGEDEL-z#inf,RANGEDEL] seqnums: [1-3] @@ -131,7 +133,7 @@ build b.RANGEDEL.1:c a.RANGEDEL.2:b ---- -pebble: keys must be added in order: b > a +pebble: keys must be added in order: b#1,RANGEDEL, a#2,RANGEDEL build-raw .RANGEDEL.1:b @@ -165,21 +167,20 @@ rangedel: [a#1,RANGEDEL-d#inf,RANGEDEL] seqnums: [1-2] build-raw -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)} -rangekey: a-b:{(#2,RANGEKEYSET,@10,foo)} +EncodeSpan: a-b:{(#2,RANGEKEYSET,@10,foo) (#1,RANGEKEYSET,@10,foo)} ---- rangekey: [a#2,RANGEKEYSET-b#inf,RANGEKEYSET] seqnums: [1-2] build-raw -rangekey: b-c:{(#2,RANGEKEYSET,@10,foo)} -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)} +EncodeSpan: b-c:{(#2,RANGEKEYSET,@10,foo)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo)} ---- -pebble: spans must be added in order: b > a +pebble: range keys starts must be added in increasing order: b#2,RANGEKEYSET, a#1,RANGEKEYSET build-raw -rangekey: a-c:{(#1,RANGEKEYSET,@10,foo)} -rangekey: c-d:{(#2,RANGEKEYSET,@10,foo)} +EncodeSpan: a-c:{(#1,RANGEKEYSET,@10,foo)} +EncodeSpan: c-d:{(#2,RANGEKEYSET,@10,foo)} ---- rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-2] @@ -188,7 +189,7 @@ seqnums: [1-2] # though the key kinds must be ordered (descending). build-raw -rangekey: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@10) (#1,RANGEKEYDEL)} +EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@10) (#1,RANGEKEYDEL)} ---- rangekey: [a#1,RANGEKEYSET-b#inf,RANGEKEYDEL] seqnums: [1-1] @@ -248,9 +249,9 @@ layout # Range keys, if present, are shown in the layout. build -rangekey: a-b:{(#3,RANGEKEYSET,@3,foo)} -rangekey: b-c:{(#2,RANGEKEYSET,@2,bar)} -rangekey: c-d:{(#1,RANGEKEYSET,@1,baz)} +EncodeSpan: a-b:{(#3,RANGEKEYSET,@3,foo)} +EncodeSpan: b-c:{(#2,RANGEKEYSET,@2,bar)} +EncodeSpan: c-d:{(#1,RANGEKEYSET,@1,baz)} ---- rangekey: [a#3,RANGEKEYSET-d#inf,RANGEKEYSET] seqnums: [1-3]