ceremonyclient/pebble/internal/keyspan/level_iter_test.go
Cassandra Heart 2e2a1e4789
v1.2.0 (#31)
2024-01-03 01:31:42 -06:00

473 lines
13 KiB
Go

// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"strings"
"testing"
"github.com/cockroachdb/datadriven"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/stretchr/testify/require"
)
func TestLevelIterEquivalence(t *testing.T) {
type level [][]Span
testCases := []struct {
name string
levels []level
}{
{
"single level, no gaps, no overlaps",
[]level{
{
{
Span{
Start: []byte("a"),
End: []byte("b"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("b"),
End: []byte("c"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("c"),
End: []byte("d"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
{
Span{
Start: []byte("d"),
End: []byte("e"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("e"),
End: []byte("f"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("f"),
End: []byte("g"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
},
},
},
{
"single level, overlapping fragments",
[]level{
{
{
Span{
Start: []byte("a"),
End: []byte("b"),
Keys: []Key{
{
Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("bar"),
},
{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
},
},
},
Span{
Start: []byte("b"),
End: []byte("c"),
Keys: []Key{
{
Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("bar"),
},
{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
},
},
},
Span{
Start: []byte("c"),
End: []byte("d"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
{
Span{
Start: []byte("d"),
End: []byte("e"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("e"),
End: []byte("f"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("f"),
End: []byte("g"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
},
},
},
{
"single level, gaps between files and range keys",
[]level{
{
{
Span{
Start: []byte("a"),
End: []byte("b"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("c"),
End: []byte("d"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("e"),
End: []byte("f"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
{
Span{
Start: []byte("g"),
End: []byte("h"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("i"),
End: []byte("j"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
Span{
Start: []byte("k"),
End: []byte("l"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
},
},
},
{
"two levels, one with overlapping unset",
[]level{
{
{
Span{
Start: []byte("a"),
End: []byte("h"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
{
Span{
Start: []byte("l"),
End: []byte("u"),
Keys: []Key{{
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeyUnset),
Suffix: nil,
Value: nil,
}},
},
},
},
{
{
Span{
Start: []byte("e"),
End: []byte("r"),
Keys: []Key{{
Trailer: base.MakeTrailer(1, base.InternalKeyKindRangeKeySet),
Suffix: nil,
Value: []byte("foo"),
}},
},
},
},
},
},
}
for _, tc := range testCases {
var fileIters []FragmentIterator
var levelIters []FragmentIterator
var iter1, iter2 MergingIter
for j, level := range tc.levels {
j := j // Copy for use in closures down below.
var levelIter LevelIter
var metas []*manifest.FileMetadata
for k, file := range level {
fileIters = append(fileIters, NewIter(base.DefaultComparer.Compare, file))
meta := &manifest.FileMetadata{
FileNum: base.FileNum(k + 1),
Size: 1024,
SmallestSeqNum: 2,
LargestSeqNum: 2,
SmallestRangeKey: base.MakeInternalKey(file[0].Start, file[0].SmallestKey().SeqNum(), file[0].SmallestKey().Kind()),
LargestRangeKey: base.MakeExclusiveSentinelKey(file[len(file)-1].LargestKey().Kind(), file[len(file)-1].End),
HasPointKeys: false,
HasRangeKeys: true,
}
meta.InitPhysicalBacking()
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, meta.SmallestRangeKey, meta.LargestRangeKey)
metas = append(metas, meta)
}
tableNewIters := func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error) {
return NewIter(base.DefaultComparer.Compare, tc.levels[j][file.FileNum-1]), nil
}
// Add all the fileMetadatas to L6.
b := &manifest.BulkVersionEdit{}
amap := make(map[base.FileNum]*manifest.FileMetadata)
for i := range metas {
amap[metas[i].FileNum] = metas[i]
}
b.Added[6] = amap
v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys)
require.NoError(t, err)
levelIter.Init(
SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters,
v.Levels[6].Iter(), 0, manifest.KeyTypeRange,
)
levelIters = append(levelIters, &levelIter)
}
iter1.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), fileIters...)
iter2.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), levelIters...)
// Check iter1 and iter2 for equivalence.
require.Equal(t, iter1.First(), iter2.First(), "failed on test case %q", tc.name)
valid := true
for valid {
f1 := iter1.Next()
var f2 *Span
for {
f2 = iter2.Next()
// The level iter could produce empty spans that straddle between
// files. Ignore those.
if f2 == nil || !f2.Empty() {
break
}
}
require.Equal(t, f1, f2, "failed on test case %q", tc.name)
valid = f1 != nil && f2 != nil
}
}
}
func TestLevelIter(t *testing.T) {
var level [][]Span
var rangedels [][]Span
var metas []*manifest.FileMetadata
var iter FragmentIterator
var extraInfo func() string
datadriven.RunTest(t, "testdata/level_iter", func(t *testing.T, d *datadriven.TestData) string {
switch d.Cmd {
case "define":
level = level[:0]
metas = metas[:0]
rangedels = rangedels[:0]
if iter != nil {
iter.Close()
iter = nil
}
var pointKeys []base.InternalKey
var currentRangeDels []Span
var currentFile []Span
for _, key := range strings.Split(d.Input, "\n") {
if strings.HasPrefix(key, "file") {
// Skip the very first file creation.
if len(level) != 0 || len(currentFile) != 0 {
meta := &manifest.FileMetadata{
FileNum: base.FileNum(len(level) + 1),
}
if len(currentFile) > 0 {
smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind())
largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End)
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest)
}
if len(pointKeys) != 0 {
meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1])
}
meta.InitPhysicalBacking()
level = append(level, currentFile)
metas = append(metas, meta)
rangedels = append(rangedels, currentRangeDels)
currentRangeDels = nil
currentFile = nil
pointKeys = nil
}
continue
}
key = strings.TrimSpace(key)
if strings.HasPrefix(key, "point:") {
key = strings.TrimPrefix(key, "point:")
j := strings.Index(key, ":")
ikey := base.ParseInternalKey(key[:j])
pointKeys = append(pointKeys, ikey)
if ikey.Kind() == base.InternalKeyKindRangeDelete {
currentRangeDels = append(currentRangeDels, Span{
Start: ikey.UserKey, End: []byte(key[j+1:]), Keys: []Key{{Trailer: ikey.Trailer}}})
}
continue
}
span := ParseSpan(key)
currentFile = append(currentFile, span)
}
meta := &manifest.FileMetadata{
FileNum: base.FileNum(len(level) + 1),
}
meta.InitPhysicalBacking()
level = append(level, currentFile)
rangedels = append(rangedels, currentRangeDels)
if len(currentFile) > 0 {
smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind())
largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End)
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest)
}
if len(pointKeys) != 0 {
meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1])
}
metas = append(metas, meta)
return ""
case "num-files":
return fmt.Sprintf("%d", len(level))
case "close-iter":
_ = iter.Close()
iter = nil
return "ok"
case "iter":
keyType := manifest.KeyTypeRange
for _, arg := range d.CmdArgs {
if strings.Contains(arg.Key, "rangedel") {
keyType = manifest.KeyTypePoint
}
}
if iter == nil {
var lastFileNum base.FileNum
tableNewIters := func(file *manifest.FileMetadata, _ SpanIterOptions) (FragmentIterator, error) {
keyType := keyType
spans := level[file.FileNum-1]
if keyType == manifest.KeyTypePoint {
spans = rangedels[file.FileNum-1]
}
lastFileNum = file.FileNum
return NewIter(base.DefaultComparer.Compare, spans), nil
}
b := &manifest.BulkVersionEdit{}
amap := make(map[base.FileNum]*manifest.FileMetadata)
for i := range metas {
amap[metas[i].FileNum] = metas[i]
}
b.Added[6] = amap
v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys)
require.NoError(t, err)
iter = NewLevelIter(
SpanIterOptions{}, base.DefaultComparer.Compare,
tableNewIters, v.Levels[6].Iter(), 6, keyType,
)
extraInfo = func() string {
return fmt.Sprintf("file = %s.sst", lastFileNum)
}
}
return runFragmentIteratorCmd(iter, d.Input, extraInfo)
default:
return fmt.Sprintf("unknown command: %s", d.Cmd)
}
})
if iter != nil {
iter.Close()
}
}