mirror of
https://source.quilibrium.com/quilibrium/ceremonyclient.git
synced 2024-12-26 16:45:18 +00:00
473 lines
13 KiB
Go
473 lines
13 KiB
Go
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
|
|
// of this source code is governed by a BSD-style license that can be found in
|
|
// the LICENSE file.
|
|
|
|
package keyspan
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/cockroachdb/datadriven"
|
|
"github.com/cockroachdb/pebble/internal/base"
|
|
"github.com/cockroachdb/pebble/internal/manifest"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestLevelIterEquivalence(t *testing.T) {
|
|
type level [][]Span
|
|
testCases := []struct {
|
|
name string
|
|
levels []level
|
|
}{
|
|
{
|
|
"single level, no gaps, no overlaps",
|
|
[]level{
|
|
{
|
|
{
|
|
Span{
|
|
Start: []byte("a"),
|
|
End: []byte("b"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("b"),
|
|
End: []byte("c"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("c"),
|
|
End: []byte("d"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
Span{
|
|
Start: []byte("d"),
|
|
End: []byte("e"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("e"),
|
|
End: []byte("f"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("f"),
|
|
End: []byte("g"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"single level, overlapping fragments",
|
|
[]level{
|
|
{
|
|
{
|
|
Span{
|
|
Start: []byte("a"),
|
|
End: []byte("b"),
|
|
Keys: []Key{
|
|
{
|
|
Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("bar"),
|
|
},
|
|
{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
},
|
|
},
|
|
},
|
|
Span{
|
|
Start: []byte("b"),
|
|
End: []byte("c"),
|
|
Keys: []Key{
|
|
{
|
|
Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("bar"),
|
|
},
|
|
{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
},
|
|
},
|
|
},
|
|
Span{
|
|
Start: []byte("c"),
|
|
End: []byte("d"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
Span{
|
|
Start: []byte("d"),
|
|
End: []byte("e"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("e"),
|
|
End: []byte("f"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("f"),
|
|
End: []byte("g"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"single level, gaps between files and range keys",
|
|
[]level{
|
|
{
|
|
{
|
|
Span{
|
|
Start: []byte("a"),
|
|
End: []byte("b"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("c"),
|
|
End: []byte("d"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("e"),
|
|
End: []byte("f"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
Span{
|
|
Start: []byte("g"),
|
|
End: []byte("h"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("i"),
|
|
End: []byte("j"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
Span{
|
|
Start: []byte("k"),
|
|
End: []byte("l"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"two levels, one with overlapping unset",
|
|
[]level{
|
|
{
|
|
{
|
|
Span{
|
|
Start: []byte("a"),
|
|
End: []byte("h"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
Span{
|
|
Start: []byte("l"),
|
|
End: []byte("u"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeyUnset),
|
|
Suffix: nil,
|
|
Value: nil,
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
{
|
|
Span{
|
|
Start: []byte("e"),
|
|
End: []byte("r"),
|
|
Keys: []Key{{
|
|
Trailer: base.MakeTrailer(1, base.InternalKeyKindRangeKeySet),
|
|
Suffix: nil,
|
|
Value: []byte("foo"),
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
var fileIters []FragmentIterator
|
|
var levelIters []FragmentIterator
|
|
var iter1, iter2 MergingIter
|
|
for j, level := range tc.levels {
|
|
j := j // Copy for use in closures down below.
|
|
var levelIter LevelIter
|
|
var metas []*manifest.FileMetadata
|
|
for k, file := range level {
|
|
fileIters = append(fileIters, NewIter(base.DefaultComparer.Compare, file))
|
|
meta := &manifest.FileMetadata{
|
|
FileNum: base.FileNum(k + 1),
|
|
Size: 1024,
|
|
SmallestSeqNum: 2,
|
|
LargestSeqNum: 2,
|
|
SmallestRangeKey: base.MakeInternalKey(file[0].Start, file[0].SmallestKey().SeqNum(), file[0].SmallestKey().Kind()),
|
|
LargestRangeKey: base.MakeExclusiveSentinelKey(file[len(file)-1].LargestKey().Kind(), file[len(file)-1].End),
|
|
HasPointKeys: false,
|
|
HasRangeKeys: true,
|
|
}
|
|
meta.InitPhysicalBacking()
|
|
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, meta.SmallestRangeKey, meta.LargestRangeKey)
|
|
metas = append(metas, meta)
|
|
}
|
|
|
|
tableNewIters := func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error) {
|
|
return NewIter(base.DefaultComparer.Compare, tc.levels[j][file.FileNum-1]), nil
|
|
}
|
|
// Add all the fileMetadatas to L6.
|
|
b := &manifest.BulkVersionEdit{}
|
|
amap := make(map[base.FileNum]*manifest.FileMetadata)
|
|
for i := range metas {
|
|
amap[metas[i].FileNum] = metas[i]
|
|
}
|
|
b.Added[6] = amap
|
|
v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys)
|
|
require.NoError(t, err)
|
|
levelIter.Init(
|
|
SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters,
|
|
v.Levels[6].Iter(), 0, manifest.KeyTypeRange,
|
|
)
|
|
levelIters = append(levelIters, &levelIter)
|
|
}
|
|
|
|
iter1.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), fileIters...)
|
|
iter2.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), levelIters...)
|
|
// Check iter1 and iter2 for equivalence.
|
|
|
|
require.Equal(t, iter1.First(), iter2.First(), "failed on test case %q", tc.name)
|
|
valid := true
|
|
for valid {
|
|
f1 := iter1.Next()
|
|
var f2 *Span
|
|
for {
|
|
f2 = iter2.Next()
|
|
// The level iter could produce empty spans that straddle between
|
|
// files. Ignore those.
|
|
if f2 == nil || !f2.Empty() {
|
|
break
|
|
}
|
|
}
|
|
|
|
require.Equal(t, f1, f2, "failed on test case %q", tc.name)
|
|
valid = f1 != nil && f2 != nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLevelIter(t *testing.T) {
|
|
var level [][]Span
|
|
var rangedels [][]Span
|
|
var metas []*manifest.FileMetadata
|
|
var iter FragmentIterator
|
|
var extraInfo func() string
|
|
|
|
datadriven.RunTest(t, "testdata/level_iter", func(t *testing.T, d *datadriven.TestData) string {
|
|
switch d.Cmd {
|
|
case "define":
|
|
level = level[:0]
|
|
metas = metas[:0]
|
|
rangedels = rangedels[:0]
|
|
if iter != nil {
|
|
iter.Close()
|
|
iter = nil
|
|
}
|
|
var pointKeys []base.InternalKey
|
|
var currentRangeDels []Span
|
|
var currentFile []Span
|
|
for _, key := range strings.Split(d.Input, "\n") {
|
|
if strings.HasPrefix(key, "file") {
|
|
// Skip the very first file creation.
|
|
if len(level) != 0 || len(currentFile) != 0 {
|
|
meta := &manifest.FileMetadata{
|
|
FileNum: base.FileNum(len(level) + 1),
|
|
}
|
|
if len(currentFile) > 0 {
|
|
smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind())
|
|
largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End)
|
|
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest)
|
|
}
|
|
if len(pointKeys) != 0 {
|
|
meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1])
|
|
}
|
|
meta.InitPhysicalBacking()
|
|
level = append(level, currentFile)
|
|
metas = append(metas, meta)
|
|
rangedels = append(rangedels, currentRangeDels)
|
|
currentRangeDels = nil
|
|
currentFile = nil
|
|
pointKeys = nil
|
|
}
|
|
continue
|
|
}
|
|
key = strings.TrimSpace(key)
|
|
if strings.HasPrefix(key, "point:") {
|
|
key = strings.TrimPrefix(key, "point:")
|
|
j := strings.Index(key, ":")
|
|
ikey := base.ParseInternalKey(key[:j])
|
|
pointKeys = append(pointKeys, ikey)
|
|
if ikey.Kind() == base.InternalKeyKindRangeDelete {
|
|
currentRangeDels = append(currentRangeDels, Span{
|
|
Start: ikey.UserKey, End: []byte(key[j+1:]), Keys: []Key{{Trailer: ikey.Trailer}}})
|
|
}
|
|
continue
|
|
}
|
|
span := ParseSpan(key)
|
|
currentFile = append(currentFile, span)
|
|
}
|
|
meta := &manifest.FileMetadata{
|
|
FileNum: base.FileNum(len(level) + 1),
|
|
}
|
|
meta.InitPhysicalBacking()
|
|
level = append(level, currentFile)
|
|
rangedels = append(rangedels, currentRangeDels)
|
|
if len(currentFile) > 0 {
|
|
smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind())
|
|
largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End)
|
|
meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest)
|
|
}
|
|
if len(pointKeys) != 0 {
|
|
meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1])
|
|
}
|
|
metas = append(metas, meta)
|
|
return ""
|
|
case "num-files":
|
|
return fmt.Sprintf("%d", len(level))
|
|
case "close-iter":
|
|
_ = iter.Close()
|
|
iter = nil
|
|
return "ok"
|
|
case "iter":
|
|
keyType := manifest.KeyTypeRange
|
|
for _, arg := range d.CmdArgs {
|
|
if strings.Contains(arg.Key, "rangedel") {
|
|
keyType = manifest.KeyTypePoint
|
|
}
|
|
}
|
|
if iter == nil {
|
|
var lastFileNum base.FileNum
|
|
tableNewIters := func(file *manifest.FileMetadata, _ SpanIterOptions) (FragmentIterator, error) {
|
|
keyType := keyType
|
|
spans := level[file.FileNum-1]
|
|
if keyType == manifest.KeyTypePoint {
|
|
spans = rangedels[file.FileNum-1]
|
|
}
|
|
lastFileNum = file.FileNum
|
|
return NewIter(base.DefaultComparer.Compare, spans), nil
|
|
}
|
|
b := &manifest.BulkVersionEdit{}
|
|
amap := make(map[base.FileNum]*manifest.FileMetadata)
|
|
for i := range metas {
|
|
amap[metas[i].FileNum] = metas[i]
|
|
}
|
|
b.Added[6] = amap
|
|
v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys)
|
|
require.NoError(t, err)
|
|
iter = NewLevelIter(
|
|
SpanIterOptions{}, base.DefaultComparer.Compare,
|
|
tableNewIters, v.Levels[6].Iter(), 6, keyType,
|
|
)
|
|
extraInfo = func() string {
|
|
return fmt.Sprintf("file = %s.sst", lastFileNum)
|
|
}
|
|
}
|
|
|
|
return runFragmentIteratorCmd(iter, d.Input, extraInfo)
|
|
|
|
default:
|
|
return fmt.Sprintf("unknown command: %s", d.Cmd)
|
|
}
|
|
})
|
|
|
|
if iter != nil {
|
|
iter.Close()
|
|
}
|
|
}
|