// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. package keyspan import ( "fmt" "strings" "testing" "github.com/cockroachdb/datadriven" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/manifest" "github.com/stretchr/testify/require" ) func TestLevelIterEquivalence(t *testing.T) { type level [][]Span testCases := []struct { name string levels []level }{ { "single level, no gaps, no overlaps", []level{ { { Span{ Start: []byte("a"), End: []byte("b"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("b"), End: []byte("c"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("c"), End: []byte("d"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, { Span{ Start: []byte("d"), End: []byte("e"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("e"), End: []byte("f"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("f"), End: []byte("g"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, }, }, }, { "single level, overlapping fragments", []level{ { { Span{ Start: []byte("a"), End: []byte("b"), Keys: []Key{ { Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("bar"), }, { Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }, }, }, Span{ Start: []byte("b"), End: []byte("c"), Keys: []Key{ { Trailer: base.MakeTrailer(4, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("bar"), }, { Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }, }, }, Span{ Start: []byte("c"), End: []byte("d"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, { Span{ Start: []byte("d"), End: []byte("e"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("e"), End: []byte("f"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("f"), End: []byte("g"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, }, }, }, { "single level, gaps between files and range keys", []level{ { { Span{ Start: []byte("a"), End: []byte("b"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("c"), End: []byte("d"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("e"), End: []byte("f"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, { Span{ Start: []byte("g"), End: []byte("h"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("i"), End: []byte("j"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, Span{ Start: []byte("k"), End: []byte("l"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, }, }, }, { "two levels, one with overlapping unset", []level{ { { Span{ Start: []byte("a"), End: []byte("h"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, { Span{ Start: []byte("l"), End: []byte("u"), Keys: []Key{{ Trailer: base.MakeTrailer(2, base.InternalKeyKindRangeKeyUnset), Suffix: nil, Value: nil, }}, }, }, }, { { Span{ Start: []byte("e"), End: []byte("r"), Keys: []Key{{ Trailer: base.MakeTrailer(1, base.InternalKeyKindRangeKeySet), Suffix: nil, Value: []byte("foo"), }}, }, }, }, }, }, } for _, tc := range testCases { var fileIters []FragmentIterator var levelIters []FragmentIterator var iter1, iter2 MergingIter for j, level := range tc.levels { j := j // Copy for use in closures down below. var levelIter LevelIter var metas []*manifest.FileMetadata for k, file := range level { fileIters = append(fileIters, NewIter(base.DefaultComparer.Compare, file)) meta := &manifest.FileMetadata{ FileNum: base.FileNum(k + 1), Size: 1024, SmallestSeqNum: 2, LargestSeqNum: 2, SmallestRangeKey: base.MakeInternalKey(file[0].Start, file[0].SmallestKey().SeqNum(), file[0].SmallestKey().Kind()), LargestRangeKey: base.MakeExclusiveSentinelKey(file[len(file)-1].LargestKey().Kind(), file[len(file)-1].End), HasPointKeys: false, HasRangeKeys: true, } meta.InitPhysicalBacking() meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, meta.SmallestRangeKey, meta.LargestRangeKey) metas = append(metas, meta) } tableNewIters := func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error) { return NewIter(base.DefaultComparer.Compare, tc.levels[j][file.FileNum-1]), nil } // Add all the fileMetadatas to L6. b := &manifest.BulkVersionEdit{} amap := make(map[base.FileNum]*manifest.FileMetadata) for i := range metas { amap[metas[i].FileNum] = metas[i] } b.Added[6] = amap v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys) require.NoError(t, err) levelIter.Init( SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters, v.Levels[6].Iter(), 0, manifest.KeyTypeRange, ) levelIters = append(levelIters, &levelIter) } iter1.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), fileIters...) iter2.Init(base.DefaultComparer.Compare, VisibleTransform(base.InternalKeySeqNumMax), new(MergingBuffers), levelIters...) // Check iter1 and iter2 for equivalence. require.Equal(t, iter1.First(), iter2.First(), "failed on test case %q", tc.name) valid := true for valid { f1 := iter1.Next() var f2 *Span for { f2 = iter2.Next() // The level iter could produce empty spans that straddle between // files. Ignore those. if f2 == nil || !f2.Empty() { break } } require.Equal(t, f1, f2, "failed on test case %q", tc.name) valid = f1 != nil && f2 != nil } } } func TestLevelIter(t *testing.T) { var level [][]Span var rangedels [][]Span var metas []*manifest.FileMetadata var iter FragmentIterator var extraInfo func() string datadriven.RunTest(t, "testdata/level_iter", func(t *testing.T, d *datadriven.TestData) string { switch d.Cmd { case "define": level = level[:0] metas = metas[:0] rangedels = rangedels[:0] if iter != nil { iter.Close() iter = nil } var pointKeys []base.InternalKey var currentRangeDels []Span var currentFile []Span for _, key := range strings.Split(d.Input, "\n") { if strings.HasPrefix(key, "file") { // Skip the very first file creation. if len(level) != 0 || len(currentFile) != 0 { meta := &manifest.FileMetadata{ FileNum: base.FileNum(len(level) + 1), } if len(currentFile) > 0 { smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind()) largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End) meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest) } if len(pointKeys) != 0 { meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1]) } meta.InitPhysicalBacking() level = append(level, currentFile) metas = append(metas, meta) rangedels = append(rangedels, currentRangeDels) currentRangeDels = nil currentFile = nil pointKeys = nil } continue } key = strings.TrimSpace(key) if strings.HasPrefix(key, "point:") { key = strings.TrimPrefix(key, "point:") j := strings.Index(key, ":") ikey := base.ParseInternalKey(key[:j]) pointKeys = append(pointKeys, ikey) if ikey.Kind() == base.InternalKeyKindRangeDelete { currentRangeDels = append(currentRangeDels, Span{ Start: ikey.UserKey, End: []byte(key[j+1:]), Keys: []Key{{Trailer: ikey.Trailer}}}) } continue } span := ParseSpan(key) currentFile = append(currentFile, span) } meta := &manifest.FileMetadata{ FileNum: base.FileNum(len(level) + 1), } meta.InitPhysicalBacking() level = append(level, currentFile) rangedels = append(rangedels, currentRangeDels) if len(currentFile) > 0 { smallest := base.MakeInternalKey(currentFile[0].Start, currentFile[0].SmallestKey().SeqNum(), currentFile[0].SmallestKey().Kind()) largest := base.MakeExclusiveSentinelKey(currentFile[len(currentFile)-1].LargestKey().Kind(), currentFile[len(currentFile)-1].End) meta.ExtendRangeKeyBounds(base.DefaultComparer.Compare, smallest, largest) } if len(pointKeys) != 0 { meta.ExtendPointKeyBounds(base.DefaultComparer.Compare, pointKeys[0], pointKeys[len(pointKeys)-1]) } metas = append(metas, meta) return "" case "num-files": return fmt.Sprintf("%d", len(level)) case "close-iter": _ = iter.Close() iter = nil return "ok" case "iter": keyType := manifest.KeyTypeRange for _, arg := range d.CmdArgs { if strings.Contains(arg.Key, "rangedel") { keyType = manifest.KeyTypePoint } } if iter == nil { var lastFileNum base.FileNum tableNewIters := func(file *manifest.FileMetadata, _ SpanIterOptions) (FragmentIterator, error) { keyType := keyType spans := level[file.FileNum-1] if keyType == manifest.KeyTypePoint { spans = rangedels[file.FileNum-1] } lastFileNum = file.FileNum return NewIter(base.DefaultComparer.Compare, spans), nil } b := &manifest.BulkVersionEdit{} amap := make(map[base.FileNum]*manifest.FileMetadata) for i := range metas { amap[metas[i].FileNum] = metas[i] } b.Added[6] = amap v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys) require.NoError(t, err) iter = NewLevelIter( SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters, v.Levels[6].Iter(), 6, keyType, ) extraInfo = func() string { return fmt.Sprintf("file = %s.sst", lastFileNum) } } return runFragmentIteratorCmd(iter, d.Input, extraInfo) default: return fmt.Sprintf("unknown command: %s", d.Cmd) } }) if iter != nil { iter.Close() } }