ceremonyclient/pebble/internal/manifest/version_edit_test.go
Cassandra Heart 2e2a1e4789
v1.2.0 (#31)
2024-01-03 01:31:42 -06:00

546 lines
15 KiB
Go

// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import (
"bytes"
"fmt"
"io"
"os"
"reflect"
"slices"
"strconv"
"strings"
"testing"
"github.com/cockroachdb/datadriven"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/record"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
)
func checkRoundTrip(e0 VersionEdit) error {
var e1 VersionEdit
buf := new(bytes.Buffer)
if err := e0.Encode(buf); err != nil {
return errors.Wrap(err, "encode")
}
if err := e1.Decode(buf); err != nil {
return errors.Wrap(err, "decode")
}
if diff := pretty.Diff(e0, e1); diff != nil {
return errors.Errorf("%s", strings.Join(diff, "\n"))
}
return nil
}
// Version edits with virtual sstables will not be the same after a round trip
// as the Decode function will not set the FileBacking for a virtual sstable.
// We test round trip + bve accumulation here, after which the virtual sstable
// FileBacking should be set.
func TestVERoundTripAndAccumulate(t *testing.T) {
cmp := base.DefaultComparer.Compare
m1 := (&FileMetadata{
FileNum: 810,
Size: 8090,
CreationTime: 809060,
SmallestSeqNum: 9,
LargestSeqNum: 11,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("a"), 0, base.InternalKeyKindSet),
base.MakeInternalKey([]byte("m"), 0, base.InternalKeyKindSet),
).ExtendRangeKeyBounds(
cmp,
base.MakeInternalKey([]byte("l"), 0, base.InternalKeyKindRangeKeySet),
base.MakeExclusiveSentinelKey(base.InternalKeyKindRangeKeySet, []byte("z")),
)
m1.InitPhysicalBacking()
m2 := (&FileMetadata{
FileNum: 812,
Size: 8090,
CreationTime: 809060,
SmallestSeqNum: 9,
LargestSeqNum: 11,
Virtual: true,
FileBacking: m1.FileBacking,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("a"), 0, base.InternalKeyKindSet),
base.MakeInternalKey([]byte("c"), 0, base.InternalKeyKindSet),
)
ve1 := VersionEdit{
ComparerName: "11",
MinUnflushedLogNum: 22,
ObsoletePrevLogNum: 33,
NextFileNum: 44,
LastSeqNum: 55,
CreatedBackingTables: []*FileBacking{m1.FileBacking},
NewFiles: []NewFileEntry{
{
Level: 4,
Meta: m2,
// Only set for the test.
BackingFileNum: m2.FileBacking.DiskFileNum,
},
},
}
var err error
buf := new(bytes.Buffer)
if err = ve1.Encode(buf); err != nil {
t.Error(err)
}
var ve2 VersionEdit
if err = ve2.Decode(buf); err != nil {
t.Error(err)
}
// Perform accumulation to set the FileBacking on the files in the Decoded
// version edit.
var bve BulkVersionEdit
require.NoError(t, bve.Accumulate(&ve2))
if diff := pretty.Diff(ve1, ve2); diff != nil {
t.Error(errors.Errorf("%s", strings.Join(diff, "\n")))
}
}
func TestVersionEditRoundTrip(t *testing.T) {
cmp := base.DefaultComparer.Compare
m1 := (&FileMetadata{
FileNum: 805,
Size: 8050,
CreationTime: 805030,
}).ExtendPointKeyBounds(
cmp,
base.DecodeInternalKey([]byte("abc\x00\x01\x02\x03\x04\x05\x06\x07")),
base.DecodeInternalKey([]byte("xyz\x01\xff\xfe\xfd\xfc\xfb\xfa\xf9")),
)
m1.InitPhysicalBacking()
m2 := (&FileMetadata{
FileNum: 806,
Size: 8060,
CreationTime: 806040,
SmallestSeqNum: 3,
LargestSeqNum: 5,
MarkedForCompaction: true,
}).ExtendPointKeyBounds(
cmp,
base.DecodeInternalKey([]byte("A\x00\x01\x02\x03\x04\x05\x06\x07")),
base.DecodeInternalKey([]byte("Z\x01\xff\xfe\xfd\xfc\xfb\xfa\xf9")),
)
m2.InitPhysicalBacking()
m3 := (&FileMetadata{
FileNum: 807,
Size: 8070,
CreationTime: 807050,
}).ExtendRangeKeyBounds(
cmp,
base.MakeInternalKey([]byte("aaa"), 0, base.InternalKeyKindRangeKeySet),
base.MakeExclusiveSentinelKey(base.InternalKeyKindRangeKeySet, []byte("zzz")),
)
m3.InitPhysicalBacking()
m4 := (&FileMetadata{
FileNum: 809,
Size: 8090,
CreationTime: 809060,
SmallestSeqNum: 9,
LargestSeqNum: 11,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("a"), 0, base.InternalKeyKindSet),
base.MakeInternalKey([]byte("m"), 0, base.InternalKeyKindSet),
).ExtendRangeKeyBounds(
cmp,
base.MakeInternalKey([]byte("l"), 0, base.InternalKeyKindRangeKeySet),
base.MakeExclusiveSentinelKey(base.InternalKeyKindRangeKeySet, []byte("z")),
)
m4.InitPhysicalBacking()
m5 := (&FileMetadata{
FileNum: 810,
Size: 8090,
CreationTime: 809060,
SmallestSeqNum: 9,
LargestSeqNum: 11,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("a"), 0, base.InternalKeyKindSet),
base.MakeInternalKey([]byte("m"), 0, base.InternalKeyKindSet),
).ExtendRangeKeyBounds(
cmp,
base.MakeInternalKey([]byte("l"), 0, base.InternalKeyKindRangeKeySet),
base.MakeExclusiveSentinelKey(base.InternalKeyKindRangeKeySet, []byte("z")),
)
m5.InitPhysicalBacking()
m6 := (&FileMetadata{
FileNum: 811,
Size: 8090,
CreationTime: 809060,
SmallestSeqNum: 9,
LargestSeqNum: 11,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("a"), 0, base.InternalKeyKindSet),
base.MakeInternalKey([]byte("m"), 0, base.InternalKeyKindSet),
).ExtendRangeKeyBounds(
cmp,
base.MakeInternalKey([]byte("l"), 0, base.InternalKeyKindRangeKeySet),
base.MakeExclusiveSentinelKey(base.InternalKeyKindRangeKeySet, []byte("z")),
)
m6.InitPhysicalBacking()
testCases := []VersionEdit{
// An empty version edit.
{},
// A complete version edit.
{
ComparerName: "11",
MinUnflushedLogNum: 22,
ObsoletePrevLogNum: 33,
NextFileNum: 44,
LastSeqNum: 55,
RemovedBackingTables: []base.DiskFileNum{
base.FileNum(10).DiskFileNum(), base.FileNum(11).DiskFileNum(),
},
CreatedBackingTables: []*FileBacking{m5.FileBacking, m6.FileBacking},
DeletedFiles: map[DeletedFileEntry]*FileMetadata{
{
Level: 3,
FileNum: 703,
}: nil,
{
Level: 4,
FileNum: 704,
}: nil,
},
NewFiles: []NewFileEntry{
{
Level: 4,
Meta: m1,
},
{
Level: 5,
Meta: m2,
},
{
Level: 6,
Meta: m3,
},
{
Level: 6,
Meta: m4,
},
},
},
}
for _, tc := range testCases {
if err := checkRoundTrip(tc); err != nil {
t.Error(err)
}
}
}
func TestVersionEditDecode(t *testing.T) {
// TODO(radu): these should be datadriven tests that output the encoded and
// decoded edits.
cmp := base.DefaultComparer.Compare
m := (&FileMetadata{
FileNum: 4,
Size: 709,
SmallestSeqNum: 12,
LargestSeqNum: 14,
CreationTime: 1701712644,
}).ExtendPointKeyBounds(
cmp,
base.MakeInternalKey([]byte("bar"), 14, base.InternalKeyKindDelete),
base.MakeInternalKey([]byte("foo"), 13, base.InternalKeyKindSet),
)
m.InitPhysicalBacking()
testCases := []struct {
filename string
encodedEdits []string
edits []VersionEdit
}{
// db-stage-1 and db-stage-2 have the same manifest.
{
filename: "db-stage-1/MANIFEST-000001",
encodedEdits: []string{
"\x01\x1aleveldb.BytewiseComparator\x03\x02\x04\x00",
"\x02\x02\x03\x03\x04\t",
},
edits: []VersionEdit{
{
ComparerName: "leveldb.BytewiseComparator",
NextFileNum: 2,
},
{
MinUnflushedLogNum: 0x2,
NextFileNum: 0x3,
LastSeqNum: 0x9,
},
},
},
// db-stage-3 and db-stage-4 have the same manifest.
{
filename: "db-stage-3/MANIFEST-000006",
encodedEdits: []string{
"\x01\x1aleveldb.BytewiseComparator\x02\x02\x03\a\x04\x00",
"\x02\x05\x03\x06\x04\x0eg\x00\x04\xc5\x05\vbar\x00\x0e\x00\x00\x00\x00\x00\x00\vfoo\x01\r\x00\x00\x00\x00\x00\x00\f\x0e\x06\x05\x84\xa6\xb8\xab\x06\x01",
},
edits: []VersionEdit{
{
ComparerName: "leveldb.BytewiseComparator",
MinUnflushedLogNum: 0x2,
NextFileNum: 0x7,
},
{
MinUnflushedLogNum: 0x5,
NextFileNum: 0x6,
LastSeqNum: 0xe,
NewFiles: []NewFileEntry{
{
Level: 0,
Meta: m,
},
},
},
},
},
}
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
f, err := os.Open("../../testdata/" + tc.filename)
if err != nil {
t.Fatalf("filename=%q: open error: %v", tc.filename, err)
}
defer f.Close()
i, r := 0, record.NewReader(f, 0 /* logNum */)
for {
rr, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("filename=%q i=%d: record reader error: %v", tc.filename, i, err)
}
if i >= len(tc.edits) {
t.Fatalf("filename=%q i=%d: too many version edits", tc.filename, i+1)
}
encodedEdit, err := io.ReadAll(rr)
if err != nil {
t.Fatalf("filename=%q i=%d: read error: %v", tc.filename, i, err)
}
if s := string(encodedEdit); s != tc.encodedEdits[i] {
t.Fatalf("filename=%q i=%d: got encoded %q, want %q", tc.filename, i, s, tc.encodedEdits[i])
}
var edit VersionEdit
err = edit.Decode(bytes.NewReader(encodedEdit))
if err != nil {
t.Fatalf("filename=%q i=%d: decode error: %v", tc.filename, i, err)
}
if !reflect.DeepEqual(edit, tc.edits[i]) {
t.Fatalf("filename=%q i=%d: decode\n\tgot %#v\n\twant %#v\n%s", tc.filename, i, edit, tc.edits[i],
strings.Join(pretty.Diff(edit, tc.edits[i]), "\n"))
}
if err := checkRoundTrip(edit); err != nil {
t.Fatalf("filename=%q i=%d: round trip: %v", tc.filename, i, err)
}
i++
}
if i != len(tc.edits) {
t.Fatalf("filename=%q: got %d edits, want %d", tc.filename, i, len(tc.edits))
}
})
}
}
func TestVersionEditEncodeLastSeqNum(t *testing.T) {
testCases := []struct {
edit VersionEdit
encoded string
}{
// If ComparerName is unset, LastSeqNum is only encoded if non-zero.
{VersionEdit{LastSeqNum: 0}, ""},
{VersionEdit{LastSeqNum: 1}, "\x04\x01"},
// For compatibility with RocksDB, if ComparerName is set we always encode
// LastSeqNum.
{VersionEdit{ComparerName: "foo", LastSeqNum: 0}, "\x01\x03\x66\x6f\x6f\x04\x00"},
{VersionEdit{ComparerName: "foo", LastSeqNum: 1}, "\x01\x03\x66\x6f\x6f\x04\x01"},
}
for _, c := range testCases {
t.Run("", func(t *testing.T) {
var buf bytes.Buffer
require.NoError(t, c.edit.Encode(&buf))
if result := buf.String(); c.encoded != result {
t.Fatalf("expected %x, but found %x", c.encoded, result)
}
if c.edit.ComparerName != "" {
// Manually decode the version edit so that we can verify the contents
// even if the LastSeqNum decodes to 0.
d := versionEditDecoder{strings.NewReader(c.encoded)}
// Decode ComparerName.
tag, err := d.readUvarint()
require.NoError(t, err)
if tag != tagComparator {
t.Fatalf("expected %d, but found %d", tagComparator, tag)
}
s, err := d.readBytes()
require.NoError(t, err)
if c.edit.ComparerName != string(s) {
t.Fatalf("expected %q, but found %q", c.edit.ComparerName, s)
}
// Decode LastSeqNum.
tag, err = d.readUvarint()
require.NoError(t, err)
if tag != tagLastSequence {
t.Fatalf("expected %d, but found %d", tagLastSequence, tag)
}
val, err := d.readUvarint()
require.NoError(t, err)
if c.edit.LastSeqNum != val {
t.Fatalf("expected %d, but found %d", c.edit.LastSeqNum, val)
}
}
})
}
}
func TestVersionEditApply(t *testing.T) {
parseMeta := func(s string) (*FileMetadata, error) {
m, err := ParseFileMetadataDebug(s)
if err != nil {
return nil, err
}
m.SmallestSeqNum = m.Smallest.SeqNum()
m.LargestSeqNum = m.Largest.SeqNum()
if m.SmallestSeqNum > m.LargestSeqNum {
m.SmallestSeqNum, m.LargestSeqNum = m.LargestSeqNum, m.SmallestSeqNum
}
m.InitPhysicalBacking()
return m, nil
}
// TODO(bananabrick): Improve the parsing logic in this test.
datadriven.RunTest(t, "testdata/version_edit_apply",
func(t *testing.T, d *datadriven.TestData) string {
switch d.Cmd {
case "apply":
// TODO(sumeer): move this Version parsing code to utils, to
// avoid repeating it, and make it the inverse of
// Version.DebugString().
var v *Version
var veList []*VersionEdit
isVersion := true
isDelete := true
var level int
var err error
versionFiles := map[base.FileNum]*FileMetadata{}
for _, data := range strings.Split(d.Input, "\n") {
data = strings.TrimSpace(data)
switch data {
case "edit":
isVersion = false
veList = append(veList, &VersionEdit{})
case "delete":
isVersion = false
isDelete = true
case "add":
isVersion = false
isDelete = false
case "L0", "L1", "L2", "L3", "L4", "L5", "L6":
level, err = strconv.Atoi(data[1:])
if err != nil {
return err.Error()
}
default:
var ve *VersionEdit
if len(veList) > 0 {
ve = veList[len(veList)-1]
}
if isVersion || !isDelete {
meta, err := parseMeta(data)
if err != nil {
return err.Error()
}
if isVersion {
if v == nil {
v = new(Version)
for l := 0; l < NumLevels; l++ {
v.Levels[l] = makeLevelMetadata(base.DefaultComparer.Compare, l, nil /* files */)
}
}
versionFiles[meta.FileNum] = meta
v.Levels[level].insert(meta)
meta.LatestRef()
} else {
ve.NewFiles =
append(ve.NewFiles, NewFileEntry{Level: level, Meta: meta})
}
} else {
fileNum, err := strconv.Atoi(data)
if err != nil {
return err.Error()
}
dfe := DeletedFileEntry{Level: level, FileNum: base.FileNum(fileNum)}
if ve.DeletedFiles == nil {
ve.DeletedFiles = make(map[DeletedFileEntry]*FileMetadata)
}
ve.DeletedFiles[dfe] = versionFiles[dfe.FileNum]
}
}
}
if v != nil {
if err := v.InitL0Sublevels(base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20); err != nil {
return err.Error()
}
}
bve := BulkVersionEdit{}
bve.AddedByFileNum = make(map[base.FileNum]*FileMetadata)
for _, ve := range veList {
if err := bve.Accumulate(ve); err != nil {
return err.Error()
}
}
zombies := make(map[base.DiskFileNum]uint64)
newv, err := bve.Apply(v, base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20, 32000, zombies, ProhibitSplitUserKeys)
if err != nil {
return err.Error()
}
zombieFileNums := make([]base.DiskFileNum, 0, len(zombies))
if len(veList) == 1 {
// Only care about zombies if a single version edit was
// being applied.
for fileNum := range zombies {
zombieFileNums = append(zombieFileNums, fileNum)
}
slices.Sort(zombieFileNums)
}
return fmt.Sprintf("%szombies %d\n", newv, zombieFileNums)
default:
return fmt.Sprintf("unknown command: %s", d.Cmd)
}
})
}