ceremonyclient/pebble/open_test.go

1391 lines
39 KiB
Go
Raw Normal View History

2024-01-03 07:31:42 +00:00
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync/atomic"
"syscall"
"testing"
"github.com/cockroachdb/datadriven"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
"github.com/cockroachdb/pebble/vfs/errorfs"
"github.com/cockroachdb/redact"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
)
func TestOpenSharedTableCache(t *testing.T) {
c := cache.New(cacheDefaultSize)
tc := NewTableCache(c, 16, 100)
defer tc.Unref()
defer c.Unref()
d0, err := Open("", testingRandomized(t, &Options{
FS: vfs.NewMem(),
Cache: c,
TableCache: tc,
}))
if err != nil {
t.Errorf("d0 Open: %s", err.Error())
}
defer d0.Close()
d1, err := Open("", testingRandomized(t, &Options{
FS: vfs.NewMem(),
Cache: c,
TableCache: tc,
}))
if err != nil {
t.Errorf("d1 Open: %s", err.Error())
}
defer d1.Close()
// Make sure that the Open function is using the passed in table cache
// when the TableCache option is set.
require.Equalf(
t, d0.tableCache.tableCache, d1.tableCache.tableCache,
"expected tableCache for both d0 and d1 to be the same",
)
}
func TestErrorIfExists(t *testing.T) {
opts := testingRandomized(t, &Options{
FS: vfs.NewMem(),
ErrorIfExists: true,
})
defer ensureFilesClosed(t, opts)()
d0, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d0.Close())
if _, err := Open("", opts); !errors.Is(err, ErrDBAlreadyExists) {
t.Fatalf("expected db-already-exists error, got %v", err)
}
opts.ErrorIfExists = false
d1, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d1.Close())
}
func TestErrorIfNotExists(t *testing.T) {
opts := testingRandomized(t, &Options{
FS: vfs.NewMem(),
ErrorIfNotExists: true,
})
defer ensureFilesClosed(t, opts)()
_, err := Open("", opts)
if !errors.Is(err, ErrDBDoesNotExist) {
t.Fatalf("expected db-does-not-exist error, got %v", err)
}
// Create the DB and try again.
opts.ErrorIfNotExists = false
d0, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d0.Close())
opts.ErrorIfNotExists = true
d1, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d1.Close())
}
func TestErrorIfNotPristine(t *testing.T) {
opts := testingRandomized(t, &Options{
FS: vfs.NewMem(),
ErrorIfNotPristine: true,
})
defer ensureFilesClosed(t, opts)()
d0, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d0.Close())
// Store is pristine; ok to open.
d1, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d1.Set([]byte("foo"), []byte("bar"), Sync))
require.NoError(t, d1.Close())
if _, err := Open("", opts); !errors.Is(err, ErrDBNotPristine) {
t.Fatalf("expected db-not-pristine error, got %v", err)
}
// Run compaction and make sure we're still not allowed to open.
opts.ErrorIfNotPristine = false
d2, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d2.Compact([]byte("a"), []byte("z"), false /* parallelize */))
require.NoError(t, d2.Close())
opts.ErrorIfNotPristine = true
if _, err := Open("", opts); !errors.Is(err, ErrDBNotPristine) {
t.Fatalf("expected db-already-exists error, got %v", err)
}
}
func TestOpenAlreadyLocked(t *testing.T) {
runTest := func(t *testing.T, dirname string, fs vfs.FS) {
opts := testingRandomized(t, &Options{FS: fs})
var err error
opts.Lock, err = LockDirectory(dirname, fs)
require.NoError(t, err)
d, err := Open(dirname, opts)
require.NoError(t, err)
require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
// Try to open the same database reusing the Options containing the same
// Lock. It should error when it observes that it's already referenced.
_, err = Open(dirname, opts)
require.Error(t, err)
// Close the database.
require.NoError(t, d.Close())
// Now Opening should succeed again.
d, err = Open(dirname, opts)
require.NoError(t, err)
require.NoError(t, d.Close())
require.NoError(t, opts.Lock.Close())
// There should be no more remaining references.
require.Equal(t, int32(0), opts.Lock.refs.Load())
}
t.Run("memfs", func(t *testing.T) {
runTest(t, "", vfs.NewMem())
})
t.Run("disk", func(t *testing.T) {
runTest(t, t.TempDir(), vfs.Default)
})
}
func TestNewDBFilenames(t *testing.T) {
versions := map[FormatMajorVersion][]string{
FormatMostCompatible: {
"000002.log",
"CURRENT",
"LOCK",
"MANIFEST-000001",
"OPTIONS-000003",
},
internalFormatNewest: {
"000002.log",
"CURRENT",
"LOCK",
"MANIFEST-000001",
"OPTIONS-000003",
"marker.format-version.000015.016",
"marker.manifest.000001.MANIFEST-000001",
},
}
for formatVers, want := range versions {
t.Run(fmt.Sprintf("vers=%s", formatVers), func(t *testing.T) {
mem := vfs.NewMem()
fooBar := mem.PathJoin("foo", "bar")
d, err := Open(fooBar, &Options{
FS: mem,
FormatMajorVersion: formatVers,
})
if err != nil {
t.Fatalf("Open: %v", err)
}
if err := d.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
got, err := mem.List(fooBar)
if err != nil {
t.Fatalf("List: %v", err)
}
sort.Strings(got)
if !reflect.DeepEqual(got, want) {
t.Errorf("\ngot %v\nwant %v", got, want)
}
})
}
}
func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
opts := testingRandomized(t, &Options{FS: fs})
for _, startFromEmpty := range []bool{false, true} {
for _, walDirname := range []string{"", "wal"} {
for _, length := range []int{-1, 0, 1, 1000, 10000, 100000} {
dirname := "sharedDatabase" + walDirname
if startFromEmpty {
dirname = "startFromEmpty" + walDirname + strconv.Itoa(length)
}
dirname = fs.PathJoin(root, dirname)
if walDirname == "" {
opts.WALDir = ""
} else {
opts.WALDir = fs.PathJoin(dirname, walDirname)
}
got, xxx := []byte(nil), ""
if length >= 0 {
xxx = strings.Repeat("x", length)
}
d0, err := Open(dirname, opts)
if err != nil {
t.Fatalf("sfe=%t, length=%d: Open #0: %v",
startFromEmpty, length, err)
continue
}
if length >= 0 {
err = d0.Set([]byte("key"), []byte(xxx), nil)
if err != nil {
t.Errorf("sfe=%t, length=%d: Set: %v",
startFromEmpty, length, err)
continue
}
}
err = d0.Close()
if err != nil {
t.Errorf("sfe=%t, length=%d: Close #0: %v",
startFromEmpty, length, err)
continue
}
d1, err := Open(dirname, opts)
if err != nil {
t.Errorf("sfe=%t, length=%d: Open #1: %v",
startFromEmpty, length, err)
continue
}
if length >= 0 {
var closer io.Closer
got, closer, err = d1.Get([]byte("key"))
if err != nil {
t.Errorf("sfe=%t, length=%d: Get: %v",
startFromEmpty, length, err)
continue
}
got = append([]byte(nil), got...)
closer.Close()
}
err = d1.Close()
if err != nil {
t.Errorf("sfe=%t, length=%d: Close #1: %v",
startFromEmpty, length, err)
continue
}
if length >= 0 && string(got) != xxx {
t.Errorf("sfe=%t, length=%d: got value differs from set value",
startFromEmpty, length)
continue
}
{
got, err := opts.FS.List(dirname)
if err != nil {
t.Fatalf("List: %v", err)
}
var optionsCount int
for _, s := range got {
if t, _, ok := base.ParseFilename(opts.FS, s); ok && t == fileTypeOptions {
optionsCount++
}
}
if optionsCount != 1 {
t.Fatalf("expected 1 OPTIONS file, but found %d", optionsCount)
}
}
}
}
}
}
func TestOpenCloseOpenClose(t *testing.T) {
for _, fstype := range []string{"disk", "mem"} {
t.Run(fstype, func(t *testing.T) {
var fs vfs.FS
var dir string
switch fstype {
case "disk":
var err error
dir, err = os.MkdirTemp("", "open-close")
require.NoError(t, err)
defer func() {
_ = os.RemoveAll(dir)
}()
fs = vfs.Default
case "mem":
dir = ""
fs = vfs.NewMem()
}
testOpenCloseOpenClose(t, fs, dir)
})
}
}
func TestOpenOptionsCheck(t *testing.T) {
mem := vfs.NewMem()
opts := &Options{FS: mem}
d, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, d.Close())
opts = &Options{
Comparer: &Comparer{Name: "foo"},
FS: mem,
}
_, err = Open("", opts)
require.Regexp(t, `comparer name from file.*!=.*`, err)
opts = &Options{
Merger: &Merger{Name: "bar"},
FS: mem,
}
_, err = Open("", opts)
require.Regexp(t, `merger name from file.*!=.*`, err)
}
func TestOpenCrashWritingOptions(t *testing.T) {
memFS := vfs.NewMem()
d, err := Open("", &Options{FS: memFS})
require.NoError(t, err)
require.NoError(t, d.Close())
// Open the database again, this time with a mocked filesystem that
// will only succeed in partially writing the OPTIONS file.
fs := optionsTornWriteFS{FS: memFS}
_, err = Open("", &Options{FS: fs})
require.Error(t, err)
// Re-opening the database must succeed.
d, err = Open("", &Options{FS: memFS})
require.NoError(t, err)
require.NoError(t, d.Close())
}
type optionsTornWriteFS struct {
vfs.FS
}
func (fs optionsTornWriteFS) Create(name string) (vfs.File, error) {
file, err := fs.FS.Create(name)
if file != nil {
file = optionsTornWriteFile{File: file}
}
return file, err
}
type optionsTornWriteFile struct {
vfs.File
}
func (f optionsTornWriteFile) Write(b []byte) (int, error) {
// Look for the OPTIONS-XXXXXX file's `comparer=` field.
comparerKey := []byte("comparer=")
i := bytes.Index(b, comparerKey)
if i == -1 {
return f.File.Write(b)
}
// Write only the contents through `comparer=` and return an error.
n, _ := f.File.Write(b[:i+len(comparerKey)])
return n, syscall.EIO
}
func TestOpenReadOnly(t *testing.T) {
mem := vfs.NewMem()
{
// Opening a non-existent DB in read-only mode should result in no mutable
// filesystem operations.
var memLog base.InMemLogger
_, err := Open("non-existent", testingRandomized(t, &Options{
FS: vfs.WithLogging(mem, memLog.Infof),
ReadOnly: true,
WALDir: "non-existent-waldir",
}))
if err == nil {
t.Fatalf("expected error, but found success")
}
const expected = `open-dir: non-existent`
if trimmed := strings.TrimSpace(memLog.String()); expected != trimmed {
t.Fatalf("expected %q, but found %q", expected, trimmed)
}
}
{
// Opening a DB with a non-existent WAL dir in read-only mode should result
// in no mutable filesystem operations other than the LOCK.
var memLog base.InMemLogger
_, err := Open("", testingRandomized(t, &Options{
FS: vfs.WithLogging(mem, memLog.Infof),
ReadOnly: true,
WALDir: "non-existent-waldir",
}))
if err == nil {
t.Fatalf("expected error, but found success")
}
const expected = "open-dir: \nopen-dir: non-existent-waldir\nclose:"
if trimmed := strings.TrimSpace(memLog.String()); expected != trimmed {
t.Fatalf("expected %q, but found %q", expected, trimmed)
}
}
var contents []string
{
// Create a new DB and populate it with a small amount of data.
d, err := Open("", testingRandomized(t, &Options{
FS: mem,
}))
require.NoError(t, err)
require.NoError(t, d.Set([]byte("test"), nil, nil))
require.NoError(t, d.Close())
contents, err = mem.List("")
require.NoError(t, err)
sort.Strings(contents)
}
{
// Re-open the DB read-only. The directory contents should be unchanged.
d, err := Open("", testingRandomized(t, &Options{
FS: mem,
ReadOnly: true,
}))
require.NoError(t, err)
// Verify various write operations fail in read-only mode.
require.EqualValues(t, ErrReadOnly, d.Compact(nil, []byte("\xff"), false))
require.EqualValues(t, ErrReadOnly, d.Flush())
require.EqualValues(t, ErrReadOnly, func() error { _, err := d.AsyncFlush(); return err }())
require.EqualValues(t, ErrReadOnly, d.Delete(nil, nil))
require.EqualValues(t, ErrReadOnly, d.DeleteRange(nil, nil, nil))
require.EqualValues(t, ErrReadOnly, d.Ingest(nil))
require.EqualValues(t, ErrReadOnly, d.LogData(nil, nil))
require.EqualValues(t, ErrReadOnly, d.Merge(nil, nil, nil))
require.EqualValues(t, ErrReadOnly, d.Set(nil, nil, nil))
// Verify we can still read in read-only mode.
require.NoError(t, func() error {
_, closer, err := d.Get([]byte("test"))
if closer != nil {
closer.Close()
}
return err
}())
checkIter := func(iter *Iterator, err error) {
t.Helper()
var keys []string
for valid := iter.First(); valid; valid = iter.Next() {
keys = append(keys, string(iter.Key()))
}
require.NoError(t, iter.Close())
expectedKeys := []string{"test"}
if diff := pretty.Diff(keys, expectedKeys); diff != nil {
t.Fatalf("%s\n%s", strings.Join(diff, "\n"), keys)
}
}
checkIter(d.NewIter(nil))
b := d.NewIndexedBatch()
checkIter(b.NewIter(nil))
require.EqualValues(t, ErrReadOnly, b.Commit(nil))
require.EqualValues(t, ErrReadOnly, d.Apply(b, nil))
s := d.NewSnapshot()
checkIter(s.NewIter(nil))
require.NoError(t, s.Close())
require.NoError(t, d.Close())
newContents, err := mem.List("")
require.NoError(t, err)
sort.Strings(newContents)
if diff := pretty.Diff(contents, newContents); diff != nil {
t.Fatalf("%s", strings.Join(diff, "\n"))
}
}
}
func TestOpenWALReplay(t *testing.T) {
largeValue := []byte(strings.Repeat("a", 100<<10))
hugeValue := []byte(strings.Repeat("b", 10<<20))
checkIter := func(iter *Iterator, err error) {
t.Helper()
var keys []string
for valid := iter.First(); valid; valid = iter.Next() {
keys = append(keys, string(iter.Key()))
}
require.NoError(t, iter.Close())
expectedKeys := []string{"1", "2", "3", "4", "5"}
if diff := pretty.Diff(keys, expectedKeys); diff != nil {
t.Fatalf("%s\n%s", strings.Join(diff, "\n"), keys)
}
}
for _, readOnly := range []bool{false, true} {
t.Run(fmt.Sprintf("read-only=%t", readOnly), func(t *testing.T) {
// Create a new DB and populate it with some data.
const dir = ""
mem := vfs.NewMem()
d, err := Open(dir, testingRandomized(t, &Options{
FS: mem,
MemTableSize: 32 << 20,
}))
require.NoError(t, err)
// All these values will fit in a single memtable, so on closing the db there
// will be no sst and all the data is in a single WAL.
require.NoError(t, d.Set([]byte("1"), largeValue, nil))
require.NoError(t, d.Set([]byte("2"), largeValue, nil))
require.NoError(t, d.Set([]byte("3"), largeValue, nil))
require.NoError(t, d.Set([]byte("4"), hugeValue, nil))
require.NoError(t, d.Set([]byte("5"), largeValue, nil))
checkIter(d.NewIter(nil))
require.NoError(t, d.Close())
files, err := mem.List(dir)
require.NoError(t, err)
sort.Strings(files)
logCount, sstCount := 0, 0
for _, fname := range files {
if strings.HasSuffix(fname, ".sst") {
sstCount++
}
if strings.HasSuffix(fname, ".log") {
logCount++
}
}
require.Equal(t, 0, sstCount)
// The memtable size starts at 256KB and doubles up to 32MB so we expect 5
// logs (one for each doubling).
require.Equal(t, 7, logCount)
// Re-open the DB with a smaller memtable. Values for 1, 2 will fit in the first memtable;
// value for 3 will go in the next memtable; value for 4 will be in a flushable batch
// which will cause the previous memtable to be flushed; value for 5 will go in the next
// memtable
d, err = Open(dir, testingRandomized(t, &Options{
FS: mem,
MemTableSize: 300 << 10,
ReadOnly: readOnly,
}))
require.NoError(t, err)
if readOnly {
m := d.Metrics()
require.Equal(t, int64(logCount), m.WAL.Files)
d.mu.Lock()
require.NotNil(t, d.mu.mem.mutable)
d.mu.Unlock()
}
checkIter(d.NewIter(nil))
require.NoError(t, d.Close())
})
}
}
// Reproduction for https://github.com/cockroachdb/pebble/issues/2234.
func TestWALReplaySequenceNumBug(t *testing.T) {
mem := vfs.NewMem()
d, err := Open("", testingRandomized(t, &Options{
FS: mem,
}))
require.NoError(t, err)
d.mu.Lock()
// Disable any flushes.
d.mu.compact.flushing = true
d.mu.Unlock()
require.NoError(t, d.Set([]byte("1"), nil, nil))
require.NoError(t, d.Set([]byte("2"), nil, nil))
// Write a large batch. This should go to a separate memtable.
largeValue := []byte(strings.Repeat("a", int(d.largeBatchThreshold)))
require.NoError(t, d.Set([]byte("1"), largeValue, nil))
// This write should go the mutable memtable after the large batch in the
// memtable queue.
d.Set([]byte("1"), nil, nil)
d.mu.Lock()
d.mu.compact.flushing = false
d.mu.Unlock()
// Make sure none of the flushables have been flushed.
require.Equal(t, 3, len(d.mu.mem.queue))
// Close the db. This doesn't cause a flush of the memtables, so they'll
// have to be replayed when the db is reopened.
require.NoError(t, d.Close())
files, err := mem.List("")
require.NoError(t, err)
sort.Strings(files)
sstCount := 0
for _, fname := range files {
if strings.HasSuffix(fname, ".sst") {
sstCount++
}
}
require.Equal(t, 0, sstCount)
// Reopen db in read only mode to force read only wal replay.
d, err = Open("", &Options{
FS: mem,
ReadOnly: true,
})
require.NoError(t, err)
val, c, _ := d.Get([]byte("1"))
require.Equal(t, []byte{}, val)
c.Close()
require.NoError(t, d.Close())
}
// Similar to TestOpenWALReplay, except we test replay behavior after a
// memtable has been flushed. We test all 3 reasons for flushing: forced, size,
// and large-batch.
func TestOpenWALReplay2(t *testing.T) {
for _, readOnly := range []bool{false, true} {
t.Run(fmt.Sprintf("read-only=%t", readOnly), func(t *testing.T) {
for _, reason := range []string{"forced", "size", "large-batch"} {
t.Run(reason, func(t *testing.T) {
mem := vfs.NewMem()
d, err := Open("", testingRandomized(t, &Options{
FS: mem,
MemTableSize: 256 << 10,
}))
require.NoError(t, err)
switch reason {
case "forced":
require.NoError(t, d.Set([]byte("1"), nil, nil))
require.NoError(t, d.Flush())
require.NoError(t, d.Set([]byte("2"), nil, nil))
case "size":
largeValue := []byte(strings.Repeat("a", 100<<10))
require.NoError(t, d.Set([]byte("1"), largeValue, nil))
require.NoError(t, d.Set([]byte("2"), largeValue, nil))
require.NoError(t, d.Set([]byte("3"), largeValue, nil))
case "large-batch":
largeValue := []byte(strings.Repeat("a", int(d.largeBatchThreshold)))
require.NoError(t, d.Set([]byte("1"), nil, nil))
require.NoError(t, d.Set([]byte("2"), largeValue, nil))
require.NoError(t, d.Set([]byte("3"), nil, nil))
}
require.NoError(t, d.Close())
files, err := mem.List("")
require.NoError(t, err)
sort.Strings(files)
sstCount := 0
for _, fname := range files {
if strings.HasSuffix(fname, ".sst") {
sstCount++
}
}
require.Equal(t, 1, sstCount)
// Re-open the DB with a smaller memtable. Values for 1, 2 will fit in the first memtable;
// value for 3 will go in the next memtable; value for 4 will be in a flushable batch
// which will cause the previous memtable to be flushed; value for 5 will go in the next
// memtable
d, err = Open("", testingRandomized(t, &Options{
FS: mem,
MemTableSize: 300 << 10,
ReadOnly: readOnly,
}))
require.NoError(t, err)
require.NoError(t, d.Close())
})
}
})
}
}
// TestTwoWALReplayCorrupt tests WAL-replay behavior when the first of the two
// WALs is corrupted with an sstable checksum error. Replay must stop at the
// first WAL because otherwise we may violate point-in-time recovery
// semantics. See #864.
func TestTwoWALReplayCorrupt(t *testing.T) {
// Use the real filesystem so that we can seek and overwrite WAL data
// easily.
dir, err := os.MkdirTemp("", "wal-replay")
require.NoError(t, err)
defer os.RemoveAll(dir)
d, err := Open(dir, testingRandomized(t, &Options{
MemTableStopWritesThreshold: 4,
MemTableSize: 2048,
}))
require.NoError(t, err)
d.mu.Lock()
d.mu.compact.flushing = true
d.mu.Unlock()
require.NoError(t, d.Set([]byte("1"), []byte(strings.Repeat("a", 1024)), nil))
require.NoError(t, d.Set([]byte("2"), nil, nil))
d.mu.Lock()
d.mu.compact.flushing = false
d.mu.Unlock()
require.NoError(t, d.Close())
// We should have two WALs.
var logs []string
ls, err := vfs.Default.List(dir)
require.NoError(t, err)
for _, name := range ls {
if filepath.Ext(name) == ".log" {
logs = append(logs, name)
}
}
sort.Strings(logs)
if len(logs) < 2 {
t.Fatalf("expected at least two log files, found %d", len(logs))
}
// Corrupt the (n-1)th WAL by zeroing four bytes, 100 bytes from the end
// of the file.
f, err := os.OpenFile(filepath.Join(dir, logs[len(logs)-2]), os.O_RDWR, os.ModePerm)
require.NoError(t, err)
off, err := f.Seek(-100, 2)
require.NoError(t, err)
_, err = f.Write([]byte{0, 0, 0, 0})
require.NoError(t, err)
require.NoError(t, f.Close())
t.Logf("zeored four bytes in %s at offset %d\n", logs[len(logs)-2], off)
// Re-opening the database should detect and report the corruption.
_, err = Open(dir, nil)
require.Error(t, err, "pebble: corruption")
}
// TestTwoWALReplayCorrupt tests WAL-replay behavior when the first of the two
// WALs is corrupted with an sstable checksum error and the OPTIONS file does
// not enable the private strict_wal_tail option, indicating that the WAL was
// produced by a database that did not guarantee clean WAL tails. See #864.
func TestTwoWALReplayPermissive(t *testing.T) {
// Use the real filesystem so that we can seek and overwrite WAL data
// easily.
dir, err := os.MkdirTemp("", "wal-replay")
require.NoError(t, err)
defer os.RemoveAll(dir)
opts := &Options{
MemTableStopWritesThreshold: 4,
MemTableSize: 2048,
}
opts.testingRandomized(t)
opts.EnsureDefaults()
d, err := Open(dir, opts)
require.NoError(t, err)
d.mu.Lock()
d.mu.compact.flushing = true
d.mu.Unlock()
require.NoError(t, d.Set([]byte("1"), []byte(strings.Repeat("a", 1024)), nil))
require.NoError(t, d.Set([]byte("2"), nil, nil))
d.mu.Lock()
d.mu.compact.flushing = false
d.mu.Unlock()
require.NoError(t, d.Close())
// We should have two WALs.
var logs []string
var optionFilename string
ls, err := vfs.Default.List(dir)
require.NoError(t, err)
for _, name := range ls {
if filepath.Ext(name) == ".log" {
logs = append(logs, name)
}
if strings.HasPrefix(filepath.Base(name), "OPTIONS") {
optionFilename = name
}
}
sort.Strings(logs)
if len(logs) < 2 {
t.Fatalf("expected at least two log files, found %d", len(logs))
}
// Corrupt the (n-1)th WAL by zeroing four bytes, 100 bytes from the end
// of the file.
f, err := os.OpenFile(filepath.Join(dir, logs[len(logs)-2]), os.O_RDWR, os.ModePerm)
require.NoError(t, err)
off, err := f.Seek(-100, 2)
require.NoError(t, err)
_, err = f.Write([]byte{0, 0, 0, 0})
require.NoError(t, err)
require.NoError(t, f.Close())
t.Logf("zeored four bytes in %s at offset %d\n", logs[len(logs)-2], off)
// Remove the OPTIONS file containing the strict_wal_tail option.
require.NoError(t, vfs.Default.Remove(filepath.Join(dir, optionFilename)))
// Re-opening the database should not report the corruption.
d, err = Open(dir, nil)
require.NoError(t, err)
require.NoError(t, d.Close())
}
// TestCrashOpenCrashAfterWALCreation tests a database that exits
// ungracefully, begins recovery, creates the new WAL but promptly exits
// ungracefully again.
//
// This sequence has the potential to be problematic with the strict_wal_tail
// behavior because the first crash's WAL has an unclean tail. By the time the
// new WAL is created, the current manifest's MinUnflushedLogNum must be
// higher than the previous WAL.
func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
fs := vfs.NewStrictMem()
getLogs := func() (logs []string) {
ls, err := fs.List("")
require.NoError(t, err)
for _, name := range ls {
if filepath.Ext(name) == ".log" {
logs = append(logs, name)
}
}
return logs
}
{
d, err := Open("", testingRandomized(t, &Options{FS: fs}))
require.NoError(t, err)
require.NoError(t, d.Set([]byte("abc"), nil, Sync))
// Ignore syncs during close to simulate a crash. This will leave the WAL
// without an EOF trailer. It won't be an 'unclean tail' yet since the
// log file was not recycled, but we'll fix that down below.
fs.SetIgnoreSyncs(true)
require.NoError(t, d.Close())
fs.ResetToSyncedState()
fs.SetIgnoreSyncs(false)
}
// There should be one WAL.
logs := getLogs()
if len(logs) != 1 {
t.Fatalf("expected one log file, found %d", len(logs))
}
// The one WAL file doesn't have an EOF trailer, but since it wasn't
// recycled it won't have garbage at the end. Rewrite it so that it has
// the same contents it currently has, followed by garbage.
{
f, err := fs.Open(logs[0])
require.NoError(t, err)
b, err := io.ReadAll(f)
require.NoError(t, err)
require.NoError(t, f.Close())
f, err = fs.Create(logs[0])
require.NoError(t, err)
_, err = f.Write(b)
require.NoError(t, err)
_, err = f.Write([]byte{0xde, 0xad, 0xbe, 0xef})
require.NoError(t, err)
require.NoError(t, f.Sync())
require.NoError(t, f.Close())
dir, err := fs.OpenDir("")
require.NoError(t, err)
require.NoError(t, dir.Sync())
require.NoError(t, dir.Close())
}
// Open the database again (with syncs respected again). Wrap the
// filesystem with an errorfs that will turn off syncs after a new .log
// file is created and after a subsequent directory sync occurs. This
// simulates a crash after the new log file is created and synced.
{
var walCreated, dirSynced atomic.Bool
d, err := Open("", &Options{
FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op) error {
if dirSynced.Load() {
fs.SetIgnoreSyncs(true)
}
if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".log" {
walCreated.Store(true)
}
// Record when there's a sync of the data directory after the
// WAL was created. The data directory will have an empty
// path because that's what we passed into Open.
if op.Kind == errorfs.OpFileSync && op.Path == "" && walCreated.Load() {
dirSynced.Store(true)
}
return nil
})),
})
require.NoError(t, err)
require.NoError(t, d.Close())
}
fs.ResetToSyncedState()
fs.SetIgnoreSyncs(false)
if n := len(getLogs()); n != 2 {
t.Fatalf("expected two logs, found %d\n", n)
}
// Finally, open the database with syncs enabled.
d, err := Open("", testingRandomized(t, &Options{FS: fs}))
require.NoError(t, err)
require.NoError(t, d.Close())
}
// TestOpenWALReplayReadOnlySeqNums tests opening a database:
// - in read-only mode
// - with multiple unflushed log files that must replayed
// - a MANIFEST that sets the last sequence number to a number greater than
// the unflushed log files
//
// See cockroachdb/cockroach#48660.
func TestOpenWALReplayReadOnlySeqNums(t *testing.T) {
const root = ""
mem := vfs.NewMem()
copyFiles := func(srcDir, dstDir string) {
files, err := mem.List(srcDir)
require.NoError(t, err)
for _, f := range files {
require.NoError(t, vfs.Copy(mem, mem.PathJoin(srcDir, f), mem.PathJoin(dstDir, f)))
}
}
// Create a new database under `/original` with a couple sstables.
dir := mem.PathJoin(root, "original")
d, err := Open(dir, testingRandomized(t, &Options{FS: mem}))
require.NoError(t, err)
require.NoError(t, d.Set([]byte("a"), nil, nil))
require.NoError(t, d.Flush())
require.NoError(t, d.Set([]byte("a"), nil, nil))
require.NoError(t, d.Flush())
// Prevent flushes so that multiple unflushed log files build up.
d.mu.Lock()
d.mu.compact.flushing = true
d.mu.Unlock()
require.NoError(t, d.Set([]byte("b"), nil, nil))
d.AsyncFlush()
require.NoError(t, d.Set([]byte("c"), nil, nil))
d.AsyncFlush()
require.NoError(t, d.Set([]byte("e"), nil, nil))
// Manually compact some of the key space so that the latest `logSeqNum` is
// written to the MANIFEST. This produces a MANIFEST where the `logSeqNum`
// is greater than the sequence numbers contained in the
// `minUnflushedLogNum` log file
require.NoError(t, d.Compact([]byte("a"), []byte("a\x00"), false))
d.mu.Lock()
for d.mu.compact.compactingCount > 0 {
d.mu.compact.cond.Wait()
}
d.mu.Unlock()
d.TestOnlyWaitForCleaning()
// While the MANIFEST is still in this state, copy all the files in the
// database to a new directory.
replayDir := mem.PathJoin(root, "replay")
require.NoError(t, mem.MkdirAll(replayDir, os.ModePerm))
copyFiles(dir, replayDir)
d.mu.Lock()
d.mu.compact.flushing = false
d.mu.Unlock()
require.NoError(t, d.Close())
// Open the copy of the database in read-only mode. Since we copied all
// the files before the flushes were allowed to complete, there should be
// multiple unflushed log files that need to replay. Since the manual
// compaction completed, the `logSeqNum` read from the manifest should be
// greater than the unflushed log files' sequence numbers.
d, err = Open(replayDir, testingRandomized(t, &Options{
FS: mem,
ReadOnly: true,
}))
require.NoError(t, err)
require.NoError(t, d.Close())
}
func TestOpenWALReplayMemtableGrowth(t *testing.T) {
mem := vfs.NewMem()
const memTableSize = 64 * 1024 * 1024
opts := &Options{
MemTableSize: memTableSize,
FS: mem,
}
opts.testingRandomized(t)
func() {
db, err := Open("", opts)
require.NoError(t, err)
defer db.Close()
b := db.NewBatch()
defer b.Close()
key := make([]byte, 8)
val := make([]byte, 16*1024*1024)
b.Set(key, val, nil)
require.NoError(t, db.Apply(b, Sync))
}()
db, err := Open("", opts)
require.NoError(t, err)
db.Close()
}
func TestGetVersion(t *testing.T) {
mem := vfs.NewMem()
opts := &Options{
FS: mem,
}
opts.testingRandomized(t)
// Case 1: No options file.
version, err := GetVersion("", mem)
require.NoError(t, err)
require.Empty(t, version)
// Case 2: Pebble created file.
db, err := Open("", opts)
require.NoError(t, err)
require.NoError(t, db.Close())
version, err = GetVersion("", mem)
require.NoError(t, err)
require.Equal(t, "0.1", version)
// Case 3: Manually created OPTIONS file with a higher number.
highestOptionsNum := FileNum(0)
ls, err := mem.List("")
require.NoError(t, err)
for _, filename := range ls {
ft, fn, ok := base.ParseFilename(mem, filename)
if !ok {
continue
}
switch ft {
case fileTypeOptions:
if fn.FileNum() > highestOptionsNum {
highestOptionsNum = fn.FileNum()
}
}
}
f, _ := mem.Create(fmt.Sprintf("OPTIONS-%d", highestOptionsNum+1))
_, err = f.Write([]byte("[Version]\n pebble_version=0.2\n"))
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)
version, err = GetVersion("", mem)
require.NoError(t, err)
require.Equal(t, "0.2", version)
// Case 4: Manually created OPTIONS file with a RocksDB number.
f, _ = mem.Create(fmt.Sprintf("OPTIONS-%d", highestOptionsNum+2))
_, err = f.Write([]byte("[Version]\n rocksdb_version=6.2.1\n"))
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)
version, err = GetVersion("", mem)
require.NoError(t, err)
require.Equal(t, "rocksdb v6.2.1", version)
}
func TestRocksDBNoFlushManifest(t *testing.T) {
mem := vfs.NewMem()
// Have the comparer and merger names match what's in the testdata
// directory.
comparer := *DefaultComparer
merger := *DefaultMerger
comparer.Name = "cockroach_comparator"
merger.Name = "cockroach_merge_operator"
opts := &Options{
FS: mem,
Comparer: &comparer,
Merger: &merger,
}
// rocksdb-ingest-only is a RocksDB-generated db directory that has not had
// a single flush yet, only ingestion operations. The manifest contains
// a next-log-num but no log-num entry. Ensure that pebble can read these
// directories without an issue.
_, err := vfs.Clone(vfs.Default, mem, "testdata/rocksdb-ingest-only", "testdata")
require.NoError(t, err)
db, err := Open("testdata", opts)
require.NoError(t, err)
defer db.Close()
val, closer, err := db.Get([]byte("ajulxeiombjiyw\x00\x00\x00\x00\x00\x00\x00\x01\x12\x09"))
require.NoError(t, err)
require.NotEmpty(t, val)
require.NoError(t, closer.Close())
}
func TestOpen_ErrorIfUnknownFormatVersion(t *testing.T) {
fs := vfs.NewMem()
d, err := Open("", &Options{
FS: fs,
FormatMajorVersion: FormatVersioned,
})
require.NoError(t, err)
require.NoError(t, d.Close())
// Move the marker to a version that does not exist.
m, _, err := atomicfs.LocateMarker(fs, "", formatVersionMarkerName)
require.NoError(t, err)
require.NoError(t, m.Move("999999"))
require.NoError(t, m.Close())
_, err = Open("", &Options{
FS: fs,
FormatMajorVersion: FormatVersioned,
})
require.Error(t, err)
require.EqualError(t, err, `pebble: database "" written in format major version 999999`)
}
// ensureFilesClosed updates the provided Options to wrap the filesystem. It
// returns a closure that when invoked fails the test if any files opened by the
// filesystem are not closed.
//
// This function is intended to be used in tests with defer.
//
// opts := &Options{FS: vfs.NewMem()}
// defer ensureFilesClosed(t, opts)()
// /* test code */
func ensureFilesClosed(t *testing.T, o *Options) func() {
fs := &closeTrackingFS{
FS: o.FS,
files: map[*closeTrackingFile]struct{}{},
}
o.FS = fs
return func() {
// fs.files should be empty if all the files were closed.
for f := range fs.files {
t.Errorf("An open file was never closed. Opened at:\n%s", f.stack)
}
}
}
type closeTrackingFS struct {
vfs.FS
files map[*closeTrackingFile]struct{}
}
func (fs *closeTrackingFS) wrap(file vfs.File, err error) (vfs.File, error) {
if err != nil {
return nil, err
}
f := &closeTrackingFile{
File: file,
fs: fs,
stack: debug.Stack(),
}
fs.files[f] = struct{}{}
return f, err
}
func (fs *closeTrackingFS) Create(name string) (vfs.File, error) {
return fs.wrap(fs.FS.Create(name))
}
func (fs *closeTrackingFS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) {
return fs.wrap(fs.FS.Open(name))
}
func (fs *closeTrackingFS) OpenDir(name string) (vfs.File, error) {
return fs.wrap(fs.FS.OpenDir(name))
}
func (fs *closeTrackingFS) ReuseForWrite(oldname, newname string) (vfs.File, error) {
return fs.wrap(fs.FS.ReuseForWrite(oldname, newname))
}
type closeTrackingFile struct {
vfs.File
fs *closeTrackingFS
stack []byte
}
func (f *closeTrackingFile) Close() error {
delete(f.fs.files, f)
return f.File.Close()
}
func TestCheckConsistency(t *testing.T) {
const dir = "./test"
mem := vfs.NewMem()
mem.MkdirAll(dir, 0755)
provider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(mem, dir))
require.NoError(t, err)
defer provider.Close()
cmp := base.DefaultComparer.Compare
fmtKey := base.DefaultComparer.FormatKey
parseMeta := func(s string) (*manifest.FileMetadata, error) {
if len(s) == 0 {
return nil, nil
}
parts := strings.Split(s, ":")
if len(parts) != 2 {
return nil, errors.Errorf("malformed table spec: %q", s)
}
fileNum, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
return nil, err
}
size, err := strconv.Atoi(strings.TrimSpace(parts[1]))
if err != nil {
return nil, err
}
m := &manifest.FileMetadata{
FileNum: base.FileNum(fileNum),
Size: uint64(size),
}
m.InitPhysicalBacking()
return m, nil
}
datadriven.RunTest(t, "testdata/version_check_consistency",
func(t *testing.T, d *datadriven.TestData) string {
switch d.Cmd {
case "check-consistency":
var filesByLevel [manifest.NumLevels][]*manifest.FileMetadata
var files *[]*manifest.FileMetadata
for _, data := range strings.Split(d.Input, "\n") {
switch data {
case "L0", "L1", "L2", "L3", "L4", "L5", "L6":
level, err := strconv.Atoi(data[1:])
if err != nil {
return err.Error()
}
files = &filesByLevel[level]
default:
m, err := parseMeta(data)
if err != nil {
return err.Error()
}
if m != nil {
*files = append(*files, m)
}
}
}
redactErr := false
for _, arg := range d.CmdArgs {
switch v := arg.String(); v {
case "redact":
redactErr = true
default:
return fmt.Sprintf("unknown argument: %q", v)
}
}
v := manifest.NewVersion(cmp, fmtKey, 0, filesByLevel)
err := checkConsistency(v, dir, provider)
if err != nil {
if redactErr {
redacted := redact.Sprint(err).Redact()
return string(redacted)
}
return err.Error()
}
return "OK"
case "build":
for _, data := range strings.Split(d.Input, "\n") {
m, err := parseMeta(data)
if err != nil {
return err.Error()
}
path := base.MakeFilepath(mem, dir, base.FileTypeTable, m.FileBacking.DiskFileNum)
_ = mem.Remove(path)
f, err := mem.Create(path)
if err != nil {
return err.Error()
}
_, err = f.Write(make([]byte, m.Size))
if err != nil {
return err.Error()
}
f.Close()
}
return ""
default:
return fmt.Sprintf("unknown command: %s", d.Cmd)
}
})
}
func TestOpenRatchetsNextFileNum(t *testing.T) {
mem := vfs.NewMem()
memShared := remote.NewInMem()
opts := &Options{FS: mem}
opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
"": memShared,
})
d, err := Open("", opts)
require.NoError(t, err)
d.SetCreatorID(1)
require.NoError(t, d.Set([]byte("foo"), []byte("value"), nil))
require.NoError(t, d.Set([]byte("bar"), []byte("value"), nil))
require.NoError(t, d.Flush())
require.NoError(t, d.Compact([]byte("a"), []byte("z"), false))
// Create a shared file with the newest file num and then close the db.
d.mu.Lock()
nextFileNum := d.mu.versions.getNextFileNum()
w, _, err := d.objProvider.Create(context.TODO(), fileTypeTable, nextFileNum.DiskFileNum(), objstorage.CreateOptions{PreferSharedStorage: true})
require.NoError(t, err)
require.NoError(t, w.Write([]byte("foobar")))
require.NoError(t, w.Finish())
require.NoError(t, d.objProvider.Sync())
d.mu.Unlock()
// Write one key and then close the db. This write will stay in the memtable,
// forcing the reopen to do a compaction on open.
require.NoError(t, d.Set([]byte("foo1"), []byte("value"), nil))
require.NoError(t, d.Close())
// Reopen db. Compactions should happen without error.
d, err = Open("", opts)
require.NoError(t, err)
require.NoError(t, d.Set([]byte("foo2"), []byte("value"), nil))
require.NoError(t, d.Set([]byte("bar2"), []byte("value"), nil))
require.NoError(t, d.Flush())
require.NoError(t, d.Compact([]byte("a"), []byte("z"), false))
}