ceremonyclient/pebble/sstable/value_block.go

958 lines
33 KiB
Go
Raw Normal View History

2024-01-03 07:31:42 +00:00
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sstable
import (
"context"
"encoding/binary"
"io"
"sync"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"golang.org/x/exp/rand"
)
// Value blocks are supported in TableFormatPebblev3.
//
// 1. Motivation and overview
//
// Value blocks are a mechanism designed for sstables storing MVCC data, where
// there can be many versions of a key that need to be kept, but only the
// latest value is typically read (see the documentation for Comparer.Split
// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions,
// which can be eagerly thrown away (except when there are snapshots), MVCC
// versions are long-lived (e.g. default CockroachDB garbage collection
// threshold for older versions is 24 hours) and can significantly slow down
// reads. We have seen CockroachDB production workloads with very slow reads
// due to:
// - 100s of versions for each key in a table.
//
// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a
// real key-value pair, followed by a key-value pair whose value (usually
// with zero byte length) indicates it is an MVCC tombstone.
//
// The value blocks mechanism attempts to improve read throughput in these
// cases when the key size is smaller than the value sizes of older versions.
// This is done by moving the value of an older version to a value block in a
// different part of the sstable. This improves spatial locality of the data
// being read by the workload, which increases caching effectiveness.
//
// Additionally, even when the key size is not smaller than the value of older
// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3
// stores the result of key comparisons done at write time inside the sstable,
// which makes stepping from one key prefix to the next prefix (i.e., skipping
// over older versions of a MVCC key) more efficient by avoiding key
// comparisons and key decoding. See the results in
// https://github.com/cockroachdb/pebble/pull/2149 and more details in the
// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also
// visible in end-to-end CockroachDB tests, as outlined in
// https://github.com/cockroachdb/cockroach/pull/96652.
//
// In TableFormatPebblev3, each SET has a one byte value prefix that tells us
// whether the value is in-place or in a value block. This 1 byte prefix
// encodes additional information:
//
// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB
// uses it to represent whether the value is a tombstone or not. This avoids
// the need to fetch a value from the value block if the caller only wants
// to figure out whether it is an MVCC tombstone. The length of the value is
// another attribute that the caller can be interested in, and it is also
// accessible without reading the value in the value block (see the value
// handle in the details section).
//
// - SET-same-prefix: this enables the aforementioned optimization when
// stepping from one key prefix to the next key prefix.
//
// We further optimize this iteration over prefixes by using the restart
// points in a block to encode whether the SET at a restart point has the same
// prefix since the last restart point. This allows us to skip over restart
// points within the same block. See the comment in blockWriter, and how both
// SET-same-prefix and the restart point information is used in
// blockIter.nextPrefixV3.
//
// This flexibility of values that are in-place or in value blocks requires
// flexibility in the iterator interface. The InternalIterator interface
// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator
// allows the caller to ask for a LazyValue. See lazy_value.go for details,
// including the memory lifetime management.
//
// For historical discussions about this feature, see the issue
// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in
// https://github.com/cockroachdb/pebble/pull/1443.
//
// The code in this file mainly covers value block and related encodings. We
// discuss these in the next section.
//
// 2. Details
//
// Note that the notion of the latest value is local to the sstable. It is
// possible that that latest value has been deleted by a sstable in a higher
// level, and what is the latest value from the perspective of the whole LSM
// is an older MVCC version. This only affects performance and not
// correctness. This local knowledge is also why we continue to store these
// older versions in the same sstable -- we need to be able to conveniently
// read them. The code in this file is agnostic to the policy regarding what
// should be stored in value blocks -- it allows even the latest MVCC version
// to be stored in a value block. The policy decision in made in the
// sstable.Writer. See Writer.makeAddPointDecisionV3.
//
// Data blocks contain two kinds of SET keys: those with in-place values and
// those with a value handle. To distinguish these two cases we use a single
// byte prefix (valuePrefix). This single byte prefix is split into multiple
// parts, where nb represents information that is encoded in n bits.
//
// +---------------+--------------------+-----------+--------------------+
// | value-kind 2b | SET-same-prefix 1b | unused 2b | short-attribute 3b |
// +---------------+--------------------+-----------+--------------------+
//
// The 2 bit value-kind specifies whether this is an in-place value or a value
// handle pointing to a value block. We use 2 bits here for future
// representation of values that are in separate files. The 1 bit
// SET-same-prefix is true if this key is a SET and is immediately preceded by
// a SET that shares the same prefix. The 3 bit short-attribute is described
// in base.ShortAttribute -- it stores user-defined attributes about the
// value. It is unused for in-place values.
//
// Value Handle and Value Blocks:
// valueHandles refer to values in value blocks. Value blocks are simpler than
// normal data blocks (that contain key-value pairs, and allow for binary
// search), which makes them cheap for value retrieval purposes. A valueHandle
// is a tuple (valueLen, blockNum, offsetInBlock), where blockNum is the 0
// indexed value block number and offsetInBlock is the byte offset in that
// block containing the value. The valueHandle.valueLen is included since
// there are multiple use cases in CockroachDB that need the value length but
// not the value, for which we can avoid reading the value in the value block
// (see
// https://github.com/cockroachdb/pebble/issues/1170#issuecomment-958203245).
//
// A value block has a checksum like other blocks, and is optionally
// compressed. An uncompressed value block is a sequence of values with no
// separator or length (we rely on the valueHandle to demarcate). The
// valueHandle.offsetInBlock points to the value, of length
// valueHandle.valueLen. While writing a sstable, all the (possibly
// compressed) value blocks need to be held in-memory until they can be
// written. Value blocks are placed after the "meta rangedel" and "meta range
// key" blocks since value blocks are considered less likely to be read.
//
// Meta Value Index Block:
// Since the (key, valueHandle) pair are written before there is any knowledge
// of the byte offset of the value block in the file, or its compressed
// length, we need another lookup to map the valueHandle.blockNum to the
// information needed to read it from the file. This information is provided
// by the "value index block". The "value index block" is referred to by the
// metaindex block. The design intentionally avoids making the "value index
// block" a general purpose key-value block, since each caller wants to lookup
// the information for a particular blockNum (there is no need for SeekGE
// etc.). Instead, this index block stores a sequence of (blockNum,
// blockOffset, blockLength) tuples, where the blockNums are consecutive
// integers, and the tuples are encoded with a fixed width encoding. This
// allows a reader to find the tuple for block K by looking at the offset
// K*fixed-width. The fixed width for each field is decided by looking at the
// maximum value of each of these fields. As a concrete example of a large
// sstable with many value blocks, we constructed a 100MB sstable with many
// versions and had 2475 value blocks (~32KB each). This sstable had this
// tuple encoded using 2+4+2=8 bytes, which means the uncompressed value index
// block was 2475*8=~19KB, which is modest. Therefore, we don't support more
// than one value index block. Consider the example of 2 byte blockNum, 4 byte
// blockOffset and 2 byte blockLen. The value index block will look like:
//
// +---------------+------------------+---------------+
// | blockNum (2B) | blockOffset (4B) | blockLen (2B) |
// +---------------+------------------+---------------+
// | 0 | 7,123,456 | 30,000 |
// +---------------+------------------+---------------+
// | 1 | 7,153,456 | 20,000 |
// +---------------+------------------+---------------+
// | 2 | 7,173,456 | 25,567 |
// +---------------+------------------+---------------+
// | .... | ... | ... |
//
//
// The metaindex block contains the valueBlocksIndexHandle which in addition
// to the BlockHandle also specifies the widths of these tuple fields. In the
// above example, the
// valueBlockIndexHandle.{blockNumByteLength,blockOffsetByteLength,blockLengthByteLength}
// will be (2,4,2).
// valueHandle is stored with a key when the value is in a value block. This
// handle is the pointer to that value.
type valueHandle struct {
valueLen uint32
blockNum uint32
offsetInBlock uint32
}
// valuePrefix is the single byte prefix for either the in-place value or the
// encoded valueHandle. It encoded multiple kinds of information.
type valuePrefix byte
const (
// 2 most-significant bits of valuePrefix encodes the value-kind.
valueKindMask valuePrefix = '\xC0'
valueKindIsValueHandle valuePrefix = '\x80'
valueKindIsInPlaceValue valuePrefix = '\x00'
// 1 bit indicates SET has same key prefix as immediately preceding key that
// is also a SET. If the immediately preceding key in the same block is a
// SET, AND this bit is 0, the prefix must have changed.
//
// Note that the current policy of only storing older MVCC versions in value
// blocks means that valueKindIsValueHandle => SET has same prefix. But no
// code should rely on this behavior. Also, SET has same prefix does *not*
// imply valueKindIsValueHandle.
setHasSameKeyPrefixMask valuePrefix = '\x20'
// 3 least-significant bits for the user-defined base.ShortAttribute.
// Undefined for valueKindIsInPlaceValue.
userDefinedShortAttributeMask valuePrefix = '\x07'
)
// valueHandle fields are varint encoded, so maximum 5 bytes each, plus 1 byte
// for the valuePrefix. This could alternatively be group varint encoded, but
// experiments were inconclusive
// (https://github.com/cockroachdb/pebble/pull/1443#issuecomment-1270298802).
const valueHandleMaxLen = 5*3 + 1
// Assert blockHandleLikelyMaxLen >= valueHandleMaxLen.
const _ = uint(blockHandleLikelyMaxLen - valueHandleMaxLen)
func encodeValueHandle(dst []byte, v valueHandle) int {
n := 0
n += binary.PutUvarint(dst[n:], uint64(v.valueLen))
n += binary.PutUvarint(dst[n:], uint64(v.blockNum))
n += binary.PutUvarint(dst[n:], uint64(v.offsetInBlock))
return n
}
func makePrefixForValueHandle(setHasSameKeyPrefix bool, attribute base.ShortAttribute) valuePrefix {
prefix := valueKindIsValueHandle | valuePrefix(attribute)
if setHasSameKeyPrefix {
prefix = prefix | setHasSameKeyPrefixMask
}
return prefix
}
func makePrefixForInPlaceValue(setHasSameKeyPrefix bool) valuePrefix {
prefix := valueKindIsInPlaceValue
if setHasSameKeyPrefix {
prefix = prefix | setHasSameKeyPrefixMask
}
return prefix
}
func isValueHandle(b valuePrefix) bool {
return b&valueKindMask == valueKindIsValueHandle
}
// REQUIRES: isValueHandle(b)
func getShortAttribute(b valuePrefix) base.ShortAttribute {
return base.ShortAttribute(b & userDefinedShortAttributeMask)
}
func setHasSamePrefix(b valuePrefix) bool {
return b&setHasSameKeyPrefixMask == setHasSameKeyPrefixMask
}
func decodeLenFromValueHandle(src []byte) (uint32, []byte) {
ptr := unsafe.Pointer(&src[0])
var v uint32
if a := *((*uint8)(ptr)); a < 128 {
v = uint32(a)
src = src[1:]
} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
v = uint32(b)<<7 | uint32(a)
src = src[2:]
} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
src = src[3:]
} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
src = src[4:]
} else {
d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
src = src[5:]
}
return v, src
}
func decodeRemainingValueHandle(src []byte) valueHandle {
var vh valueHandle
ptr := unsafe.Pointer(&src[0])
// Manually inlined uvarint decoding. Saves ~25% in benchmarks. Unrolling
// a loop for i:=0; i<2; i++, saves ~6%.
var v uint32
if a := *((*uint8)(ptr)); a < 128 {
v = uint32(a)
ptr = unsafe.Pointer(uintptr(ptr) + 1)
} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
v = uint32(b)<<7 | uint32(a)
ptr = unsafe.Pointer(uintptr(ptr) + 2)
} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
ptr = unsafe.Pointer(uintptr(ptr) + 3)
} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
ptr = unsafe.Pointer(uintptr(ptr) + 4)
} else {
d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
ptr = unsafe.Pointer(uintptr(ptr) + 5)
}
vh.blockNum = v
if a := *((*uint8)(ptr)); a < 128 {
v = uint32(a)
} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
v = uint32(b)<<7 | uint32(a)
} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
} else {
d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
}
vh.offsetInBlock = v
return vh
}
func decodeValueHandle(src []byte) valueHandle {
valLen, src := decodeLenFromValueHandle(src)
vh := decodeRemainingValueHandle(src)
vh.valueLen = valLen
return vh
}
// valueBlocksIndexHandle is placed in the metaindex if there are any value
// blocks. If there are no value blocks, there is no value blocks index, and
// no entry in the metaindex. Note that the lack of entry in the metaindex
// should not be used to ascertain whether the values are prefixed, since the
// former is an emergent property of the data that was written and not known
// until all the key-value pairs in the sstable are written.
type valueBlocksIndexHandle struct {
h BlockHandle
blockNumByteLength uint8
blockOffsetByteLength uint8
blockLengthByteLength uint8
}
const valueBlocksIndexHandleMaxLen = blockHandleMaxLenWithoutProperties + 3
// Assert blockHandleLikelyMaxLen >= valueBlocksIndexHandleMaxLen.
const _ = uint(blockHandleLikelyMaxLen - valueBlocksIndexHandleMaxLen)
func encodeValueBlocksIndexHandle(dst []byte, v valueBlocksIndexHandle) int {
n := encodeBlockHandle(dst, v.h)
dst[n] = v.blockNumByteLength
n++
dst[n] = v.blockOffsetByteLength
n++
dst[n] = v.blockLengthByteLength
n++
return n
}
func decodeValueBlocksIndexHandle(src []byte) (valueBlocksIndexHandle, int, error) {
var vbih valueBlocksIndexHandle
var n int
vbih.h, n = decodeBlockHandle(src)
if n <= 0 {
return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
}
if len(src) != n+3 {
return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
}
vbih.blockNumByteLength = src[n]
vbih.blockOffsetByteLength = src[n+1]
vbih.blockLengthByteLength = src[n+2]
return vbih, n + 3, nil
}
type valueBlocksAndIndexStats struct {
numValueBlocks uint64
numValuesInValueBlocks uint64
// Includes both value blocks and value index block.
valueBlocksAndIndexSize uint64
}
// valueBlockWriter writes a sequence of value blocks, and the value blocks
// index, for a sstable.
type valueBlockWriter struct {
// The configured uncompressed block size and size threshold
blockSize, blockSizeThreshold int
// Configured compression.
compression Compression
// checksummer with configured checksum type.
checksummer checksummer
// Block finished callback.
blockFinishedFunc func(compressedSize int)
// buf is the current block being written to (uncompressed).
buf *blockBuffer
// compressedBuf is used for compressing the block.
compressedBuf *blockBuffer
// Sequence of blocks that are finished.
blocks []blockAndHandle
// Cumulative value block bytes written so far.
totalBlockBytes uint64
numValues uint64
}
type blockAndHandle struct {
block *blockBuffer
handle BlockHandle
compressed bool
}
type blockBuffer struct {
b []byte
}
// Pool of block buffers that should be roughly the blockSize.
var uncompressedValueBlockBufPool = sync.Pool{
New: func() interface{} {
return &blockBuffer{}
},
}
// Pool of block buffers for compressed value blocks. These may widely vary in
// size based on compression ratios.
var compressedValueBlockBufPool = sync.Pool{
New: func() interface{} {
return &blockBuffer{}
},
}
func releaseToValueBlockBufPool(pool *sync.Pool, b *blockBuffer) {
// Don't pool buffers larger than 128KB, in case we had some rare large
// values.
if len(b.b) > 128*1024 {
return
}
if invariants.Enabled {
// Set the bytes to a random value. Cap the number of bytes being
// randomized to prevent test timeouts.
length := cap(b.b)
if length > 1000 {
length = 1000
}
b.b = b.b[:length:length]
rand.Read(b.b)
}
pool.Put(b)
}
var valueBlockWriterPool = sync.Pool{
New: func() interface{} {
return &valueBlockWriter{}
},
}
func newValueBlockWriter(
blockSize int,
blockSizeThreshold int,
compression Compression,
checksumType ChecksumType,
// compressedSize should exclude the block trailer.
blockFinishedFunc func(compressedSize int),
) *valueBlockWriter {
w := valueBlockWriterPool.Get().(*valueBlockWriter)
*w = valueBlockWriter{
blockSize: blockSize,
blockSizeThreshold: blockSizeThreshold,
compression: compression,
checksummer: checksummer{
checksumType: checksumType,
},
blockFinishedFunc: blockFinishedFunc,
buf: uncompressedValueBlockBufPool.Get().(*blockBuffer),
compressedBuf: compressedValueBlockBufPool.Get().(*blockBuffer),
blocks: w.blocks[:0],
}
w.buf.b = w.buf.b[:0]
w.compressedBuf.b = w.compressedBuf.b[:0]
return w
}
func releaseValueBlockWriter(w *valueBlockWriter) {
for i := range w.blocks {
if w.blocks[i].compressed {
releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.blocks[i].block)
} else {
releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.blocks[i].block)
}
w.blocks[i].block = nil
}
if w.buf != nil {
releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.buf)
}
if w.compressedBuf != nil {
releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.compressedBuf)
}
*w = valueBlockWriter{
blocks: w.blocks[:0],
}
valueBlockWriterPool.Put(w)
}
func (w *valueBlockWriter) addValue(v []byte) (valueHandle, error) {
if invariants.Enabled && len(v) == 0 {
return valueHandle{}, errors.Errorf("cannot write empty value to value block")
}
w.numValues++
blockLen := len(w.buf.b)
valueLen := len(v)
if blockLen >= w.blockSize ||
(blockLen > w.blockSizeThreshold && blockLen+valueLen > w.blockSize) {
// Block is not currently empty and adding this value will become too big,
// so finish this block.
w.compressAndFlush()
blockLen = len(w.buf.b)
if invariants.Enabled && blockLen != 0 {
panic("blockLen of new block should be 0")
}
}
vh := valueHandle{
valueLen: uint32(valueLen),
blockNum: uint32(len(w.blocks)),
offsetInBlock: uint32(blockLen),
}
blockLen = int(vh.offsetInBlock + vh.valueLen)
if cap(w.buf.b) < blockLen {
size := 2 * cap(w.buf.b)
if size < 1024 {
size = 1024
}
for size < blockLen {
size *= 2
}
buf := make([]byte, blockLen, size)
_ = copy(buf, w.buf.b)
w.buf.b = buf
} else {
w.buf.b = w.buf.b[:blockLen]
}
buf := w.buf.b[vh.offsetInBlock:]
n := copy(buf, v)
if n != len(buf) {
panic("incorrect length computation")
}
return vh, nil
}
func (w *valueBlockWriter) compressAndFlush() {
// Compress the buffer, discarding the result if the improvement isn't at
// least 12.5%.
blockType := noCompressionBlockType
b := w.buf
if w.compression != NoCompression {
blockType, w.compressedBuf.b =
compressBlock(w.compression, w.buf.b, w.compressedBuf.b[:cap(w.compressedBuf.b)])
if len(w.compressedBuf.b) < len(w.buf.b)-len(w.buf.b)/8 {
b = w.compressedBuf
} else {
blockType = noCompressionBlockType
}
}
n := len(b.b)
if n+blockTrailerLen > cap(b.b) {
block := make([]byte, n+blockTrailerLen)
copy(block, b.b)
b.b = block
} else {
b.b = b.b[:n+blockTrailerLen]
}
b.b[n] = byte(blockType)
w.computeChecksum(b.b)
bh := BlockHandle{Offset: w.totalBlockBytes, Length: uint64(n)}
w.totalBlockBytes += uint64(len(b.b))
// blockFinishedFunc length excludes the block trailer.
w.blockFinishedFunc(n)
compressed := blockType != noCompressionBlockType
w.blocks = append(w.blocks, blockAndHandle{
block: b,
handle: bh,
compressed: compressed,
})
// Handed off a buffer to w.blocks, so need get a new one.
if compressed {
w.compressedBuf = compressedValueBlockBufPool.Get().(*blockBuffer)
} else {
w.buf = uncompressedValueBlockBufPool.Get().(*blockBuffer)
}
w.buf.b = w.buf.b[:0]
}
func (w *valueBlockWriter) computeChecksum(block []byte) {
n := len(block) - blockTrailerLen
checksum := w.checksummer.checksum(block[:n], block[n:n+1])
binary.LittleEndian.PutUint32(block[n+1:], checksum)
}
func (w *valueBlockWriter) finish(
writer io.Writer, fileOffset uint64,
) (valueBlocksIndexHandle, valueBlocksAndIndexStats, error) {
if len(w.buf.b) > 0 {
w.compressAndFlush()
}
n := len(w.blocks)
if n == 0 {
return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, nil
}
largestOffset := uint64(0)
largestLength := uint64(0)
for i := range w.blocks {
_, err := writer.Write(w.blocks[i].block.b)
if err != nil {
return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
}
w.blocks[i].handle.Offset += fileOffset
largestOffset = w.blocks[i].handle.Offset
if largestLength < w.blocks[i].handle.Length {
largestLength = w.blocks[i].handle.Length
}
}
vbihOffset := fileOffset + w.totalBlockBytes
vbih := valueBlocksIndexHandle{
h: BlockHandle{
Offset: vbihOffset,
},
blockNumByteLength: uint8(lenLittleEndian(uint64(n - 1))),
blockOffsetByteLength: uint8(lenLittleEndian(largestOffset)),
blockLengthByteLength: uint8(lenLittleEndian(largestLength)),
}
var err error
if vbih, err = w.writeValueBlocksIndex(writer, vbih); err != nil {
return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
}
stats := valueBlocksAndIndexStats{
numValueBlocks: uint64(n),
numValuesInValueBlocks: w.numValues,
valueBlocksAndIndexSize: w.totalBlockBytes + vbih.h.Length + blockTrailerLen,
}
return vbih, stats, err
}
func (w *valueBlockWriter) writeValueBlocksIndex(
writer io.Writer, h valueBlocksIndexHandle,
) (valueBlocksIndexHandle, error) {
blockLen :=
int(h.blockNumByteLength+h.blockOffsetByteLength+h.blockLengthByteLength) * len(w.blocks)
h.h.Length = uint64(blockLen)
blockLen += blockTrailerLen
var buf []byte
if cap(w.buf.b) < blockLen {
buf = make([]byte, blockLen)
w.buf.b = buf
} else {
buf = w.buf.b[:blockLen]
}
b := buf
for i := range w.blocks {
littleEndianPut(uint64(i), b, int(h.blockNumByteLength))
b = b[int(h.blockNumByteLength):]
littleEndianPut(w.blocks[i].handle.Offset, b, int(h.blockOffsetByteLength))
b = b[int(h.blockOffsetByteLength):]
littleEndianPut(w.blocks[i].handle.Length, b, int(h.blockLengthByteLength))
b = b[int(h.blockLengthByteLength):]
}
if len(b) != blockTrailerLen {
panic("incorrect length calculation")
}
b[0] = byte(noCompressionBlockType)
w.computeChecksum(buf)
if _, err := writer.Write(buf); err != nil {
return valueBlocksIndexHandle{}, err
}
return h, nil
}
// littleEndianPut writes v to b using little endian encoding, under the
// assumption that v can be represented using n bytes.
func littleEndianPut(v uint64, b []byte, n int) {
_ = b[n-1] // bounds check
for i := 0; i < n; i++ {
b[i] = byte(v)
v = v >> 8
}
}
// lenLittleEndian returns the minimum number of bytes needed to encode v
// using little endian encoding.
func lenLittleEndian(v uint64) int {
n := 0
for i := 0; i < 8; i++ {
n++
v = v >> 8
if v == 0 {
break
}
}
return n
}
func littleEndianGet(b []byte, n int) uint64 {
_ = b[n-1] // bounds check
v := uint64(b[0])
for i := 1; i < n; i++ {
v |= uint64(b[i]) << (8 * i)
}
return v
}
// UserKeyPrefixBound represents a [Lower,Upper) bound of user key prefixes.
// If both are nil, there is no bound specified. Else, Compare(Lower,Upper)
// must be < 0.
type UserKeyPrefixBound struct {
// Lower is a lower bound user key prefix.
Lower []byte
// Upper is an upper bound user key prefix.
Upper []byte
}
// IsEmpty returns true iff the bound is empty.
func (ukb *UserKeyPrefixBound) IsEmpty() bool {
return len(ukb.Lower) == 0 && len(ukb.Upper) == 0
}
type blockProviderWhenOpen interface {
readBlockForVBR(
h BlockHandle, stats *base.InternalIteratorStats,
) (bufferHandle, error)
}
type blockProviderWhenClosed struct {
rp ReaderProvider
r *Reader
}
func (bpwc *blockProviderWhenClosed) open() error {
var err error
bpwc.r, err = bpwc.rp.GetReader()
return err
}
func (bpwc *blockProviderWhenClosed) close() {
bpwc.rp.Close()
bpwc.r = nil
}
func (bpwc blockProviderWhenClosed) readBlockForVBR(
h BlockHandle, stats *base.InternalIteratorStats,
) (bufferHandle, error) {
// This is rare, since most block reads happen when the corresponding
// sstable iterator is open. So we are willing to sacrifice a proper context
// for tracing.
//
// TODO(sumeer): consider fixing this. See
// https://github.com/cockroachdb/pebble/pull/3065#issue-1991175365 for an
// alternative.
ctx := objiotracing.WithBlockType(context.Background(), objiotracing.ValueBlock)
// TODO(jackson,sumeer): Consider whether to use a buffer pool in this case.
// The bpwc is not allowed to outlive the iterator tree, so it cannot
// outlive the buffer pool.
return bpwc.r.readBlock(
ctx, h, nil, nil, stats, nil /* iterStats */, nil /* buffer pool */)
}
// ReaderProvider supports the implementation of blockProviderWhenClosed.
// GetReader and Close can be called multiple times in pairs.
type ReaderProvider interface {
GetReader() (r *Reader, err error)
Close()
}
// TrivialReaderProvider implements ReaderProvider for a Reader that will
// outlive the top-level iterator in the iterator tree.
type TrivialReaderProvider struct {
*Reader
}
var _ ReaderProvider = TrivialReaderProvider{}
// GetReader implements ReaderProvider.
func (trp TrivialReaderProvider) GetReader() (*Reader, error) {
return trp.Reader, nil
}
// Close implements ReaderProvider.
func (trp TrivialReaderProvider) Close() {}
// valueBlockReader is used to retrieve values in value
// blocks. It is used when the sstable was written with
// Properties.ValueBlocksAreEnabled.
type valueBlockReader struct {
bpOpen blockProviderWhenOpen
rp ReaderProvider
vbih valueBlocksIndexHandle
stats *base.InternalIteratorStats
// The value blocks index is lazily retrieved the first time the reader
// needs to read a value that resides in a value block.
vbiBlock []byte
vbiCache bufferHandle
// When sequentially iterating through all key-value pairs, the cost of
// repeatedly getting a block that is already in the cache and releasing the
// bufferHandle can be ~40% of the cpu overhead. So the reader remembers the
// last value block it retrieved, in case there is locality of access, and
// this value block can be used for the next value retrieval.
valueBlockNum uint32
valueBlock []byte
valueBlockPtr unsafe.Pointer
valueCache bufferHandle
lazyFetcher base.LazyFetcher
closed bool
bufToMangle []byte
}
func (r *valueBlockReader) getLazyValueForPrefixAndValueHandle(handle []byte) base.LazyValue {
fetcher := &r.lazyFetcher
valLen, h := decodeLenFromValueHandle(handle[1:])
*fetcher = base.LazyFetcher{
Fetcher: r,
Attribute: base.AttributeAndLen{
ValueLen: int32(valLen),
ShortAttribute: getShortAttribute(valuePrefix(handle[0])),
},
}
if r.stats != nil {
r.stats.SeparatedPointValue.Count++
r.stats.SeparatedPointValue.ValueBytes += uint64(valLen)
}
return base.LazyValue{
ValueOrHandle: h,
Fetcher: fetcher,
}
}
func (r *valueBlockReader) close() {
r.bpOpen = nil
r.vbiBlock = nil
r.vbiCache.Release()
// Set the handle to empty since Release does not nil the Handle.value. If
// we were to reopen this valueBlockReader and retrieve the same
// Handle.value from the cache, we don't want to accidentally unref it when
// attempting to unref the old handle.
r.vbiCache = bufferHandle{}
r.valueBlock = nil
r.valueBlockPtr = nil
r.valueCache.Release()
// See comment above.
r.valueCache = bufferHandle{}
r.closed = true
// rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be
// implemented.
}
// Fetch implements base.ValueFetcher.
func (r *valueBlockReader) Fetch(
handle []byte, valLen int32, buf []byte,
) (val []byte, callerOwned bool, err error) {
if !r.closed {
val, err := r.getValueInternal(handle, valLen)
if invariants.Enabled {
val = r.doValueMangling(val)
}
return val, false, err
}
bp := blockProviderWhenClosed{rp: r.rp}
err = bp.open()
if err != nil {
return nil, false, err
}
defer bp.close()
defer r.close()
r.bpOpen = bp
var v []byte
v, err = r.getValueInternal(handle, valLen)
if err != nil {
return nil, false, err
}
buf = append(buf[:0], v...)
return buf, true, nil
}
// doValueMangling attempts to uncover violations of the contract listed in
// the declaration comment of LazyValue. It is expensive, hence only called
// when invariants.Enabled.
func (r *valueBlockReader) doValueMangling(v []byte) []byte {
// Randomly set the bytes in the previous retrieved value to 0, since
// property P1 only requires the valueBlockReader to maintain the memory of
// one fetched value.
if rand.Intn(2) == 0 {
for i := range r.bufToMangle {
r.bufToMangle[i] = 0
}
}
// Store the current value in a new buffer for future mangling.
r.bufToMangle = append([]byte(nil), v...)
return r.bufToMangle
}
func (r *valueBlockReader) getValueInternal(handle []byte, valLen int32) (val []byte, err error) {
vh := decodeRemainingValueHandle(handle)
vh.valueLen = uint32(valLen)
if r.vbiBlock == nil {
ch, err := r.bpOpen.readBlockForVBR(r.vbih.h, r.stats)
if err != nil {
return nil, err
}
r.vbiCache = ch
r.vbiBlock = ch.Get()
}
if r.valueBlock == nil || r.valueBlockNum != vh.blockNum {
vbh, err := r.getBlockHandle(vh.blockNum)
if err != nil {
return nil, err
}
vbCacheHandle, err := r.bpOpen.readBlockForVBR(vbh, r.stats)
if err != nil {
return nil, err
}
r.valueBlockNum = vh.blockNum
r.valueCache.Release()
r.valueCache = vbCacheHandle
r.valueBlock = vbCacheHandle.Get()
r.valueBlockPtr = unsafe.Pointer(&r.valueBlock[0])
}
if r.stats != nil {
r.stats.SeparatedPointValue.ValueBytesFetched += uint64(valLen)
}
return r.valueBlock[vh.offsetInBlock : vh.offsetInBlock+vh.valueLen], nil
}
func (r *valueBlockReader) getBlockHandle(blockNum uint32) (BlockHandle, error) {
indexEntryLen :=
int(r.vbih.blockNumByteLength + r.vbih.blockOffsetByteLength + r.vbih.blockLengthByteLength)
offsetInIndex := indexEntryLen * int(blockNum)
if len(r.vbiBlock) < offsetInIndex+indexEntryLen {
return BlockHandle{}, errors.Errorf(
"cannot read at offset %d and length %d from block of length %d",
offsetInIndex, indexEntryLen, len(r.vbiBlock))
}
b := r.vbiBlock[offsetInIndex : offsetInIndex+indexEntryLen]
n := int(r.vbih.blockNumByteLength)
bn := littleEndianGet(b, n)
if uint32(bn) != blockNum {
return BlockHandle{},
errors.Errorf("expected block num %d but found %d", blockNum, bn)
}
b = b[n:]
n = int(r.vbih.blockOffsetByteLength)
blockOffset := littleEndianGet(b, n)
b = b[n:]
n = int(r.vbih.blockLengthByteLength)
blockLen := littleEndianGet(b, n)
return BlockHandle{Offset: blockOffset, Length: blockLen}, nil
}