mirror of
				https://source.quilibrium.com/quilibrium/ceremonyclient.git
				synced 2025-11-04 13:27:27 +00:00 
			
		
		
		
	
		
			
				
	
	
		
			901 lines
		
	
	
		
			28 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			901 lines
		
	
	
		
			28 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
 | 
						|
// of this source code is governed by a BSD-style license that can be found in
 | 
						|
// the LICENSE file.
 | 
						|
 | 
						|
package sharedcache
 | 
						|
 | 
						|
import (
 | 
						|
	"context"
 | 
						|
	"fmt"
 | 
						|
	"io"
 | 
						|
	"math/bits"
 | 
						|
	"sync"
 | 
						|
	"sync/atomic"
 | 
						|
	"time"
 | 
						|
 | 
						|
	"github.com/cockroachdb/errors"
 | 
						|
	"github.com/cockroachdb/pebble/internal/base"
 | 
						|
	"github.com/cockroachdb/pebble/internal/invariants"
 | 
						|
	"github.com/cockroachdb/pebble/objstorage/remote"
 | 
						|
	"github.com/cockroachdb/pebble/vfs"
 | 
						|
	"github.com/prometheus/client_golang/prometheus"
 | 
						|
)
 | 
						|
 | 
						|
// Exported to enable exporting from package pebble to enable
 | 
						|
// exporting metrics with below buckets in CRDB.
 | 
						|
var (
 | 
						|
	IOBuckets           = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
 | 
						|
	ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
 | 
						|
)
 | 
						|
 | 
						|
// Cache is a persistent cache backed by a local filesystem. It is intended
 | 
						|
// to cache data that is in slower shared storage (e.g. S3), hence the
 | 
						|
// package name 'sharedcache'.
 | 
						|
type Cache struct {
 | 
						|
	shards       []shard
 | 
						|
	writeWorkers writeWorkers
 | 
						|
 | 
						|
	bm                blockMath
 | 
						|
	shardingBlockSize int64
 | 
						|
 | 
						|
	logger  base.Logger
 | 
						|
	metrics internalMetrics
 | 
						|
}
 | 
						|
 | 
						|
// Metrics is a struct containing metrics exported by the secondary cache.
 | 
						|
// TODO(josh): Reconsider the set of metrics exported by the secondary cache
 | 
						|
// before we release the secondary cache to users. We choose to export many metrics
 | 
						|
// right now, so we learn a lot from the benchmarking we are doing over the 23.2
 | 
						|
// cycle.
 | 
						|
type Metrics struct {
 | 
						|
	// The number of sstable bytes stored in the cache.
 | 
						|
	Size int64
 | 
						|
	// The count of cache blocks in the cache (not sstable blocks).
 | 
						|
	Count int64
 | 
						|
 | 
						|
	// The number of calls to ReadAt.
 | 
						|
	TotalReads int64
 | 
						|
	// The number of calls to ReadAt that require reading data from 2+ shards.
 | 
						|
	MultiShardReads int64
 | 
						|
	// The number of calls to ReadAt that require reading data from 2+ cache blocks.
 | 
						|
	MultiBlockReads int64
 | 
						|
	// The number of calls to ReadAt where all data returned was read from the cache.
 | 
						|
	ReadsWithFullHit int64
 | 
						|
	// The number of calls to ReadAt where some data returned was read from the cache.
 | 
						|
	ReadsWithPartialHit int64
 | 
						|
	// The number of calls to ReadAt where no data returned was read from the cache.
 | 
						|
	ReadsWithNoHit int64
 | 
						|
 | 
						|
	// The number of times a cache block was evicted from the cache.
 | 
						|
	Evictions int64
 | 
						|
	// The number of times writing a cache block to the cache failed.
 | 
						|
	WriteBackFailures int64
 | 
						|
 | 
						|
	// The latency of calls to get some data from the cache.
 | 
						|
	GetLatency prometheus.Histogram
 | 
						|
	// The latency of reads of a single cache block from disk.
 | 
						|
	DiskReadLatency prometheus.Histogram
 | 
						|
	// The latency of writing data to write back to the cache to a channel.
 | 
						|
	// Generally should be low, but if the channel is full, could be high.
 | 
						|
	QueuePutLatency prometheus.Histogram
 | 
						|
	// The latency of calls to put some data read from block storage into the cache.
 | 
						|
	PutLatency prometheus.Histogram
 | 
						|
	// The latency of writes of a single cache block to disk.
 | 
						|
	DiskWriteLatency prometheus.Histogram
 | 
						|
}
 | 
						|
 | 
						|
// See docs at Metrics.
 | 
						|
type internalMetrics struct {
 | 
						|
	count atomic.Int64
 | 
						|
 | 
						|
	totalReads          atomic.Int64
 | 
						|
	multiShardReads     atomic.Int64
 | 
						|
	multiBlockReads     atomic.Int64
 | 
						|
	readsWithFullHit    atomic.Int64
 | 
						|
	readsWithPartialHit atomic.Int64
 | 
						|
	readsWithNoHit      atomic.Int64
 | 
						|
 | 
						|
	evictions         atomic.Int64
 | 
						|
	writeBackFailures atomic.Int64
 | 
						|
 | 
						|
	getLatency       prometheus.Histogram
 | 
						|
	diskReadLatency  prometheus.Histogram
 | 
						|
	queuePutLatency  prometheus.Histogram
 | 
						|
	putLatency       prometheus.Histogram
 | 
						|
	diskWriteLatency prometheus.Histogram
 | 
						|
}
 | 
						|
 | 
						|
const (
 | 
						|
	// writeWorkersPerShard is used to establish the number of worker goroutines
 | 
						|
	// that perform writes to the cache.
 | 
						|
	writeWorkersPerShard = 4
 | 
						|
	// writeTaskPerWorker is used to establish how many tasks can be queued up
 | 
						|
	// until we have to block.
 | 
						|
	writeTasksPerWorker = 4
 | 
						|
)
 | 
						|
 | 
						|
// Open opens a cache. If there is no existing cache at fsDir, a new one
 | 
						|
// is created.
 | 
						|
func Open(
 | 
						|
	fs vfs.FS,
 | 
						|
	logger base.Logger,
 | 
						|
	fsDir string,
 | 
						|
	blockSize int,
 | 
						|
	// shardingBlockSize is the size of a shard block. The cache is split into contiguous
 | 
						|
	// shardingBlockSize units. The units are distributed across multiple independent shards
 | 
						|
	// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
 | 
						|
	// policies operate at the level of shard, not whole cache. This is done to reduce lock
 | 
						|
	// contention.
 | 
						|
	shardingBlockSize int64,
 | 
						|
	sizeBytes int64,
 | 
						|
	numShards int,
 | 
						|
) (*Cache, error) {
 | 
						|
	if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
 | 
						|
		// Up the size so that we have one block per shard. In practice, this should
 | 
						|
		// only happen in tests.
 | 
						|
		sizeBytes = minSize
 | 
						|
	}
 | 
						|
 | 
						|
	c := &Cache{
 | 
						|
		logger:            logger,
 | 
						|
		bm:                makeBlockMath(blockSize),
 | 
						|
		shardingBlockSize: shardingBlockSize,
 | 
						|
	}
 | 
						|
	c.shards = make([]shard, numShards)
 | 
						|
	blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
 | 
						|
	for i := range c.shards {
 | 
						|
		if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil {
 | 
						|
			return nil, err
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
 | 
						|
 | 
						|
	c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
 | 
						|
	c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
 | 
						|
	c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
 | 
						|
	c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
 | 
						|
 | 
						|
	// Measures a channel write, so lower min.
 | 
						|
	c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
 | 
						|
 | 
						|
	return c, nil
 | 
						|
}
 | 
						|
 | 
						|
// Close closes the cache. Methods such as ReadAt should not be called after Close is
 | 
						|
// called.
 | 
						|
func (c *Cache) Close() error {
 | 
						|
	c.writeWorkers.Stop()
 | 
						|
 | 
						|
	var retErr error
 | 
						|
	for i := range c.shards {
 | 
						|
		if err := c.shards[i].close(); err != nil && retErr == nil {
 | 
						|
			retErr = err
 | 
						|
		}
 | 
						|
	}
 | 
						|
	c.shards = nil
 | 
						|
	return retErr
 | 
						|
}
 | 
						|
 | 
						|
// Metrics return metrics for the cache. Callers should not mutate
 | 
						|
// the returned histograms, which are pointer types.
 | 
						|
func (c *Cache) Metrics() Metrics {
 | 
						|
	return Metrics{
 | 
						|
		Count:               c.metrics.count.Load(),
 | 
						|
		Size:                c.metrics.count.Load() * int64(c.bm.BlockSize()),
 | 
						|
		TotalReads:          c.metrics.totalReads.Load(),
 | 
						|
		MultiShardReads:     c.metrics.multiShardReads.Load(),
 | 
						|
		MultiBlockReads:     c.metrics.multiBlockReads.Load(),
 | 
						|
		ReadsWithFullHit:    c.metrics.readsWithFullHit.Load(),
 | 
						|
		ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(),
 | 
						|
		ReadsWithNoHit:      c.metrics.readsWithNoHit.Load(),
 | 
						|
		Evictions:           c.metrics.evictions.Load(),
 | 
						|
		WriteBackFailures:   c.metrics.writeBackFailures.Load(),
 | 
						|
		GetLatency:          c.metrics.getLatency,
 | 
						|
		DiskReadLatency:     c.metrics.diskReadLatency,
 | 
						|
		QueuePutLatency:     c.metrics.queuePutLatency,
 | 
						|
		PutLatency:          c.metrics.putLatency,
 | 
						|
		DiskWriteLatency:    c.metrics.diskWriteLatency,
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// ReadFlags contains options for Cache.ReadAt.
 | 
						|
type ReadFlags struct {
 | 
						|
	// ReadOnly instructs ReadAt to not write any new data into the cache; it is
 | 
						|
	// used when the data is unlikely to be used again.
 | 
						|
	ReadOnly bool
 | 
						|
}
 | 
						|
 | 
						|
// ReadAt performs a read form an object, attempting to use cached data when
 | 
						|
// possible.
 | 
						|
func (c *Cache) ReadAt(
 | 
						|
	ctx context.Context,
 | 
						|
	fileNum base.DiskFileNum,
 | 
						|
	p []byte,
 | 
						|
	ofs int64,
 | 
						|
	objReader remote.ObjectReader,
 | 
						|
	objSize int64,
 | 
						|
	flags ReadFlags,
 | 
						|
) error {
 | 
						|
	c.metrics.totalReads.Add(1)
 | 
						|
	if ofs >= objSize {
 | 
						|
		if invariants.Enabled {
 | 
						|
			panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize))
 | 
						|
		}
 | 
						|
		return io.EOF
 | 
						|
	}
 | 
						|
	// TODO(radu): for compaction reads, we may not want to read from the cache at
 | 
						|
	// all.
 | 
						|
	{
 | 
						|
		start := time.Now()
 | 
						|
		n, err := c.get(fileNum, p, ofs)
 | 
						|
		c.metrics.getLatency.Observe(float64(time.Since(start)))
 | 
						|
		if err != nil {
 | 
						|
			return err
 | 
						|
		}
 | 
						|
		if n == len(p) {
 | 
						|
			// Everything was in cache!
 | 
						|
			c.metrics.readsWithFullHit.Add(1)
 | 
						|
			return nil
 | 
						|
		}
 | 
						|
		if n == 0 {
 | 
						|
			c.metrics.readsWithNoHit.Add(1)
 | 
						|
		} else {
 | 
						|
			c.metrics.readsWithPartialHit.Add(1)
 | 
						|
		}
 | 
						|
 | 
						|
		// Note this. The below code does not need the original ofs, as with the earlier
 | 
						|
		// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
 | 
						|
		ofs += int64(n)
 | 
						|
		p = p[n:]
 | 
						|
 | 
						|
		if invariants.Enabled {
 | 
						|
			if n != 0 && c.bm.Remainder(ofs) != 0 {
 | 
						|
				panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	if flags.ReadOnly {
 | 
						|
		return objReader.ReadAt(ctx, p, ofs)
 | 
						|
	}
 | 
						|
 | 
						|
	// We must do reads with offset & size that are multiples of the block size. Else
 | 
						|
	// later cache hits may return incorrect zeroed results from the cache.
 | 
						|
	firstBlockInd := c.bm.Block(ofs)
 | 
						|
	adjustedOfs := c.bm.BlockOffset(firstBlockInd)
 | 
						|
 | 
						|
	// Take the length of what is left to read plus the length of the adjustment of
 | 
						|
	// the offset plus the size of a block minus one and divide by the size of a block
 | 
						|
	// to get the number of blocks to read from the object.
 | 
						|
	sizeOfOffAdjustment := int(ofs - adjustedOfs)
 | 
						|
	adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment)))
 | 
						|
	adjustedP := make([]byte, adjustedLen)
 | 
						|
 | 
						|
	// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
 | 
						|
	eofCap := int64(adjustedLen)
 | 
						|
	if adjustedOfs+eofCap > objSize {
 | 
						|
		eofCap = objSize - adjustedOfs
 | 
						|
	}
 | 
						|
	if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil {
 | 
						|
		return err
 | 
						|
	}
 | 
						|
	copy(p, adjustedP[sizeOfOffAdjustment:])
 | 
						|
 | 
						|
	start := time.Now()
 | 
						|
	c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs)
 | 
						|
	c.metrics.queuePutLatency.Observe(float64(time.Since(start)))
 | 
						|
 | 
						|
	return nil
 | 
						|
}
 | 
						|
 | 
						|
// get attempts to read the requested data from the cache, if it is already
 | 
						|
// there.
 | 
						|
//
 | 
						|
// If all data is available, returns n = len(p).
 | 
						|
//
 | 
						|
// If data is partially available, a prefix of the data is read; returns n < len(p)
 | 
						|
// and no error. If no prefix is available, returns n = 0 and no error.
 | 
						|
func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
 | 
						|
	// The data extent might cross shard boundaries, hence the loop. In the hot
 | 
						|
	// path, max two iterations of this loop will be executed, since reads are sized
 | 
						|
	// in units of sstable block size.
 | 
						|
	var multiShard bool
 | 
						|
	for {
 | 
						|
		shard := c.getShard(fileNum, ofs+int64(n))
 | 
						|
		cappedLen := len(p[n:])
 | 
						|
		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
 | 
						|
			cappedLen = toBoundary
 | 
						|
		}
 | 
						|
		numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
 | 
						|
		if err != nil {
 | 
						|
			return n, err
 | 
						|
		}
 | 
						|
		n += numRead
 | 
						|
		if numRead < cappedLen {
 | 
						|
			// We only read a prefix from this shard.
 | 
						|
			return n, nil
 | 
						|
		}
 | 
						|
		if n == len(p) {
 | 
						|
			// We are done.
 | 
						|
			return n, nil
 | 
						|
		}
 | 
						|
		// Data extent crosses shard boundary, continue with next shard.
 | 
						|
		if !multiShard {
 | 
						|
			c.metrics.multiShardReads.Add(1)
 | 
						|
			multiShard = true
 | 
						|
		}
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// set attempts to write the requested data to the cache. Both ofs & len(p) must
 | 
						|
// be multiples of the block size.
 | 
						|
//
 | 
						|
// If all of p is not written to the shard, set returns a non-nil error.
 | 
						|
func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
 | 
						|
	if invariants.Enabled {
 | 
						|
		if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 {
 | 
						|
			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	// The data extent might cross shard boundaries, hence the loop. In the hot
 | 
						|
	// path, max two iterations of this loop will be executed, since reads are sized
 | 
						|
	// in units of sstable block size.
 | 
						|
	n := 0
 | 
						|
	for {
 | 
						|
		shard := c.getShard(fileNum, ofs+int64(n))
 | 
						|
		cappedLen := len(p[n:])
 | 
						|
		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
 | 
						|
			cappedLen = toBoundary
 | 
						|
		}
 | 
						|
		err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
 | 
						|
		if err != nil {
 | 
						|
			return err
 | 
						|
		}
 | 
						|
		// set returns an error if cappedLen bytes aren't written to the shard.
 | 
						|
		n += cappedLen
 | 
						|
		if n == len(p) {
 | 
						|
			// We are done.
 | 
						|
			return nil
 | 
						|
		}
 | 
						|
		// Data extent crosses shard boundary, continue with next shard.
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
 | 
						|
	const prime64 = 1099511628211
 | 
						|
	hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize)
 | 
						|
	// TODO(josh): Instance change ops are often run in production. Such an operation
 | 
						|
	// updates len(c.shards); see openSharedCache. As a result, the behavior of this
 | 
						|
	// function changes, and the cache empties out at restart time. We may want a better
 | 
						|
	// story here eventually.
 | 
						|
	return &c.shards[hash%uint64(len(c.shards))]
 | 
						|
}
 | 
						|
 | 
						|
type shard struct {
 | 
						|
	cache             *Cache
 | 
						|
	file              vfs.File
 | 
						|
	sizeInBlocks      int64
 | 
						|
	bm                blockMath
 | 
						|
	shardingBlockSize int64
 | 
						|
	mu                struct {
 | 
						|
		sync.Mutex
 | 
						|
		// TODO(josh): None of these datastructures are space-efficient.
 | 
						|
		// Focusing on correctness to start.
 | 
						|
		where  whereMap
 | 
						|
		blocks []cacheBlockState
 | 
						|
		// Head of LRU list (doubly-linked circular).
 | 
						|
		lruHead cacheBlockIndex
 | 
						|
		// Head of free list (singly-linked chain).
 | 
						|
		freeHead cacheBlockIndex
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
type cacheBlockState struct {
 | 
						|
	lock    lockState
 | 
						|
	logical logicalBlockID
 | 
						|
 | 
						|
	// next is the next block in the LRU or free list (or invalidBlockIndex if it
 | 
						|
	// is the last block in the free list).
 | 
						|
	next cacheBlockIndex
 | 
						|
 | 
						|
	// prev is the previous block in the LRU list. It is not used when the block
 | 
						|
	// is in the free list.
 | 
						|
	prev cacheBlockIndex
 | 
						|
}
 | 
						|
 | 
						|
// Maps a logical block in an SST to an index of the cache block with the
 | 
						|
// file contents (to the "cache block index").
 | 
						|
type whereMap map[logicalBlockID]cacheBlockIndex
 | 
						|
 | 
						|
type logicalBlockID struct {
 | 
						|
	filenum       base.DiskFileNum
 | 
						|
	cacheBlockIdx cacheBlockIndex
 | 
						|
}
 | 
						|
 | 
						|
type lockState int64
 | 
						|
 | 
						|
const (
 | 
						|
	unlocked lockState = 0
 | 
						|
	// >0 lockState tracks the number of distinct readers of some cache block / logical block
 | 
						|
	// which is in the secondary cache. It is used to ensure that a cache block is not evicted
 | 
						|
	// and overwritten, while there are active readers.
 | 
						|
	readLockTakenInc = 1
 | 
						|
	// -1 lockState indicates that some cache block is currently being populated with data from
 | 
						|
	// blob storage. It is used to ensure that a cache block is not read or evicted again, while
 | 
						|
	// it is being populated.
 | 
						|
	writeLockTaken = -1
 | 
						|
)
 | 
						|
 | 
						|
func (s *shard) init(
 | 
						|
	cache *Cache,
 | 
						|
	fs vfs.FS,
 | 
						|
	fsDir string,
 | 
						|
	shardIdx int,
 | 
						|
	sizeInBlocks int64,
 | 
						|
	blockSize int,
 | 
						|
	shardingBlockSize int64,
 | 
						|
) error {
 | 
						|
	*s = shard{
 | 
						|
		cache:        cache,
 | 
						|
		sizeInBlocks: sizeInBlocks,
 | 
						|
	}
 | 
						|
	if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 {
 | 
						|
		return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
 | 
						|
	}
 | 
						|
	s.bm = makeBlockMath(blockSize)
 | 
						|
	s.shardingBlockSize = shardingBlockSize
 | 
						|
	file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
 | 
						|
	if err != nil {
 | 
						|
		return err
 | 
						|
	}
 | 
						|
	// TODO(radu): truncate file if necessary (especially important if we restart
 | 
						|
	// with more shards).
 | 
						|
	if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil {
 | 
						|
		return err
 | 
						|
	}
 | 
						|
	s.file = file
 | 
						|
 | 
						|
	// TODO(josh): Right now, the secondary cache is not persistent. All existing
 | 
						|
	// cache contents will be over-written, since all metadata is only stored in
 | 
						|
	// memory.
 | 
						|
	s.mu.where = make(whereMap)
 | 
						|
	s.mu.blocks = make([]cacheBlockState, sizeInBlocks)
 | 
						|
	s.mu.lruHead = invalidBlockIndex
 | 
						|
	s.mu.freeHead = invalidBlockIndex
 | 
						|
	for i := range s.mu.blocks {
 | 
						|
		s.freePush(cacheBlockIndex(i))
 | 
						|
	}
 | 
						|
 | 
						|
	return nil
 | 
						|
}
 | 
						|
 | 
						|
func (s *shard) close() error {
 | 
						|
	defer func() {
 | 
						|
		s.file = nil
 | 
						|
	}()
 | 
						|
	return s.file.Close()
 | 
						|
}
 | 
						|
 | 
						|
// freePush pushes a block to the front of the free list.
 | 
						|
func (s *shard) freePush(index cacheBlockIndex) {
 | 
						|
	s.mu.blocks[index].next = s.mu.freeHead
 | 
						|
	s.mu.freeHead = index
 | 
						|
}
 | 
						|
 | 
						|
// freePop removes the block from the front of the free list. Must not be called
 | 
						|
// if the list is empty (i.e. freeHead = invalidBlockIndex).
 | 
						|
func (s *shard) freePop() cacheBlockIndex {
 | 
						|
	index := s.mu.freeHead
 | 
						|
	s.mu.freeHead = s.mu.blocks[index].next
 | 
						|
	return index
 | 
						|
}
 | 
						|
 | 
						|
// lruInsertFront inserts a block at the front of the LRU list.
 | 
						|
func (s *shard) lruInsertFront(index cacheBlockIndex) {
 | 
						|
	b := &s.mu.blocks[index]
 | 
						|
	if s.mu.lruHead == invalidBlockIndex {
 | 
						|
		b.next = index
 | 
						|
		b.prev = index
 | 
						|
	} else {
 | 
						|
		b.next = s.mu.lruHead
 | 
						|
		h := &s.mu.blocks[s.mu.lruHead]
 | 
						|
		b.prev = h.prev
 | 
						|
		s.mu.blocks[h.prev].next = index
 | 
						|
		h.prev = index
 | 
						|
	}
 | 
						|
	s.mu.lruHead = index
 | 
						|
}
 | 
						|
 | 
						|
func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex {
 | 
						|
	return s.mu.blocks[index].next
 | 
						|
}
 | 
						|
 | 
						|
func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex {
 | 
						|
	return s.mu.blocks[index].prev
 | 
						|
}
 | 
						|
 | 
						|
// lruUnlink removes a block from the LRU list.
 | 
						|
func (s *shard) lruUnlink(index cacheBlockIndex) {
 | 
						|
	b := &s.mu.blocks[index]
 | 
						|
	if b.next == index {
 | 
						|
		s.mu.lruHead = invalidBlockIndex
 | 
						|
	} else {
 | 
						|
		s.mu.blocks[b.prev].next = b.next
 | 
						|
		s.mu.blocks[b.next].prev = b.prev
 | 
						|
		if s.mu.lruHead == index {
 | 
						|
			s.mu.lruHead = b.next
 | 
						|
		}
 | 
						|
	}
 | 
						|
	b.next, b.prev = invalidBlockIndex, invalidBlockIndex
 | 
						|
}
 | 
						|
 | 
						|
// get attempts to read the requested data from the shard. The data must not
 | 
						|
// cross a shard boundary.
 | 
						|
//
 | 
						|
// If all data is available, returns n = len(p).
 | 
						|
//
 | 
						|
// If data is partially available, a prefix of the data is read; returns n < len(p)
 | 
						|
// and no error. If no prefix is available, returns n = 0 and no error.
 | 
						|
//
 | 
						|
// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the
 | 
						|
// first block is not in the cache and the second one is, we will read both from
 | 
						|
// blob storage. We should fix this. This is not an unlikely scenario if we are doing
 | 
						|
// a reverse scan, since those iterate over sstable blocks in reverse order and due to
 | 
						|
// cache block aligned reads will have read the suffix of the sstable block that will
 | 
						|
// be needed next.
 | 
						|
func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
 | 
						|
	if invariants.Enabled {
 | 
						|
		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
 | 
						|
			panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
 | 
						|
		}
 | 
						|
		s.assertShardStateIsConsistent()
 | 
						|
	}
 | 
						|
 | 
						|
	// The data extent might cross cache block boundaries, hence the loop. In the hot
 | 
						|
	// path, max two iterations of this loop will be executed, since reads are sized
 | 
						|
	// in units of sstable block size.
 | 
						|
	var multiBlock bool
 | 
						|
	for {
 | 
						|
		k := logicalBlockID{
 | 
						|
			filenum:       fileNum,
 | 
						|
			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
 | 
						|
		}
 | 
						|
		s.mu.Lock()
 | 
						|
		cacheBlockIdx, ok := s.mu.where[k]
 | 
						|
		// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
 | 
						|
		// than blob storage read latency) that miss on the same logical block ID will not necessarily
 | 
						|
		// be rare. We may want to do only one read, with the later readers blocking on the first read
 | 
						|
		// completing. This could be implemented either here or in the primary block cache. See
 | 
						|
		// https://github.com/cockroachdb/pebble/pull/2586 for additional discussion.
 | 
						|
		if !ok {
 | 
						|
			s.mu.Unlock()
 | 
						|
			return n, nil
 | 
						|
		}
 | 
						|
		if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken {
 | 
						|
			// In practice, if we have two reads of the same SST block in close succession, we
 | 
						|
			// would expect the second to hit in the in-memory block cache. So it's not worth
 | 
						|
			// optimizing this case here.
 | 
						|
			s.mu.Unlock()
 | 
						|
			return n, nil
 | 
						|
		}
 | 
						|
		s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc
 | 
						|
		// Move to front of the LRU list.
 | 
						|
		s.lruUnlink(cacheBlockIdx)
 | 
						|
		s.lruInsertFront(cacheBlockIdx)
 | 
						|
		s.mu.Unlock()
 | 
						|
 | 
						|
		readAt := s.bm.BlockOffset(cacheBlockIdx)
 | 
						|
		readSize := s.bm.BlockSize()
 | 
						|
		if n == 0 { // if first read
 | 
						|
			rem := s.bm.Remainder(ofs)
 | 
						|
			readAt += rem
 | 
						|
			readSize -= int(rem)
 | 
						|
		}
 | 
						|
 | 
						|
		if len(p[n:]) <= readSize {
 | 
						|
			start := time.Now()
 | 
						|
			numRead, err := s.file.ReadAt(p[n:], readAt)
 | 
						|
			s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
 | 
						|
			s.dropReadLock(cacheBlockIdx)
 | 
						|
			return n + numRead, err
 | 
						|
		}
 | 
						|
		start := time.Now()
 | 
						|
		numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
 | 
						|
		s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
 | 
						|
		s.dropReadLock(cacheBlockIdx)
 | 
						|
		if err != nil {
 | 
						|
			return 0, err
 | 
						|
		}
 | 
						|
 | 
						|
		// Note that numRead == readSize, since we checked for an error above.
 | 
						|
		n += numRead
 | 
						|
 | 
						|
		if !multiBlock {
 | 
						|
			s.cache.metrics.multiBlockReads.Add(1)
 | 
						|
			multiBlock = true
 | 
						|
		}
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// set attempts to write the requested data to the shard. The data must not
 | 
						|
// cross a shard boundary, and both ofs & len(p) must be multiples of the
 | 
						|
// block size.
 | 
						|
//
 | 
						|
// If all of p is not written to the shard, set returns a non-nil error.
 | 
						|
func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
 | 
						|
	if invariants.Enabled {
 | 
						|
		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
 | 
						|
			panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
 | 
						|
		}
 | 
						|
		if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 {
 | 
						|
			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
 | 
						|
		}
 | 
						|
		s.assertShardStateIsConsistent()
 | 
						|
	}
 | 
						|
 | 
						|
	// The data extent might cross cache block boundaries, hence the loop. In the hot
 | 
						|
	// path, max two iterations of this loop will be executed, since reads are sized
 | 
						|
	// in units of sstable block size.
 | 
						|
	n := 0
 | 
						|
	for {
 | 
						|
		if n == len(p) {
 | 
						|
			return nil
 | 
						|
		}
 | 
						|
		if invariants.Enabled {
 | 
						|
			if n > len(p) {
 | 
						|
				panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p)))
 | 
						|
			}
 | 
						|
		}
 | 
						|
 | 
						|
		// If the logical block is already in the cache, we should skip doing a set.
 | 
						|
		k := logicalBlockID{
 | 
						|
			filenum:       fileNum,
 | 
						|
			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
 | 
						|
		}
 | 
						|
		s.mu.Lock()
 | 
						|
		if _, ok := s.mu.where[k]; ok {
 | 
						|
			s.mu.Unlock()
 | 
						|
			n += s.bm.BlockSize()
 | 
						|
			continue
 | 
						|
		}
 | 
						|
 | 
						|
		var cacheBlockIdx cacheBlockIndex
 | 
						|
		if s.mu.freeHead == invalidBlockIndex {
 | 
						|
			if invariants.Enabled && s.mu.lruHead == invalidBlockIndex {
 | 
						|
				panic("both LRU and free lists empty")
 | 
						|
			}
 | 
						|
 | 
						|
			// Find the last element in the LRU list which is not locked.
 | 
						|
			for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) {
 | 
						|
				if lock := s.mu.blocks[idx].lock; lock == unlocked {
 | 
						|
					cacheBlockIdx = idx
 | 
						|
					break
 | 
						|
				}
 | 
						|
				if idx == s.mu.lruHead {
 | 
						|
					// No unlocked block to evict.
 | 
						|
					//
 | 
						|
					// TODO(josh): We may want to block until a block frees up, instead of returning
 | 
						|
					// an error here. But I think we can do that later on, e.g. after running some production
 | 
						|
					// experiments.
 | 
						|
					s.mu.Unlock()
 | 
						|
					return errors.New("no block to evict so skipping write to cache")
 | 
						|
				}
 | 
						|
			}
 | 
						|
			s.cache.metrics.evictions.Add(1)
 | 
						|
			s.lruUnlink(cacheBlockIdx)
 | 
						|
			delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical)
 | 
						|
		} else {
 | 
						|
			s.cache.metrics.count.Add(1)
 | 
						|
			cacheBlockIdx = s.freePop()
 | 
						|
		}
 | 
						|
 | 
						|
		s.lruInsertFront(cacheBlockIdx)
 | 
						|
		s.mu.where[k] = cacheBlockIdx
 | 
						|
		s.mu.blocks[cacheBlockIdx].logical = k
 | 
						|
		s.mu.blocks[cacheBlockIdx].lock = writeLockTaken
 | 
						|
		s.mu.Unlock()
 | 
						|
 | 
						|
		writeAt := s.bm.BlockOffset(cacheBlockIdx)
 | 
						|
 | 
						|
		writeSize := s.bm.BlockSize()
 | 
						|
		if len(p[n:]) <= writeSize {
 | 
						|
			writeSize = len(p[n:])
 | 
						|
		}
 | 
						|
 | 
						|
		start := time.Now()
 | 
						|
		_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
 | 
						|
		s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start)))
 | 
						|
		if err != nil {
 | 
						|
			// Free the block.
 | 
						|
			s.mu.Lock()
 | 
						|
			defer s.mu.Unlock()
 | 
						|
 | 
						|
			delete(s.mu.where, k)
 | 
						|
			s.lruUnlink(cacheBlockIdx)
 | 
						|
			s.freePush(cacheBlockIdx)
 | 
						|
			return err
 | 
						|
		}
 | 
						|
		s.dropWriteLock(cacheBlockIdx)
 | 
						|
		n += writeSize
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// Doesn't inline currently. This might be okay, but something to keep in mind.
 | 
						|
func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
 | 
						|
	s.mu.Lock()
 | 
						|
	s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc
 | 
						|
	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 {
 | 
						|
		panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock))
 | 
						|
	}
 | 
						|
	s.mu.Unlock()
 | 
						|
}
 | 
						|
 | 
						|
// Doesn't inline currently. This might be okay, but something to keep in mind.
 | 
						|
func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
 | 
						|
	s.mu.Lock()
 | 
						|
	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken {
 | 
						|
		panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock))
 | 
						|
	}
 | 
						|
	s.mu.blocks[cacheBlockInd].lock = unlocked
 | 
						|
	s.mu.Unlock()
 | 
						|
}
 | 
						|
 | 
						|
func (s *shard) assertShardStateIsConsistent() {
 | 
						|
	s.mu.Lock()
 | 
						|
	defer s.mu.Unlock()
 | 
						|
 | 
						|
	lruLen := 0
 | 
						|
	if s.mu.lruHead != invalidBlockIndex {
 | 
						|
		for b := s.mu.lruHead; ; {
 | 
						|
			lruLen++
 | 
						|
			if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b {
 | 
						|
				panic("block in LRU list with no entry in where map")
 | 
						|
			}
 | 
						|
			b = s.lruNext(b)
 | 
						|
			if b == s.mu.lruHead {
 | 
						|
				break
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
	if lruLen != len(s.mu.where) {
 | 
						|
		panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where)))
 | 
						|
	}
 | 
						|
	freeLen := 0
 | 
						|
	for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next {
 | 
						|
		freeLen++
 | 
						|
	}
 | 
						|
 | 
						|
	if lruLen+freeLen != int(s.sizeInBlocks) {
 | 
						|
		panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks))
 | 
						|
	}
 | 
						|
	for i := range s.mu.blocks {
 | 
						|
		if state := s.mu.blocks[i].lock; state < writeLockTaken {
 | 
						|
			panic(fmt.Sprintf("lock state %v is not allowed", state))
 | 
						|
		}
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// cacheBlockIndex is the index of a blockSize-aligned cache block.
 | 
						|
type cacheBlockIndex int64
 | 
						|
 | 
						|
// invalidBlockIndex is used for the head of a list when the list is empty.
 | 
						|
const invalidBlockIndex cacheBlockIndex = -1
 | 
						|
 | 
						|
// blockMath is a helper type for performing conversions between offsets and
 | 
						|
// block indexes.
 | 
						|
type blockMath struct {
 | 
						|
	blockSizeBits int8
 | 
						|
}
 | 
						|
 | 
						|
func makeBlockMath(blockSize int) blockMath {
 | 
						|
	bm := blockMath{
 | 
						|
		blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1),
 | 
						|
	}
 | 
						|
	if blockSize != (1 << bm.blockSizeBits) {
 | 
						|
		panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize))
 | 
						|
	}
 | 
						|
	return bm
 | 
						|
}
 | 
						|
 | 
						|
func (bm blockMath) mask() int64 {
 | 
						|
	return (1 << bm.blockSizeBits) - 1
 | 
						|
}
 | 
						|
 | 
						|
// BlockSize returns the block size.
 | 
						|
func (bm blockMath) BlockSize() int {
 | 
						|
	return 1 << bm.blockSizeBits
 | 
						|
}
 | 
						|
 | 
						|
// Block returns the block index containing the given offset.
 | 
						|
func (bm blockMath) Block(offset int64) cacheBlockIndex {
 | 
						|
	return cacheBlockIndex(offset >> bm.blockSizeBits)
 | 
						|
}
 | 
						|
 | 
						|
// Remainder returns the offset relative to the start of the cache block.
 | 
						|
func (bm blockMath) Remainder(offset int64) int64 {
 | 
						|
	return offset & bm.mask()
 | 
						|
}
 | 
						|
 | 
						|
// BlockOffset returns the object offset where the given block starts.
 | 
						|
func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 {
 | 
						|
	return int64(block) << bm.blockSizeBits
 | 
						|
}
 | 
						|
 | 
						|
// RoundUp rounds up the given value to the closest multiple of block size.
 | 
						|
func (bm blockMath) RoundUp(x int64) int64 {
 | 
						|
	return (x + bm.mask()) & ^(bm.mask())
 | 
						|
}
 | 
						|
 | 
						|
type writeWorkers struct {
 | 
						|
	doneCh        chan struct{}
 | 
						|
	doneWaitGroup sync.WaitGroup
 | 
						|
 | 
						|
	numWorkers int
 | 
						|
	tasksCh    chan writeTask
 | 
						|
}
 | 
						|
 | 
						|
type writeTask struct {
 | 
						|
	fileNum base.DiskFileNum
 | 
						|
	p       []byte
 | 
						|
	offset  int64
 | 
						|
}
 | 
						|
 | 
						|
// Start starts the worker goroutines.
 | 
						|
func (w *writeWorkers) Start(c *Cache, numWorkers int) {
 | 
						|
	doneCh := make(chan struct{})
 | 
						|
	tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker)
 | 
						|
 | 
						|
	w.numWorkers = numWorkers
 | 
						|
	w.doneCh = doneCh
 | 
						|
	w.tasksCh = tasksCh
 | 
						|
	w.doneWaitGroup.Add(numWorkers)
 | 
						|
	for i := 0; i < numWorkers; i++ {
 | 
						|
		go func() {
 | 
						|
			defer w.doneWaitGroup.Done()
 | 
						|
			for {
 | 
						|
				select {
 | 
						|
				case <-doneCh:
 | 
						|
					return
 | 
						|
				case task, ok := <-tasksCh:
 | 
						|
					if !ok {
 | 
						|
						// The tasks channel was closed; this is used in testing code to
 | 
						|
						// ensure all writes are completed.
 | 
						|
						return
 | 
						|
					}
 | 
						|
					// TODO(radu): set() can perform multiple writes; perhaps each one
 | 
						|
					// should be its own task.
 | 
						|
					start := time.Now()
 | 
						|
					err := c.set(task.fileNum, task.p, task.offset)
 | 
						|
					c.metrics.putLatency.Observe(float64(time.Since(start)))
 | 
						|
					if err != nil {
 | 
						|
						c.metrics.writeBackFailures.Add(1)
 | 
						|
						// TODO(radu): throttle logs.
 | 
						|
						c.logger.Errorf("writing back to cache after miss failed: %v", err)
 | 
						|
					}
 | 
						|
				}
 | 
						|
			}
 | 
						|
		}()
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
// Stop waits for any in-progress writes to complete and stops the worker
 | 
						|
// goroutines and waits for any in-pro. Any queued writes not yet started are
 | 
						|
// discarded.
 | 
						|
func (w *writeWorkers) Stop() {
 | 
						|
	close(w.doneCh)
 | 
						|
	w.doneCh = nil
 | 
						|
	w.tasksCh = nil
 | 
						|
	w.doneWaitGroup.Wait()
 | 
						|
}
 | 
						|
 | 
						|
// QueueWrite adds a write task to the queue. Can block if the queue is full.
 | 
						|
func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) {
 | 
						|
	w.tasksCh <- writeTask{
 | 
						|
		fileNum: fileNum,
 | 
						|
		p:       p,
 | 
						|
		offset:  offset,
 | 
						|
	}
 | 
						|
}
 |