ceremonyclient/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go

255 lines
7.0 KiB
Go
Raw Normal View History

2024-01-03 07:31:42 +00:00
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remoteobjcat
import (
"bufio"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/remote"
)
// VersionEdit is a modification to the remote object state which can be encoded
// into a record.
//
// TODO(radu): consider adding creation and deletion time for debugging purposes.
type VersionEdit struct {
NewObjects []RemoteObjectMetadata
DeletedObjects []base.DiskFileNum
CreatorID objstorage.CreatorID
}
const (
// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
// cleanup method, optional new object tags, and ending with a 0 byte.
tagNewObject = 1
// tagDeletedObject is followed by the FileNum.
tagDeletedObject = 2
// tagCreatorID is followed by the Creator ID for this store. This ID can
// never change.
tagCreatorID = 3
// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
// is followed by the encoded length of the locator string and the string.
tagNewObjectLocator = 4
// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
// It is followed by the encoded length of the custom object name string
// followed by the string.
tagNewObjectCustomName = 5
)
// Object type values. We don't want to encode FileType directly because it is
// more general (and we want freedom to change it in the future).
const (
objTypeTable = 1
)
func objTypeToFileType(objType uint64) (base.FileType, error) {
switch objType {
case objTypeTable:
return base.FileTypeTable, nil
default:
return 0, errors.Newf("unknown object type %d", objType)
}
}
func fileTypeToObjType(fileType base.FileType) (uint64, error) {
switch fileType {
case base.FileTypeTable:
return objTypeTable, nil
default:
return 0, errors.Newf("unknown object type for file type %d", fileType)
}
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
for _, meta := range v.NewObjects {
objType, err := fileTypeToObjType(meta.FileType)
if err != nil {
return err
}
buf = binary.AppendUvarint(buf, uint64(tagNewObject))
buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
buf = binary.AppendUvarint(buf, objType)
buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
if meta.Locator != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
buf = encodeString(buf, string(meta.Locator))
}
if meta.CustomObjectName != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
buf = encodeString(buf, meta.CustomObjectName)
}
// Append 0 as the terminator for optional new object tags.
buf = binary.AppendUvarint(buf, 0)
}
for _, dfn := range v.DeletedObjects {
buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
}
if v.CreatorID.IsSet() {
buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
}
_, err := w.Write(buf)
return err
}
// Decode decodes an edit from the specified reader.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(io.ByteReader)
if !ok {
br = bufio.NewReader(r)
}
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return err
}
err = nil
switch tag {
case tagNewObject:
var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
var locator, customName string
var fileType base.FileType
fileNum, err = binary.ReadUvarint(br)
if err == nil {
var objType uint64
objType, err = binary.ReadUvarint(br)
if err == nil {
fileType, err = objTypeToFileType(objType)
}
}
if err == nil {
creatorID, err = binary.ReadUvarint(br)
}
if err == nil {
creatorFileNum, err = binary.ReadUvarint(br)
}
if err == nil {
cleanupMethod, err = binary.ReadUvarint(br)
}
for err == nil {
var optionalTag uint64
optionalTag, err = binary.ReadUvarint(br)
if err != nil || optionalTag == 0 {
break
}
switch optionalTag {
case tagNewObjectLocator:
locator, err = decodeString(br)
case tagNewObjectCustomName:
customName, err = decodeString(br)
default:
err = errors.Newf("unknown newObject tag %d", optionalTag)
}
}
if err == nil {
v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
FileNum: base.FileNum(fileNum).DiskFileNum(),
FileType: fileType,
CreatorID: objstorage.CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod),
Locator: remote.Locator(locator),
CustomObjectName: customName,
})
}
case tagDeletedObject:
var fileNum uint64
fileNum, err = binary.ReadUvarint(br)
if err == nil {
v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
}
case tagCreatorID:
var id uint64
id, err = binary.ReadUvarint(br)
if err == nil {
v.CreatorID = objstorage.CreatorID(id)
}
default:
err = errors.Newf("unknown tag %d", tag)
}
if err != nil {
if err == io.EOF {
return errCorruptCatalog
}
return err
}
}
return nil
}
func encodeString(buf []byte, s string) []byte {
buf = binary.AppendUvarint(buf, uint64(len(s)))
buf = append(buf, []byte(s)...)
return buf
}
func decodeString(br io.ByteReader) (string, error) {
length, err := binary.ReadUvarint(br)
if err != nil || length == 0 {
return "", err
}
buf := make([]byte, length)
for i := range buf {
buf[i], err = br.ReadByte()
if err != nil {
return "", err
}
}
return string(buf), nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
// Apply the version edit to a creator ID and a map of objects.
func (v *VersionEdit) Apply(
creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
) error {
if v.CreatorID.IsSet() {
*creatorID = v.CreatorID
}
for _, meta := range v.NewObjects {
if invariants.Enabled {
if _, exists := objects[meta.FileNum]; exists {
return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
}
}
objects[meta.FileNum] = meta
}
for _, fileNum := range v.DeletedObjects {
if invariants.Enabled {
if _, exists := objects[fileNum]; !exists {
return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
}
}
delete(objects, fileNum)
}
return nil
}