feat(cli): add shard CLI command (#1785)

* stub out shard CLI command

* prune blocks before and after desired range

* update rollback to use patched cometbft

* temp override for local patched versions

* handle pruning cometbft & block store state

* include docs & support -1 for "latest"

* update changelog

* add --only-app-state flag to match cosmos-sdk prune cmd

* give -1 magic number a name & reuse home from ctx

* refactor to only open state.db & blockstore.db once

* write rollback progress to one line

* prevent attempting rollback of future blocks

* make shard inclusive of endblock

* use tagged cosmo-sdk & cometbft versions
This commit is contained in:
Robert Pirtle 2024-01-05 11:25:53 -08:00 committed by GitHub
parent f00727fe85
commit 58621577ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 245 additions and 8 deletions

View File

@ -36,6 +36,9 @@ Ref: https://keepachangelog.com/en/1.0.0/
## [Unreleased] ## [Unreleased]
### Features
- (cli) [#1785] Add `shard` CLI command to support creating partitions of data for standalone nodes
## [v0.25.0] ## [v0.25.0]
@ -317,6 +320,7 @@ the [changelog](https://github.com/cosmos/cosmos-sdk/blob/v0.38.4/CHANGELOG.md).
- [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run - [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run
large-scale simulations remotely using aws-batch large-scale simulations remotely using aws-batch
[#1785]: https://github.com/Kava-Labs/kava/pull/1785
[#1784]: https://github.com/Kava-Labs/kava/pull/1784 [#1784]: https://github.com/Kava-Labs/kava/pull/1784
[#1776]: https://github.com/Kava-Labs/kava/pull/1776 [#1776]: https://github.com/Kava-Labs/kava/pull/1776
[#1770]: https://github.com/Kava-Labs/kava/pull/1770 [#1770]: https://github.com/Kava-Labs/kava/pull/1770

View File

@ -123,5 +123,6 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de
newQueryCmd(), newQueryCmd(),
newTxCmd(), newTxCmd(),
keyCommands(app.DefaultNodeHome), keyCommands(app.DefaultNodeHome),
newShardCmd(opts),
) )
} }

232
cmd/kava/cmd/shard.go Normal file
View File

@ -0,0 +1,232 @@
package cmd
import (
"fmt"
"strings"
"github.com/spf13/cobra"
dbm "github.com/cometbft/cometbft-db"
"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/flags"
pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
"github.com/cosmos/cosmos-sdk/server"
"github.com/cosmos/cosmos-sdk/store/rootmulti"
tmconfig "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/node"
tmstate "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/store"
ethermintserver "github.com/evmos/ethermint/server"
)
const (
flagShardStartBlock = "start"
flagShardEndBlock = "end"
flagShardOnlyAppState = "only-app-state"
// TODO: --preserve flag for creating & operating on a copy?
// allow using -1 to mean "latest" (perform no rollbacks)
shardEndBlockLatest = -1
)
func newShardCmd(opts ethermintserver.StartOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "shard --home <path-to-home-dir> --start <start-block> --end <end-block> [--only-app-state]",
Short: "Strip all blocks from the database outside of a given range",
Long: `shard opens a local kava home directory's databases and removes all blocks outside a range defined by --start and --end. The range is inclusive of the end block.
It works by first rolling back the latest state to the block before the end block, and then by pruning all state before the start block.
Setting the end block to -1 signals to keep the latest block (no rollbacks).
The --only-app-state flag can be used to skip the pruning of the blockstore and cometbft state. This matches the functionality of the cosmos-sdk's "prune" command. Note that rolled back blocks will still affect all stores.
WARNING: this is a destructive action.`,
Example: `Create a 1M block data shard (keeps blocks kava 1,000,000 to 2,000,000)
$ kava shard --home path/to/.kava --start 1000000 --end 2000000
Prune all blocks up to 5,000,000:
$ kava shard --home path/to/.kava --start 5000000 --end -1
Prune first 1M blocks _without_ affecting blockstore or cometBFT state:
$ kava shard --home path/to/.kava --start 1000000 --end -1 --only-app-state`,
RunE: func(cmd *cobra.Command, args []string) error {
// read & validate flags
startBlock, err := cmd.Flags().GetInt64(flagShardStartBlock)
if err != nil {
return err
}
endBlock, err := cmd.Flags().GetInt64(flagShardEndBlock)
if err != nil {
return err
}
if (endBlock == 0 || endBlock < startBlock) && endBlock != shardEndBlockLatest {
return fmt.Errorf("end block (%d) must be greater than start block (%d)", endBlock, startBlock)
}
onlyAppState, err := cmd.Flags().GetBool(flagShardOnlyAppState)
if err != nil {
return err
}
clientCtx := client.GetClientContextFromCmd(cmd)
ctx := server.GetServerContextFromCmd(cmd)
ctx.Config.SetRoot(clientCtx.HomeDir)
//////////////////////////////
// Rollback state to endBlock
//////////////////////////////
// connect to database
db, err := opts.DBOpener(ctx.Viper, clientCtx.HomeDir, server.GetAppDBBackend(ctx.Viper))
if err != nil {
return err
}
// close db connection when done
defer func() {
if err := db.Close(); err != nil {
ctx.Logger.Error("error closing db", "error", err.Error())
}
}()
// get the multistore
app := opts.AppCreator(ctx.Logger, db, nil, ctx.Viper)
cms := app.CommitMultiStore()
multistore, ok := cms.(*rootmulti.Store)
if !ok {
return fmt.Errorf("only sharding of rootmulti.Store type is supported")
}
// handle desired endblock being latest
latest := multistore.LatestVersion()
fmt.Printf("latest height: %d\n", latest)
if endBlock == shardEndBlockLatest {
endBlock = latest
}
shardSize := endBlock - startBlock + 1
// error if requesting block range the database does not have
if endBlock > latest {
return fmt.Errorf("data does not contain end block (%d): latest version is %d", endBlock, latest)
}
fmt.Printf("pruning data in %s down to heights %d - %d (%d blocks)\n", clientCtx.HomeDir, startBlock, endBlock, shardSize)
// set pruning options to prevent no-ops from `PruneStores`
multistore.SetPruning(pruningtypes.PruningOptions{KeepRecent: uint64(shardSize), Interval: 0})
// rollback application state
if err = multistore.RollbackToVersion(endBlock); err != nil {
return fmt.Errorf("failed to rollback application state: %s", err)
}
// open block store & cometbft state
blockStore, stateStore, err := openCometBftDbs(ctx.Config)
if err != nil {
return fmt.Errorf("failed to open cometbft dbs: %s", err)
}
// prep for outputting progress repeatedly to same line
needsRollback := endBlock < latest
progress := "rolling back blockstore & cometbft state to height %d"
numChars := len(fmt.Sprintf(progress, latest))
clearLine := fmt.Sprintf("\r%s\r", strings.Repeat(" ", numChars))
printRollbackProgress := func(h int64) {
fmt.Print(clearLine)
fmt.Printf(progress, h)
}
// rollback tendermint db
height := latest
for height > endBlock {
printRollbackProgress(height - 1)
height, _, err = tmstate.Rollback(blockStore, stateStore, true)
if err != nil {
return fmt.Errorf("failed to rollback tendermint state: %w", err)
}
}
if needsRollback {
fmt.Println()
} else {
fmt.Printf("latest store height is already %d\n", latest)
}
//////////////////////////////
// Prune blocks to startBlock
//////////////////////////////
// enumerate all heights to prune
pruneHeights := make([]int64, 0, latest-shardSize)
for i := int64(1); i < startBlock; i++ {
pruneHeights = append(pruneHeights, i)
}
if len(pruneHeights) > 0 {
// prune application state
fmt.Printf("pruning application state to height %d\n", startBlock)
if err := multistore.PruneStores(true, pruneHeights); err != nil {
return fmt.Errorf("failed to prune application state: %s", err)
}
}
// get starting block of block store
baseBlock := blockStore.Base()
// only prune if data exists, otherwise blockStore.PruneBlocks will panic
if !onlyAppState && baseBlock < startBlock {
// prune block store
fmt.Printf("pruning block store from %d - %d\n", baseBlock, startBlock)
if _, err := blockStore.PruneBlocks(startBlock); err != nil {
return fmt.Errorf("failed to prune block store (retainHeight=%d): %s", startBlock, err)
}
// prune cometbft state
fmt.Printf("pruning cometbft state from %d - %d\n", baseBlock, startBlock)
if err := stateStore.PruneStates(baseBlock, startBlock); err != nil {
return fmt.Errorf("failed to prune cometbft state store (%d - %d): %s", baseBlock, startBlock, err)
}
} else {
fmt.Printf("blockstore and cometbft state begins at block %d\n", baseBlock)
}
// TODO: db compaction
return nil
},
}
cmd.Flags().String(flags.FlagHome, opts.DefaultNodeHome, "The application home directory")
cmd.Flags().Int64(flagShardStartBlock, 1, "Start block of data shard (inclusive)")
cmd.Flags().Int64(flagShardEndBlock, 0, "End block of data shard (inclusive)")
cmd.Flags().Bool(flagShardOnlyAppState, false, "Skip pruning of blockstore & cometbft state")
return cmd
}
// inspired by https://github.com/Kava-Labs/cometbft/blob/277b0853db3f67865a55aa1c54f59790b5f591be/node/node.go#L234
func openCometBftDbs(config *tmconfig.Config) (blockStore *store.BlockStore, stateStore tmstate.Store, err error) {
dbProvider := node.DefaultDBProvider
var blockStoreDB dbm.DB
blockStoreDB, err = dbProvider(&node.DBContext{ID: "blockstore", Config: config})
if err != nil {
return
}
blockStore = store.NewBlockStore(blockStoreDB)
stateDB, err := dbProvider(&node.DBContext{ID: "state", Config: config})
if err != nil {
return
}
stateStore = tmstate.NewStore(stateDB, tmstate.StoreOptions{
DiscardABCIResponses: config.Storage.DiscardABCIResponses,
})
return
}

6
go.mod
View File

@ -6,6 +6,7 @@ require (
cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/errors v1.0.0-beta.7
cosmossdk.io/math v1.0.0-beta.6.0.20230216172121-959ce49135e4 cosmossdk.io/math v1.0.0-beta.6.0.20230216172121-959ce49135e4
github.com/cenkalti/backoff/v4 v4.1.3 github.com/cenkalti/backoff/v4 v4.1.3
github.com/cometbft/cometbft-db v0.7.0
github.com/cosmos/cosmos-proto v1.0.0-beta.3 github.com/cosmos/cosmos-proto v1.0.0-beta.3
github.com/cosmos/cosmos-sdk v0.46.11 github.com/cosmos/cosmos-sdk v0.46.11
github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/go-bip39 v1.0.0
@ -62,7 +63,6 @@ require (
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect
github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect
github.com/cometbft/cometbft-db v0.7.0 // indirect
github.com/confio/ics23/go v0.9.0 // indirect github.com/confio/ics23/go v0.9.0 // indirect
github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/btcutil v1.0.5 // indirect
github.com/cosmos/gogoproto v1.4.6 // indirect github.com/cosmos/gogoproto v1.4.6 // indirect
@ -205,7 +205,7 @@ replace (
// Use rocksdb 7.9.2 // Use rocksdb 7.9.2
github.com/cometbft/cometbft-db => github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1 github.com/cometbft/cometbft-db => github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1
// Use cosmos-sdk fork with backported fix for unsafe-reset-all, staking transfer events, and custom tally handler support // Use cosmos-sdk fork with backported fix for unsafe-reset-all, staking transfer events, and custom tally handler support
github.com/cosmos/cosmos-sdk => github.com/kava-labs/cosmos-sdk v0.46.11-kava.3 github.com/cosmos/cosmos-sdk => github.com/kava-labs/cosmos-sdk v0.46.11-kava.4
// See https://github.com/cosmos/cosmos-sdk/pull/13093 // See https://github.com/cosmos/cosmos-sdk/pull/13093
github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2 github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2
// Use ethermint fork that respects min-gas-price with NoBaseFee true and london enabled, and includes eip712 support // Use ethermint fork that respects min-gas-price with NoBaseFee true and london enabled, and includes eip712 support
@ -217,7 +217,7 @@ replace (
// Downgraded to avoid bugs in following commits which causes "version does not exist" errors // Downgraded to avoid bugs in following commits which causes "version does not exist" errors
github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
// Use cometbft fork of tendermint // Use cometbft fork of tendermint
github.com/tendermint/tendermint => github.com/kava-labs/cometbft v0.34.27-kava.0 github.com/tendermint/tendermint => github.com/kava-labs/cometbft v0.34.27-kava.1
// Indirect dependencies still use tendermint/tm-db // Indirect dependencies still use tendermint/tm-db
github.com/tendermint/tm-db => github.com/kava-labs/tm-db v0.6.7-kava.4 github.com/tendermint/tm-db => github.com/kava-labs/tm-db v0.6.7-kava.4
) )

8
go.sum
View File

@ -802,12 +802,12 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/kava-labs/cometbft v0.34.27-kava.0 h1:FUEGRkF3xtrJH+h9A5G4eA2skf7QaNoOCPaoVqHkh8k= github.com/kava-labs/cometbft v0.34.27-kava.1 h1:JkTspNCrz9matgrr7nsWgEkgNzDz5YwZhR5jZyxVt/0=
github.com/kava-labs/cometbft v0.34.27-kava.0/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/kava-labs/cometbft v0.34.27-kava.1/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1 h1:EZnZAkZ+dqK+1OM4AK+e6wYH8a5xuyg4yFTR4Ez3AXk= github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1 h1:EZnZAkZ+dqK+1OM4AK+e6wYH8a5xuyg4yFTR4Ez3AXk=
github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1/go.mod h1:mI/4J4IxRzPrXvMiwefrt0fucGwaQ5Hm9IKS7HnoJeI= github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1/go.mod h1:mI/4J4IxRzPrXvMiwefrt0fucGwaQ5Hm9IKS7HnoJeI=
github.com/kava-labs/cosmos-sdk v0.46.11-kava.3 h1:TOhyyW/xHso/9uIOgYdsrOWDIhXi6foORWZxVRe/wS0= github.com/kava-labs/cosmos-sdk v0.46.11-kava.4 h1:lCcClhrGCwHunwbSG/zmY8o/g8rE6GYH1x2OOPMkk0g=
github.com/kava-labs/cosmos-sdk v0.46.11-kava.3/go.mod h1:bSUUbmVwWkv1ZNVTWrQHa/i+73xIUvYYPsCvl5doiCs= github.com/kava-labs/cosmos-sdk v0.46.11-kava.4/go.mod h1:VVdf1w8CSIhh4FnlxOGVVAlvUnvXrS8VV6DwUBCuAOs=
github.com/kava-labs/ethermint v0.21.0-kava-v24-1 h1:XgEeXbLtGpOxSt7yFVznrjics5mRsUpMMfqlrhhH2pc= github.com/kava-labs/ethermint v0.21.0-kava-v24-1 h1:XgEeXbLtGpOxSt7yFVznrjics5mRsUpMMfqlrhhH2pc=
github.com/kava-labs/ethermint v0.21.0-kava-v24-1/go.mod h1:rdm6AinxZ4dzPEv/cjH+/AGyTbKufJ3RE7M2MDyklH0= github.com/kava-labs/ethermint v0.21.0-kava-v24-1/go.mod h1:rdm6AinxZ4dzPEv/cjH+/AGyTbKufJ3RE7M2MDyklH0=
github.com/kava-labs/tm-db v0.6.7-kava.4 h1:M2RibOKmbi+k2OhAFry8z9+RJF0CYuDETB7/PrSdoro= github.com/kava-labs/tm-db v0.6.7-kava.4 h1:M2RibOKmbi+k2OhAFry8z9+RJF0CYuDETB7/PrSdoro=

@ -1 +1 @@
Subproject commit e1085562d203fd81c5dd8576170b29715b2de9ef Subproject commit 075328bbb54010f2dcddd29fe0a0a0cfccaf41e2