From 58621577ae93005f20e93cfa6a2f421485ddcc19 Mon Sep 17 00:00:00 2001 From: Robert Pirtle Date: Fri, 5 Jan 2024 11:25:53 -0800 Subject: [PATCH] feat(cli): add shard CLI command (#1785) * stub out shard CLI command * prune blocks before and after desired range * update rollback to use patched cometbft * temp override for local patched versions * handle pruning cometbft & block store state * include docs & support -1 for "latest" * update changelog * add --only-app-state flag to match cosmos-sdk prune cmd * give -1 magic number a name & reuse home from ctx * refactor to only open state.db & blockstore.db once * write rollback progress to one line * prevent attempting rollback of future blocks * make shard inclusive of endblock * use tagged cosmo-sdk & cometbft versions --- CHANGELOG.md | 4 + cmd/kava/cmd/root.go | 1 + cmd/kava/cmd/shard.go | 232 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 6 +- go.sum | 8 +- tests/e2e/kvtool | 2 +- 6 files changed, 245 insertions(+), 8 deletions(-) create mode 100644 cmd/kava/cmd/shard.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d941c303..2a083309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Features + +- (cli) [#1785] Add `shard` CLI command to support creating partitions of data for standalone nodes ## [v0.25.0] @@ -317,6 +320,7 @@ the [changelog](https://github.com/cosmos/cosmos-sdk/blob/v0.38.4/CHANGELOG.md). - [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run large-scale simulations remotely using aws-batch +[#1785]: https://github.com/Kava-Labs/kava/pull/1785 [#1784]: https://github.com/Kava-Labs/kava/pull/1784 [#1776]: https://github.com/Kava-Labs/kava/pull/1776 [#1770]: https://github.com/Kava-Labs/kava/pull/1770 diff --git a/cmd/kava/cmd/root.go b/cmd/kava/cmd/root.go index 7747b556..70df2de9 100644 --- a/cmd/kava/cmd/root.go +++ b/cmd/kava/cmd/root.go @@ -123,5 +123,6 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de newQueryCmd(), newTxCmd(), keyCommands(app.DefaultNodeHome), + newShardCmd(opts), ) } diff --git a/cmd/kava/cmd/shard.go b/cmd/kava/cmd/shard.go new file mode 100644 index 00000000..c43ad7dd --- /dev/null +++ b/cmd/kava/cmd/shard.go @@ -0,0 +1,232 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + + dbm "github.com/cometbft/cometbft-db" + + "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/flags" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + "github.com/cosmos/cosmos-sdk/server" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + + tmconfig "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/node" + tmstate "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/store" + + ethermintserver "github.com/evmos/ethermint/server" +) + +const ( + flagShardStartBlock = "start" + flagShardEndBlock = "end" + flagShardOnlyAppState = "only-app-state" + // TODO: --preserve flag for creating & operating on a copy? + + // allow using -1 to mean "latest" (perform no rollbacks) + shardEndBlockLatest = -1 +) + +func newShardCmd(opts ethermintserver.StartOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "shard --home --start --end [--only-app-state]", + Short: "Strip all blocks from the database outside of a given range", + Long: `shard opens a local kava home directory's databases and removes all blocks outside a range defined by --start and --end. The range is inclusive of the end block. + +It works by first rolling back the latest state to the block before the end block, and then by pruning all state before the start block. + +Setting the end block to -1 signals to keep the latest block (no rollbacks). + +The --only-app-state flag can be used to skip the pruning of the blockstore and cometbft state. This matches the functionality of the cosmos-sdk's "prune" command. Note that rolled back blocks will still affect all stores. + +WARNING: this is a destructive action.`, + Example: `Create a 1M block data shard (keeps blocks kava 1,000,000 to 2,000,000) +$ kava shard --home path/to/.kava --start 1000000 --end 2000000 + +Prune all blocks up to 5,000,000: +$ kava shard --home path/to/.kava --start 5000000 --end -1 + +Prune first 1M blocks _without_ affecting blockstore or cometBFT state: +$ kava shard --home path/to/.kava --start 1000000 --end -1 --only-app-state`, + RunE: func(cmd *cobra.Command, args []string) error { + // read & validate flags + startBlock, err := cmd.Flags().GetInt64(flagShardStartBlock) + if err != nil { + return err + } + endBlock, err := cmd.Flags().GetInt64(flagShardEndBlock) + if err != nil { + return err + } + if (endBlock == 0 || endBlock < startBlock) && endBlock != shardEndBlockLatest { + return fmt.Errorf("end block (%d) must be greater than start block (%d)", endBlock, startBlock) + } + onlyAppState, err := cmd.Flags().GetBool(flagShardOnlyAppState) + if err != nil { + return err + } + + clientCtx := client.GetClientContextFromCmd(cmd) + + ctx := server.GetServerContextFromCmd(cmd) + ctx.Config.SetRoot(clientCtx.HomeDir) + + ////////////////////////////// + // Rollback state to endBlock + ////////////////////////////// + + // connect to database + db, err := opts.DBOpener(ctx.Viper, clientCtx.HomeDir, server.GetAppDBBackend(ctx.Viper)) + if err != nil { + return err + } + + // close db connection when done + defer func() { + if err := db.Close(); err != nil { + ctx.Logger.Error("error closing db", "error", err.Error()) + } + }() + + // get the multistore + app := opts.AppCreator(ctx.Logger, db, nil, ctx.Viper) + cms := app.CommitMultiStore() + multistore, ok := cms.(*rootmulti.Store) + if !ok { + return fmt.Errorf("only sharding of rootmulti.Store type is supported") + } + + // handle desired endblock being latest + latest := multistore.LatestVersion() + fmt.Printf("latest height: %d\n", latest) + if endBlock == shardEndBlockLatest { + endBlock = latest + } + shardSize := endBlock - startBlock + 1 + + // error if requesting block range the database does not have + if endBlock > latest { + return fmt.Errorf("data does not contain end block (%d): latest version is %d", endBlock, latest) + } + + fmt.Printf("pruning data in %s down to heights %d - %d (%d blocks)\n", clientCtx.HomeDir, startBlock, endBlock, shardSize) + + // set pruning options to prevent no-ops from `PruneStores` + multistore.SetPruning(pruningtypes.PruningOptions{KeepRecent: uint64(shardSize), Interval: 0}) + + // rollback application state + if err = multistore.RollbackToVersion(endBlock); err != nil { + return fmt.Errorf("failed to rollback application state: %s", err) + } + + // open block store & cometbft state + blockStore, stateStore, err := openCometBftDbs(ctx.Config) + if err != nil { + return fmt.Errorf("failed to open cometbft dbs: %s", err) + } + + // prep for outputting progress repeatedly to same line + needsRollback := endBlock < latest + progress := "rolling back blockstore & cometbft state to height %d" + numChars := len(fmt.Sprintf(progress, latest)) + clearLine := fmt.Sprintf("\r%s\r", strings.Repeat(" ", numChars)) + printRollbackProgress := func(h int64) { + fmt.Print(clearLine) + fmt.Printf(progress, h) + } + + // rollback tendermint db + height := latest + for height > endBlock { + printRollbackProgress(height - 1) + height, _, err = tmstate.Rollback(blockStore, stateStore, true) + if err != nil { + return fmt.Errorf("failed to rollback tendermint state: %w", err) + } + } + + if needsRollback { + fmt.Println() + } else { + fmt.Printf("latest store height is already %d\n", latest) + } + + ////////////////////////////// + // Prune blocks to startBlock + ////////////////////////////// + + // enumerate all heights to prune + pruneHeights := make([]int64, 0, latest-shardSize) + for i := int64(1); i < startBlock; i++ { + pruneHeights = append(pruneHeights, i) + } + + if len(pruneHeights) > 0 { + // prune application state + fmt.Printf("pruning application state to height %d\n", startBlock) + if err := multistore.PruneStores(true, pruneHeights); err != nil { + return fmt.Errorf("failed to prune application state: %s", err) + } + } + + // get starting block of block store + baseBlock := blockStore.Base() + + // only prune if data exists, otherwise blockStore.PruneBlocks will panic + if !onlyAppState && baseBlock < startBlock { + // prune block store + fmt.Printf("pruning block store from %d - %d\n", baseBlock, startBlock) + if _, err := blockStore.PruneBlocks(startBlock); err != nil { + return fmt.Errorf("failed to prune block store (retainHeight=%d): %s", startBlock, err) + } + + // prune cometbft state + fmt.Printf("pruning cometbft state from %d - %d\n", baseBlock, startBlock) + if err := stateStore.PruneStates(baseBlock, startBlock); err != nil { + return fmt.Errorf("failed to prune cometbft state store (%d - %d): %s", baseBlock, startBlock, err) + } + } else { + fmt.Printf("blockstore and cometbft state begins at block %d\n", baseBlock) + } + + // TODO: db compaction + + return nil + }, + } + + cmd.Flags().String(flags.FlagHome, opts.DefaultNodeHome, "The application home directory") + cmd.Flags().Int64(flagShardStartBlock, 1, "Start block of data shard (inclusive)") + cmd.Flags().Int64(flagShardEndBlock, 0, "End block of data shard (inclusive)") + cmd.Flags().Bool(flagShardOnlyAppState, false, "Skip pruning of blockstore & cometbft state") + + return cmd +} + +// inspired by https://github.com/Kava-Labs/cometbft/blob/277b0853db3f67865a55aa1c54f59790b5f591be/node/node.go#L234 +func openCometBftDbs(config *tmconfig.Config) (blockStore *store.BlockStore, stateStore tmstate.Store, err error) { + dbProvider := node.DefaultDBProvider + + var blockStoreDB dbm.DB + blockStoreDB, err = dbProvider(&node.DBContext{ID: "blockstore", Config: config}) + if err != nil { + return + } + blockStore = store.NewBlockStore(blockStoreDB) + + stateDB, err := dbProvider(&node.DBContext{ID: "state", Config: config}) + if err != nil { + return + } + + stateStore = tmstate.NewStore(stateDB, tmstate.StoreOptions{ + DiscardABCIResponses: config.Storage.DiscardABCIResponses, + }) + + return +} diff --git a/go.mod b/go.mod index c98457c8..6a531a91 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-beta.6.0.20230216172121-959ce49135e4 github.com/cenkalti/backoff/v4 v4.1.3 + github.com/cometbft/cometbft-db v0.7.0 github.com/cosmos/cosmos-proto v1.0.0-beta.3 github.com/cosmos/cosmos-sdk v0.46.11 github.com/cosmos/go-bip39 v1.0.0 @@ -62,7 +63,6 @@ require ( github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect - github.com/cometbft/cometbft-db v0.7.0 // indirect github.com/confio/ics23/go v0.9.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/gogoproto v1.4.6 // indirect @@ -205,7 +205,7 @@ replace ( // Use rocksdb 7.9.2 github.com/cometbft/cometbft-db => github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1 // Use cosmos-sdk fork with backported fix for unsafe-reset-all, staking transfer events, and custom tally handler support - github.com/cosmos/cosmos-sdk => github.com/kava-labs/cosmos-sdk v0.46.11-kava.3 + github.com/cosmos/cosmos-sdk => github.com/kava-labs/cosmos-sdk v0.46.11-kava.4 // See https://github.com/cosmos/cosmos-sdk/pull/13093 github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2 // Use ethermint fork that respects min-gas-price with NoBaseFee true and london enabled, and includes eip712 support @@ -217,7 +217,7 @@ replace ( // Downgraded to avoid bugs in following commits which causes "version does not exist" errors github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // Use cometbft fork of tendermint - github.com/tendermint/tendermint => github.com/kava-labs/cometbft v0.34.27-kava.0 + github.com/tendermint/tendermint => github.com/kava-labs/cometbft v0.34.27-kava.1 // Indirect dependencies still use tendermint/tm-db github.com/tendermint/tm-db => github.com/kava-labs/tm-db v0.6.7-kava.4 ) diff --git a/go.sum b/go.sum index 95d73f27..dc4ab787 100644 --- a/go.sum +++ b/go.sum @@ -802,12 +802,12 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kava-labs/cometbft v0.34.27-kava.0 h1:FUEGRkF3xtrJH+h9A5G4eA2skf7QaNoOCPaoVqHkh8k= -github.com/kava-labs/cometbft v0.34.27-kava.0/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= +github.com/kava-labs/cometbft v0.34.27-kava.1 h1:JkTspNCrz9matgrr7nsWgEkgNzDz5YwZhR5jZyxVt/0= +github.com/kava-labs/cometbft v0.34.27-kava.1/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1 h1:EZnZAkZ+dqK+1OM4AK+e6wYH8a5xuyg4yFTR4Ez3AXk= github.com/kava-labs/cometbft-db v0.7.0-rocksdb-v7.9.2-kava.1/go.mod h1:mI/4J4IxRzPrXvMiwefrt0fucGwaQ5Hm9IKS7HnoJeI= -github.com/kava-labs/cosmos-sdk v0.46.11-kava.3 h1:TOhyyW/xHso/9uIOgYdsrOWDIhXi6foORWZxVRe/wS0= -github.com/kava-labs/cosmos-sdk v0.46.11-kava.3/go.mod h1:bSUUbmVwWkv1ZNVTWrQHa/i+73xIUvYYPsCvl5doiCs= +github.com/kava-labs/cosmos-sdk v0.46.11-kava.4 h1:lCcClhrGCwHunwbSG/zmY8o/g8rE6GYH1x2OOPMkk0g= +github.com/kava-labs/cosmos-sdk v0.46.11-kava.4/go.mod h1:VVdf1w8CSIhh4FnlxOGVVAlvUnvXrS8VV6DwUBCuAOs= github.com/kava-labs/ethermint v0.21.0-kava-v24-1 h1:XgEeXbLtGpOxSt7yFVznrjics5mRsUpMMfqlrhhH2pc= github.com/kava-labs/ethermint v0.21.0-kava-v24-1/go.mod h1:rdm6AinxZ4dzPEv/cjH+/AGyTbKufJ3RE7M2MDyklH0= github.com/kava-labs/tm-db v0.6.7-kava.4 h1:M2RibOKmbi+k2OhAFry8z9+RJF0CYuDETB7/PrSdoro= diff --git a/tests/e2e/kvtool b/tests/e2e/kvtool index e1085562..075328bb 160000 --- a/tests/e2e/kvtool +++ b/tests/e2e/kvtool @@ -1 +1 @@ -Subproject commit e1085562d203fd81c5dd8576170b29715b2de9ef +Subproject commit 075328bbb54010f2dcddd29fe0a0a0cfccaf41e2