From 0156b0e64588486d352facf46eb4ffc4583807d8 Mon Sep 17 00:00:00 2001 From: Robert Pirtle Date: Tue, 28 Mar 2023 15:32:36 -0700 Subject: [PATCH] support chian upgrades in e2e tests (#1513) * update kvtool * add env vars for e2e automated upgrades * run kvtool with upgrade flags when enabled * add e2e test placeholder for upgrade handler tests * allow override of kvtool's kava.configTemplate * use the correct image tag when running e2e tests * update kvtool * always docker-build when running make test-e2e * refactor skip shutdown to always skip shutdown * add CtxAtHeight grpc helper * add Community & Earn queriers * expose UpgradeHeight in suite * document e2e chain upgrades * add reference to example branch --- Makefile | 2 +- tests/e2e/.env | 15 ++++++++ tests/e2e/e2e_upgrade_handler_test.go | 12 +++++++ tests/e2e/kvtool | 2 +- tests/e2e/readme.md | 27 ++++++++++++-- tests/e2e/runner/main.go | 25 ++++++++++++- tests/e2e/testutil/chain.go | 7 ++++ tests/e2e/testutil/config.go | 52 +++++++++++++++++++++------ tests/e2e/testutil/suite.go | 22 ++++++++++-- tests/util/grpc.go | 10 ++++++ 10 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 tests/e2e/e2e_upgrade_handler_test.go diff --git a/Makefile b/Makefile index 1534604f..41422288 100644 --- a/Makefile +++ b/Makefile @@ -290,7 +290,7 @@ test-basic: test @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 4 -v -timeout 2m # run end-to-end tests (local docker container must be built, see docker-build) -test-e2e: +test-e2e: docker-build export E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'; \ go test -failfast -count=1 -v ./tests/e2e/... diff --git a/tests/e2e/.env b/tests/e2e/.env index d68ae06e..545d3673 100644 --- a/tests/e2e/.env +++ b/tests/e2e/.env @@ -1,9 +1,24 @@ # E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC is for a funded account used to intialize all new testing accounts. E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold' +# E2E_KVTOOL_KAVA_CONFIG_TEMPLATE is the kvtool template used to start the chain. See the `kava.configTemplate` flag in kvtool. +# Note that the config tempalte must support overriding the docker image tag via the KAVA_TAG variable. +E2E_KVTOOL_KAVA_CONFIG_TEMPLATE="master" + # E2E_INCLUDE_IBC_TESTS when true will start a 2nd chain & open an IBC channel. It will enable all IBC tests. E2E_INCLUDE_IBC_TESTS=true # E2E_SKIP_SHUTDOWN when true will keep the networks running after tests complete (pass or fail) # This is useful for debugging chain state when writing tests. E2E_SKIP_SHUTDOWN=false + +# The following variables should be defined to run an upgrade. +# E2E_INCLUDE_AUTOMATED_UPGRADE when true enables the automated upgrade & corresponding tests in the suite. +E2E_INCLUDE_AUTOMATED_UPGRADE=false +# E2E_KAVA_UPGRADE_NAME is the name of the upgrade that must be in the current local image. +E2E_KAVA_UPGRADE_NAME= +# E2E_KAVA_UPGRADE_HEIGHT is the height at which the upgrade will be applied. +# If IBC tests are enabled this should be >50. Otherwise, this should be >10. +E2E_KAVA_UPGRADE_HEIGHT= +# E2E_KAVA_UPGRADE_BASE_IMAGE_TAG is the tag of the docker image the chain should upgrade from. +E2E_KAVA_UPGRADE_BASE_IMAGE_TAG= diff --git a/tests/e2e/e2e_upgrade_handler_test.go b/tests/e2e/e2e_upgrade_handler_test.go new file mode 100644 index 00000000..f7c8d8b7 --- /dev/null +++ b/tests/e2e/e2e_upgrade_handler_test.go @@ -0,0 +1,12 @@ +package e2e_test + +import "fmt" + +// TestUpgradeHandler can be used to run tests post-upgrade. If an upgrade is enabled, all tests +// are run against the upgraded chain. However, this file is a good place to consolidate all +// acceptance tests for a given set of upgrade handlers. +func (suite IntegrationTestSuite) TestUpgradeHandler() { + suite.SkipIfUpgradeDisabled() + fmt.Println("An upgrade has run!") + suite.True(true) +} diff --git a/tests/e2e/kvtool b/tests/e2e/kvtool index c80093f2..8adc0437 160000 --- a/tests/e2e/kvtool +++ b/tests/e2e/kvtool @@ -1 +1 @@ -Subproject commit c80093f2b9133c30e9e24da32a74a870d6e33b57 +Subproject commit 8adc0437e849f86bdcb2df3833abbfbd0980775e diff --git a/tests/e2e/readme.md b/tests/e2e/readme.md index fe5084bc..2119964f 100644 --- a/tests/e2e/readme.md +++ b/tests/e2e/readme.md @@ -5,9 +5,9 @@ and then runs tests against the running network. It is a git sub-repository in t present, you must initialize the subrepo: `git submodule update --init`. Steps to run -1. Build a Kava docker image tagged `kava/kava:local`: `make build-docker` -2. Ensure latest `kvtool` is installed: `make update-kvtool` -3. Run the test suite: `make test-e2e` +1. Ensure latest `kvtool` is installed: `make update-kvtool` +2. Run the test suite: `make test-e2e` + This will build a docker image tagged `kava/kava:local` that will be run by kvtool. **Note:** The suite will use your locally installed `kvtool` if present. If not present, it will be installed. If the `kvtool` repo is updated, you must manually update your existing local binary: `make update-kvtool` @@ -61,3 +61,24 @@ The IBC network runs kava with a different chain id and staking denom (see [runn The IBC chain queriers & accounts are accessible via `suite.Ibc`. IBC tests can be disabled by setting `E2E_INCLUDE_IBC_TESTS` to `false`. + +## Chain Upgrades + +When a named upgrade handler is included in the current working repo of Kava, the e2e test suite can +be configured to run all the tests on the upgraded chain. This includes the ability to add additional +tests to verify and do acceptance on the post-upgrade chain. + +This configuration is controlled by the following env variables: +* `E2E_INCLUDE_AUTOMATED_UPGRADE` - toggles on the upgrade functionality. Must be set to `true`. +* `E2E_KAVA_UPGRADE_NAME` - the named upgrade, likely defined in [`app/upgrades.go`](../../app/upgrades.go) +* `E2E_KAVA_UPGRADE_HEIGHT` - the height at which to run the upgrade +* `E2E_KAVA_UPGRADE_BASE_IMAGE_TAG` - the [kava docker image tag](https://hub.docker.com/r/kava/kava/tags) to base the upgrade on + +When all these are set, the chain is started with the binary contained in the docker image tagged +`E2E_KAVA_UPGRADE_BASE_IMAGE_TAG`. Then an upgrade proposal is submitted with the desired name and +height. The chain runs until that height and then is shutdown due to needing the upgrade. The chain +is restarted with the local repo's Kava code and the upgrade is run. Once completed, the whole test +suite is run. + +For a full example of how this looks, see [this commit](https://github.com/Kava-Labs/kava/commit/5da48c892f0a5837141fc7de88632c7c68fff4ae) +on the [example/e2e-test-upgrade-handler](https://github.com/Kava-Labs/kava/tree/example/e2e-test-upgrade-handler) branch. diff --git a/tests/e2e/runner/main.go b/tests/e2e/runner/main.go index 4aaaad5d..825b255d 100644 --- a/tests/e2e/runner/main.go +++ b/tests/e2e/runner/main.go @@ -12,8 +12,17 @@ import ( ) type Config struct { + KavaConfigTemplate string + ImageTag string IncludeIBC bool + + EnableAutomatedUpgrade bool + KavaUpgradeName string + KavaUpgradeHeight int64 + KavaUpgradeBaseImageTag string + + SkipShutdown bool } // NodeRunner is responsible for starting and managing docker containers to run a node. @@ -45,13 +54,23 @@ func (k *KavaNodeRunner) StartChains() Chains { } log.Println("starting kava node") - kvtoolArgs := []string{"testnet", "bootstrap"} + kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate} if k.config.IncludeIBC { kvtoolArgs = append(kvtoolArgs, "--ibc") } + if k.config.EnableAutomatedUpgrade { + kvtoolArgs = append(kvtoolArgs, + "--upgrade-name", k.config.KavaUpgradeName, + "--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight), + "--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag, + ) + } startKavaCmd := exec.Command("kvtool", kvtoolArgs...) + startKavaCmd.Env = os.Environ() + startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag)) startKavaCmd.Stdout = os.Stdout startKavaCmd.Stderr = os.Stderr + log.Println(startKavaCmd.String()) if err := startKavaCmd.Run(); err != nil { panic(fmt.Sprintf("failed to start kava: %s", err.Error())) } @@ -74,6 +93,10 @@ func (k *KavaNodeRunner) StartChains() Chains { } func (k *KavaNodeRunner) Shutdown() { + if k.config.SkipShutdown { + log.Printf("would shut down but SkipShutdown is true") + return + } log.Println("shutting down kava node") shutdownKavaCmd := exec.Command("kvtool", "testnet", "down") shutdownKavaCmd.Stdout = os.Stdout diff --git a/tests/e2e/testutil/chain.go b/tests/e2e/testutil/chain.go index b0add223..9487af3a 100644 --- a/tests/e2e/testutil/chain.go +++ b/tests/e2e/testutil/chain.go @@ -18,6 +18,8 @@ import ( kavaparams "github.com/kava-labs/kava/app/params" "github.com/kava-labs/kava/tests/e2e/runner" "github.com/kava-labs/kava/tests/util" + communitytypes "github.com/kava-labs/kava/x/community/types" + earntypes "github.com/kava-labs/kava/x/earn/types" ) // Chain wraps query clients & accounts for a network @@ -30,8 +32,11 @@ type Chain struct { ChainId string EvmClient *ethclient.Client + Auth authtypes.QueryClient Bank banktypes.QueryClient + Community communitytypes.QueryClient + Earn earntypes.QueryClient Tm tmservice.ServiceClient Tx txtypes.ServiceClient } @@ -61,6 +66,8 @@ func NewChain(t *testing.T, details *runner.ChainDetails, fundedAccountMnemonic chain.Auth = authtypes.NewQueryClient(grpcConn) chain.Bank = banktypes.NewQueryClient(grpcConn) + chain.Community = communitytypes.NewQueryClient(grpcConn) + chain.Earn = earntypes.NewQueryClient(grpcConn) chain.Tm = tmservice.NewServiceClient(grpcConn) chain.Tx = txtypes.NewServiceClient(grpcConn) diff --git a/tests/e2e/testutil/config.go b/tests/e2e/testutil/config.go index 2a9af22f..fc384ff7 100644 --- a/tests/e2e/testutil/config.go +++ b/tests/e2e/testutil/config.go @@ -16,29 +16,51 @@ func init() { type SuiteConfig struct { // A funded account used to fnd all other accounts. FundedAccountMnemonic string + // The kava.configTemplate flag to be passed to kvtool, usually "master". + // This allows one to change the base genesis used to start the chain. + KavaConfigTemplate string // Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests. IncludeIbcTests bool + + // Whether or not to run a chain upgrade & run post-upgrade tests. Use `suite.SkipIfUpgradeDisabled()` in post-upgrade tests. + IncludeAutomatedUpgrade bool + // Name of the upgrade, if upgrade is enabled. + KavaUpgradeName string + // Height upgrade will be applied to the test chain, if upgrade is enabled. + KavaUpgradeHeight int64 + // Tag of kava docker image that will be upgraded to the current image before tests are run, if upgrade is enabled. + KavaUpgradeBaseImageTag string + // When true, the chains will remain running after tests complete (pass or fail) SkipShutdown bool } func ParseSuiteConfig() SuiteConfig { - // this mnemonic is expected to be a funded account that can seed the funds for all - // new accounts created during tests. it will be available under Accounts["whale"] - fundedAccountMnemonic := os.Getenv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC") - if fundedAccountMnemonic == "" { - panic("no E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC provided") + config := SuiteConfig{ + // this mnemonic is expected to be a funded account that can seed the funds for all + // new accounts created during tests. it will be available under Accounts["whale"] + FundedAccountMnemonic: nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"), + KavaConfigTemplate: nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"), + IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"), + IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"), } - var skipShutdown bool + skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN") if skipShutdownEnv != "" { - skipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN") + config.SkipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN") } - return SuiteConfig{ - FundedAccountMnemonic: fundedAccountMnemonic, - IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"), - SkipShutdown: skipShutdown, + + if config.IncludeAutomatedUpgrade { + config.KavaUpgradeName = nonemptyStringEnv("E2E_KAVA_UPGRADE_NAME") + config.KavaUpgradeBaseImageTag = nonemptyStringEnv("E2E_KAVA_UPGRADE_BASE_IMAGE_TAG") + upgradeHeight, err := strconv.ParseInt(nonemptyStringEnv("E2E_KAVA_UPGRADE_HEIGHT"), 10, 64) + if err != nil { + panic(fmt.Sprintf("E2E_KAVA_UPGRADE_HEIGHT must be a number: %s", err)) + } + config.KavaUpgradeHeight = upgradeHeight } + + return config } func mustParseBool(name string) bool { @@ -52,3 +74,11 @@ func mustParseBool(name string) bool { } return value } + +func nonemptyStringEnv(name string) string { + value := os.Getenv(name) + if value == "" { + panic(fmt.Sprintf("no %s env variable provided", name)) + } + return value +} diff --git a/tests/e2e/testutil/suite.go b/tests/e2e/testutil/suite.go index e29f75b8..594d7aa3 100644 --- a/tests/e2e/testutil/suite.go +++ b/tests/e2e/testutil/suite.go @@ -28,6 +28,8 @@ type E2eTestSuite struct { Kava *Chain Ibc *Chain + + UpgradeHeight int64 } func (suite *E2eTestSuite) SetupSuite() { @@ -37,10 +39,20 @@ func (suite *E2eTestSuite) SetupSuite() { suiteConfig := ParseSuiteConfig() suite.config = suiteConfig + suite.UpgradeHeight = suiteConfig.KavaUpgradeHeight runnerConfig := runner.Config{ + KavaConfigTemplate: suiteConfig.KavaConfigTemplate, + IncludeIBC: suiteConfig.IncludeIbcTests, ImageTag: "local", + + EnableAutomatedUpgrade: suiteConfig.IncludeAutomatedUpgrade, + KavaUpgradeName: suiteConfig.KavaUpgradeName, + KavaUpgradeHeight: suiteConfig.KavaUpgradeHeight, + KavaUpgradeBaseImageTag: suiteConfig.KavaUpgradeBaseImageTag, + + SkipShutdown: suiteConfig.SkipShutdown, } suite.runner = runner.NewKavaNode(runnerConfig) @@ -70,9 +82,7 @@ func (suite *E2eTestSuite) TearDownSuite() { suite.Ibc.Shutdown() } // gracefully shutdown docker container(s) - if !suite.config.SkipShutdown { - suite.runner.Shutdown() - } + suite.runner.Shutdown() } func (suite *E2eTestSuite) SkipIfIbcDisabled() { @@ -80,3 +90,9 @@ func (suite *E2eTestSuite) SkipIfIbcDisabled() { suite.T().SkipNow() } } + +func (suite *E2eTestSuite) SkipIfUpgradeDisabled() { + if !suite.config.IncludeAutomatedUpgrade { + suite.T().SkipNow() + } +} diff --git a/tests/util/grpc.go b/tests/util/grpc.go index 99221543..d7c17ba5 100644 --- a/tests/util/grpc.go +++ b/tests/util/grpc.go @@ -1,12 +1,17 @@ package util import ( + "context" "crypto/tls" "fmt" "net/url" + "strconv" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + + grpctypes "github.com/cosmos/cosmos-sdk/types/grpc" ) // NewGrpcConnection parses a GRPC endpoint and creates a connection to it @@ -34,3 +39,8 @@ func NewGrpcConnection(endpoint string) (*grpc.ClientConn, error) { return grpcConn, nil } + +func CtxAtHeight(height int64) context.Context { + heightStr := strconv.FormatInt(height, 10) + return metadata.AppendToOutgoingContext(context.Background(), grpctypes.GRPCBlockHeightHeader, heightStr) +}