support chian upgrades in e2e tests (#1513)

* update kvtool

* add env vars for e2e automated upgrades

* run kvtool with upgrade flags when enabled

* add e2e test placeholder for upgrade handler tests

* allow override of kvtool's kava.configTemplate

* use the correct image tag when running e2e tests

* update kvtool

* always docker-build when running make test-e2e

* refactor skip shutdown to always skip shutdown

* add CtxAtHeight grpc helper

* add Community & Earn queriers

* expose UpgradeHeight in suite

* document e2e chain upgrades

* add reference to example branch

Makefile

@@ -290,7 +290,7 @@ test-basic: test
 	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 4 -v -timeout 2m
 
 # run end-to-end tests (local docker container must be built, see docker-build)
-test-e2e:
+test-e2e: docker-build
 	export E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'; \
 	go test -failfast -count=1 -v ./tests/e2e/...
 

tests/e2e/.env

@@ -1,9 +1,24 @@
 # E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC is for a funded account used to intialize all new testing accounts.
 E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'
 
+# E2E_KVTOOL_KAVA_CONFIG_TEMPLATE is the kvtool template used to start the chain. See the `kava.configTemplate` flag in kvtool.
+# Note that the config tempalte must support overriding the docker image tag via the KAVA_TAG variable.
+E2E_KVTOOL_KAVA_CONFIG_TEMPLATE="master"
+
 # E2E_INCLUDE_IBC_TESTS when true will start a 2nd chain & open an IBC channel. It will enable all IBC tests.
 E2E_INCLUDE_IBC_TESTS=true
 
 # E2E_SKIP_SHUTDOWN when true will keep the networks running after tests complete (pass or fail)
 # This is useful for debugging chain state when writing tests.
 E2E_SKIP_SHUTDOWN=false
+
+# The following variables should be defined to run an upgrade.
+# E2E_INCLUDE_AUTOMATED_UPGRADE when true enables the automated upgrade & corresponding tests in the suite.
+E2E_INCLUDE_AUTOMATED_UPGRADE=false
+# E2E_KAVA_UPGRADE_NAME is the name of the upgrade that must be in the current local image.
+E2E_KAVA_UPGRADE_NAME=
+# E2E_KAVA_UPGRADE_HEIGHT is the height at which the upgrade will be applied.
+# If IBC tests are enabled this should be >50. Otherwise, this should be >10.
+E2E_KAVA_UPGRADE_HEIGHT=
+# E2E_KAVA_UPGRADE_BASE_IMAGE_TAG is the tag of the docker image the chain should upgrade from.
+E2E_KAVA_UPGRADE_BASE_IMAGE_TAG=

tests/e2e/e2e_upgrade_handler_test.go

@@ -0,0 +1,12 @@
+package e2e_test
+
+import "fmt"
+
+// TestUpgradeHandler can be used to run tests post-upgrade. If an upgrade is enabled, all tests
+// are run against the upgraded chain. However, this file is a good place to consolidate all
+// acceptance tests for a given set of upgrade handlers.
+func (suite IntegrationTestSuite) TestUpgradeHandler() {
+	suite.SkipIfUpgradeDisabled()
+	fmt.Println("An upgrade has run!")
+	suite.True(true)
+}

tests/e2e/kvtool

@@ -1 +1 @@
-Subproject commit c80093f2b9133c30e9e24da32a74a870d6e33b57
+Subproject commit 8adc0437e849f86bdcb2df3833abbfbd0980775e

tests/e2e/readme.md

@@ -5,9 +5,9 @@ and then runs tests against the running network. It is a git sub-repository in t
 present, you must initialize the subrepo: `git submodule update --init`.
 
 Steps to run
-1. Build a Kava docker image tagged `kava/kava:local`: `make build-docker`
+1. Ensure latest `kvtool` is installed: `make update-kvtool`
-2. Ensure latest `kvtool` is installed: `make update-kvtool`
+2. Run the test suite: `make test-e2e`
-3. Run the test suite: `make test-e2e`
+   This will build a docker image tagged `kava/kava:local` that will be run by kvtool.
 
 **Note:** The suite will use your locally installed `kvtool` if present. If not present, it will be
 installed. If the `kvtool` repo is updated, you must manually update your existing local binary: `make update-kvtool`
@@ -61,3 +61,24 @@ The IBC network runs kava with a different chain id and staking denom (see [runn
 The IBC chain queriers & accounts are accessible via `suite.Ibc`.
 
 IBC tests can be disabled by setting `E2E_INCLUDE_IBC_TESTS` to `false`.
+
+## Chain Upgrades
+
+When a named upgrade handler is included in the current working repo of Kava, the e2e test suite can
+be configured to run all the tests on the upgraded chain. This includes the ability to add additional
+tests to verify and do acceptance on the post-upgrade chain.
+
+This configuration is controlled by the following env variables:
+* `E2E_INCLUDE_AUTOMATED_UPGRADE` - toggles on the upgrade functionality. Must be set to `true`.
+* `E2E_KAVA_UPGRADE_NAME` - the named upgrade, likely defined in [`app/upgrades.go`](../../app/upgrades.go)
+* `E2E_KAVA_UPGRADE_HEIGHT` - the height at which to run the upgrade
+* `E2E_KAVA_UPGRADE_BASE_IMAGE_TAG` - the [kava docker image tag](https://hub.docker.com/r/kava/kava/tags) to base the upgrade on
+
+When all these are set, the chain is started with the binary contained in the docker image tagged
+`E2E_KAVA_UPGRADE_BASE_IMAGE_TAG`. Then an upgrade proposal is submitted with the desired name and
+height. The chain runs until that height and then is shutdown due to needing the upgrade. The chain
+is restarted with the local repo's Kava code and the upgrade is run. Once completed, the whole test
+suite is run.
+
+For a full example of how this looks, see [this commit](https://github.com/Kava-Labs/kava/commit/5da48c892f0a5837141fc7de88632c7c68fff4ae)
+on the [example/e2e-test-upgrade-handler](https://github.com/Kava-Labs/kava/tree/example/e2e-test-upgrade-handler) branch.

tests/e2e/runner/main.go

@@ -12,8 +12,17 @@ import (
 )
 
 type Config struct {
+	KavaConfigTemplate string
+
 	ImageTag   string
 	IncludeIBC bool
+
+	EnableAutomatedUpgrade  bool
+	KavaUpgradeName         string
+	KavaUpgradeHeight       int64
+	KavaUpgradeBaseImageTag string
+
+	SkipShutdown bool
 }
 
 // NodeRunner is responsible for starting and managing docker containers to run a node.
@@ -45,13 +54,23 @@ func (k *KavaNodeRunner) StartChains() Chains {
 	}
 
 	log.Println("starting kava node")
-	kvtoolArgs := []string{"testnet", "bootstrap"}
+	kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate}
 	if k.config.IncludeIBC {
 		kvtoolArgs = append(kvtoolArgs, "--ibc")
 	}
+	if k.config.EnableAutomatedUpgrade {
+		kvtoolArgs = append(kvtoolArgs,
+			"--upgrade-name", k.config.KavaUpgradeName,
+			"--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight),
+			"--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag,
+		)
+	}
 	startKavaCmd := exec.Command("kvtool", kvtoolArgs...)
+	startKavaCmd.Env = os.Environ()
+	startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag))
 	startKavaCmd.Stdout = os.Stdout
 	startKavaCmd.Stderr = os.Stderr
+	log.Println(startKavaCmd.String())
 	if err := startKavaCmd.Run(); err != nil {
 		panic(fmt.Sprintf("failed to start kava: %s", err.Error()))
 	}
@@ -74,6 +93,10 @@ func (k *KavaNodeRunner) StartChains() Chains {
 }
 
 func (k *KavaNodeRunner) Shutdown() {
+	if k.config.SkipShutdown {
+		log.Printf("would shut down but SkipShutdown is true")
+		return
+	}
 	log.Println("shutting down kava node")
 	shutdownKavaCmd := exec.Command("kvtool", "testnet", "down")
 	shutdownKavaCmd.Stdout = os.Stdout

tests/e2e/testutil/chain.go

@@ -18,6 +18,8 @@ import (
 	kavaparams "github.com/kava-labs/kava/app/params"
 	"github.com/kava-labs/kava/tests/e2e/runner"
 	"github.com/kava-labs/kava/tests/util"
+	communitytypes "github.com/kava-labs/kava/x/community/types"
+	earntypes "github.com/kava-labs/kava/x/earn/types"
 )
 
 // Chain wraps query clients & accounts for a network
@@ -30,8 +32,11 @@ type Chain struct {
 	ChainId      string
 
 	EvmClient *ethclient.Client
+
 	Auth      authtypes.QueryClient
 	Bank      banktypes.QueryClient
+	Community communitytypes.QueryClient
+	Earn      earntypes.QueryClient
 	Tm        tmservice.ServiceClient
 	Tx        txtypes.ServiceClient
 }
@@ -61,6 +66,8 @@ func NewChain(t *testing.T, details *runner.ChainDetails, fundedAccountMnemonic
 
 	chain.Auth = authtypes.NewQueryClient(grpcConn)
 	chain.Bank = banktypes.NewQueryClient(grpcConn)
+	chain.Community = communitytypes.NewQueryClient(grpcConn)
+	chain.Earn = earntypes.NewQueryClient(grpcConn)
 	chain.Tm = tmservice.NewServiceClient(grpcConn)
 	chain.Tx = txtypes.NewServiceClient(grpcConn)
 

tests/e2e/testutil/config.go

@@ -16,29 +16,51 @@ func init() {
 type SuiteConfig struct {
 	// A funded account used to fnd all other accounts.
 	FundedAccountMnemonic string
+	// The kava.configTemplate flag to be passed to kvtool, usually "master".
+	// This allows one to change the base genesis used to start the chain.
+	KavaConfigTemplate string
 	// Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests.
 	IncludeIbcTests bool
+
+	// Whether or not to run a chain upgrade & run post-upgrade tests. Use `suite.SkipIfUpgradeDisabled()` in post-upgrade tests.
+	IncludeAutomatedUpgrade bool
+	// Name of the upgrade, if upgrade is enabled.
+	KavaUpgradeName string
+	// Height upgrade will be applied to the test chain, if upgrade is enabled.
+	KavaUpgradeHeight int64
+	// Tag of kava docker image that will be upgraded to the current image before tests are run, if upgrade is enabled.
+	KavaUpgradeBaseImageTag string
+
 	// When true, the chains will remain running after tests complete (pass or fail)
 	SkipShutdown bool
 }
 
 func ParseSuiteConfig() SuiteConfig {
-	// this mnemonic is expected to be a funded account that can seed the funds for all
+	config := SuiteConfig{
-	// new accounts created during tests. it will be available under Accounts["whale"]
+		// this mnemonic is expected to be a funded account that can seed the funds for all
-	fundedAccountMnemonic := os.Getenv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC")
+		// new accounts created during tests. it will be available under Accounts["whale"]
-	if fundedAccountMnemonic == "" {
+		FundedAccountMnemonic:   nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"),
-		panic("no E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC provided")
+		KavaConfigTemplate:      nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"),
+		IncludeIbcTests:         mustParseBool("E2E_INCLUDE_IBC_TESTS"),
+		IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"),
 	}
-	var skipShutdown bool
+
 	skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN")
 	if skipShutdownEnv != "" {
-		skipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN")
+		config.SkipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN")
 	}
-	return SuiteConfig{
+
-		FundedAccountMnemonic: fundedAccountMnemonic,
+	if config.IncludeAutomatedUpgrade {
-		IncludeIbcTests:       mustParseBool("E2E_INCLUDE_IBC_TESTS"),
+		config.KavaUpgradeName = nonemptyStringEnv("E2E_KAVA_UPGRADE_NAME")
-		SkipShutdown:          skipShutdown,
+		config.KavaUpgradeBaseImageTag = nonemptyStringEnv("E2E_KAVA_UPGRADE_BASE_IMAGE_TAG")
+		upgradeHeight, err := strconv.ParseInt(nonemptyStringEnv("E2E_KAVA_UPGRADE_HEIGHT"), 10, 64)
+		if err != nil {
+			panic(fmt.Sprintf("E2E_KAVA_UPGRADE_HEIGHT must be a number: %s", err))
+		}
+		config.KavaUpgradeHeight = upgradeHeight
 	}
+
+	return config
 }
 
 func mustParseBool(name string) bool {
@@ -52,3 +74,11 @@ func mustParseBool(name string) bool {
 	}
 	return value
 }
+
+func nonemptyStringEnv(name string) string {
+	value := os.Getenv(name)
+	if value == "" {
+		panic(fmt.Sprintf("no %s env variable provided", name))
+	}
+	return value
+}

tests/e2e/testutil/suite.go

@@ -28,6 +28,8 @@ type E2eTestSuite struct {
 
 	Kava *Chain
 	Ibc  *Chain
+
+	UpgradeHeight int64
 }
 
 func (suite *E2eTestSuite) SetupSuite() {
@@ -37,10 +39,20 @@ func (suite *E2eTestSuite) SetupSuite() {
 
 	suiteConfig := ParseSuiteConfig()
 	suite.config = suiteConfig
+	suite.UpgradeHeight = suiteConfig.KavaUpgradeHeight
 
 	runnerConfig := runner.Config{
+		KavaConfigTemplate: suiteConfig.KavaConfigTemplate,
+
 		IncludeIBC: suiteConfig.IncludeIbcTests,
 		ImageTag:   "local",
+
+		EnableAutomatedUpgrade:  suiteConfig.IncludeAutomatedUpgrade,
+		KavaUpgradeName:         suiteConfig.KavaUpgradeName,
+		KavaUpgradeHeight:       suiteConfig.KavaUpgradeHeight,
+		KavaUpgradeBaseImageTag: suiteConfig.KavaUpgradeBaseImageTag,
+
+		SkipShutdown: suiteConfig.SkipShutdown,
 	}
 	suite.runner = runner.NewKavaNode(runnerConfig)
 
@@ -70,9 +82,7 @@ func (suite *E2eTestSuite) TearDownSuite() {
 		suite.Ibc.Shutdown()
 	}
 	// gracefully shutdown docker container(s)
-	if !suite.config.SkipShutdown {
+	suite.runner.Shutdown()
-		suite.runner.Shutdown()
-	}
 }
 
 func (suite *E2eTestSuite) SkipIfIbcDisabled() {
@@ -80,3 +90,9 @@ func (suite *E2eTestSuite) SkipIfIbcDisabled() {
 		suite.T().SkipNow()
 	}
 }
+
+func (suite *E2eTestSuite) SkipIfUpgradeDisabled() {
+	if !suite.config.IncludeAutomatedUpgrade {
+		suite.T().SkipNow()
+	}
+}

tests/util/grpc.go

@@ -1,12 +1,17 @@
 package util
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net/url"
+	"strconv"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/metadata"
+
+	grpctypes "github.com/cosmos/cosmos-sdk/types/grpc"
 )
 
 // NewGrpcConnection parses a GRPC endpoint and creates a connection to it
@@ -34,3 +39,8 @@ func NewGrpcConnection(endpoint string) (*grpc.ClientConn, error) {
 
 	return grpcConn, nil
 }
+
+func CtxAtHeight(height int64) context.Context {
+	heightStr := strconv.FormatInt(height, 10)
+	return metadata.AppendToOutgoingContext(context.Background(), grpctypes.GRPCBlockHeightHeader, heightStr)
+}