support chian upgrades in e2e tests (#1513)

* update kvtool

* add env vars for e2e automated upgrades

* run kvtool with upgrade flags when enabled

* add e2e test placeholder for upgrade handler tests

* allow override of kvtool's kava.configTemplate

* use the correct image tag when running e2e tests

* update kvtool

* always docker-build when running make test-e2e

* refactor skip shutdown to always skip shutdown

* add CtxAtHeight grpc helper

* add Community & Earn queriers

* expose UpgradeHeight in suite

* document e2e chain upgrades

* add reference to example branch
This commit is contained in:
Robert Pirtle 2023-03-28 15:32:36 -07:00 committed by GitHub
parent ecb0af6c68
commit 0156b0e645
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 154 additions and 20 deletions

View File

@ -290,7 +290,7 @@ test-basic: test
@go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 4 -v -timeout 2m @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 4 -v -timeout 2m
# run end-to-end tests (local docker container must be built, see docker-build) # run end-to-end tests (local docker container must be built, see docker-build)
test-e2e: test-e2e: docker-build
export E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'; \ export E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'; \
go test -failfast -count=1 -v ./tests/e2e/... go test -failfast -count=1 -v ./tests/e2e/...

View File

@ -1,9 +1,24 @@
# E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC is for a funded account used to intialize all new testing accounts. # E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC is for a funded account used to intialize all new testing accounts.
E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold' E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'
# E2E_KVTOOL_KAVA_CONFIG_TEMPLATE is the kvtool template used to start the chain. See the `kava.configTemplate` flag in kvtool.
# Note that the config tempalte must support overriding the docker image tag via the KAVA_TAG variable.
E2E_KVTOOL_KAVA_CONFIG_TEMPLATE="master"
# E2E_INCLUDE_IBC_TESTS when true will start a 2nd chain & open an IBC channel. It will enable all IBC tests. # E2E_INCLUDE_IBC_TESTS when true will start a 2nd chain & open an IBC channel. It will enable all IBC tests.
E2E_INCLUDE_IBC_TESTS=true E2E_INCLUDE_IBC_TESTS=true
# E2E_SKIP_SHUTDOWN when true will keep the networks running after tests complete (pass or fail) # E2E_SKIP_SHUTDOWN when true will keep the networks running after tests complete (pass or fail)
# This is useful for debugging chain state when writing tests. # This is useful for debugging chain state when writing tests.
E2E_SKIP_SHUTDOWN=false E2E_SKIP_SHUTDOWN=false
# The following variables should be defined to run an upgrade.
# E2E_INCLUDE_AUTOMATED_UPGRADE when true enables the automated upgrade & corresponding tests in the suite.
E2E_INCLUDE_AUTOMATED_UPGRADE=false
# E2E_KAVA_UPGRADE_NAME is the name of the upgrade that must be in the current local image.
E2E_KAVA_UPGRADE_NAME=
# E2E_KAVA_UPGRADE_HEIGHT is the height at which the upgrade will be applied.
# If IBC tests are enabled this should be >50. Otherwise, this should be >10.
E2E_KAVA_UPGRADE_HEIGHT=
# E2E_KAVA_UPGRADE_BASE_IMAGE_TAG is the tag of the docker image the chain should upgrade from.
E2E_KAVA_UPGRADE_BASE_IMAGE_TAG=

View File

@ -0,0 +1,12 @@
package e2e_test
import "fmt"
// TestUpgradeHandler can be used to run tests post-upgrade. If an upgrade is enabled, all tests
// are run against the upgraded chain. However, this file is a good place to consolidate all
// acceptance tests for a given set of upgrade handlers.
func (suite IntegrationTestSuite) TestUpgradeHandler() {
suite.SkipIfUpgradeDisabled()
fmt.Println("An upgrade has run!")
suite.True(true)
}

@ -1 +1 @@
Subproject commit c80093f2b9133c30e9e24da32a74a870d6e33b57 Subproject commit 8adc0437e849f86bdcb2df3833abbfbd0980775e

View File

@ -5,9 +5,9 @@ and then runs tests against the running network. It is a git sub-repository in t
present, you must initialize the subrepo: `git submodule update --init`. present, you must initialize the subrepo: `git submodule update --init`.
Steps to run Steps to run
1. Build a Kava docker image tagged `kava/kava:local`: `make build-docker` 1. Ensure latest `kvtool` is installed: `make update-kvtool`
2. Ensure latest `kvtool` is installed: `make update-kvtool` 2. Run the test suite: `make test-e2e`
3. Run the test suite: `make test-e2e` This will build a docker image tagged `kava/kava:local` that will be run by kvtool.
**Note:** The suite will use your locally installed `kvtool` if present. If not present, it will be **Note:** The suite will use your locally installed `kvtool` if present. If not present, it will be
installed. If the `kvtool` repo is updated, you must manually update your existing local binary: `make update-kvtool` installed. If the `kvtool` repo is updated, you must manually update your existing local binary: `make update-kvtool`
@ -61,3 +61,24 @@ The IBC network runs kava with a different chain id and staking denom (see [runn
The IBC chain queriers & accounts are accessible via `suite.Ibc`. The IBC chain queriers & accounts are accessible via `suite.Ibc`.
IBC tests can be disabled by setting `E2E_INCLUDE_IBC_TESTS` to `false`. IBC tests can be disabled by setting `E2E_INCLUDE_IBC_TESTS` to `false`.
## Chain Upgrades
When a named upgrade handler is included in the current working repo of Kava, the e2e test suite can
be configured to run all the tests on the upgraded chain. This includes the ability to add additional
tests to verify and do acceptance on the post-upgrade chain.
This configuration is controlled by the following env variables:
* `E2E_INCLUDE_AUTOMATED_UPGRADE` - toggles on the upgrade functionality. Must be set to `true`.
* `E2E_KAVA_UPGRADE_NAME` - the named upgrade, likely defined in [`app/upgrades.go`](../../app/upgrades.go)
* `E2E_KAVA_UPGRADE_HEIGHT` - the height at which to run the upgrade
* `E2E_KAVA_UPGRADE_BASE_IMAGE_TAG` - the [kava docker image tag](https://hub.docker.com/r/kava/kava/tags) to base the upgrade on
When all these are set, the chain is started with the binary contained in the docker image tagged
`E2E_KAVA_UPGRADE_BASE_IMAGE_TAG`. Then an upgrade proposal is submitted with the desired name and
height. The chain runs until that height and then is shutdown due to needing the upgrade. The chain
is restarted with the local repo's Kava code and the upgrade is run. Once completed, the whole test
suite is run.
For a full example of how this looks, see [this commit](https://github.com/Kava-Labs/kava/commit/5da48c892f0a5837141fc7de88632c7c68fff4ae)
on the [example/e2e-test-upgrade-handler](https://github.com/Kava-Labs/kava/tree/example/e2e-test-upgrade-handler) branch.

View File

@ -12,8 +12,17 @@ import (
) )
type Config struct { type Config struct {
KavaConfigTemplate string
ImageTag string ImageTag string
IncludeIBC bool IncludeIBC bool
EnableAutomatedUpgrade bool
KavaUpgradeName string
KavaUpgradeHeight int64
KavaUpgradeBaseImageTag string
SkipShutdown bool
} }
// NodeRunner is responsible for starting and managing docker containers to run a node. // NodeRunner is responsible for starting and managing docker containers to run a node.
@ -45,13 +54,23 @@ func (k *KavaNodeRunner) StartChains() Chains {
} }
log.Println("starting kava node") log.Println("starting kava node")
kvtoolArgs := []string{"testnet", "bootstrap"} kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate}
if k.config.IncludeIBC { if k.config.IncludeIBC {
kvtoolArgs = append(kvtoolArgs, "--ibc") kvtoolArgs = append(kvtoolArgs, "--ibc")
} }
if k.config.EnableAutomatedUpgrade {
kvtoolArgs = append(kvtoolArgs,
"--upgrade-name", k.config.KavaUpgradeName,
"--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight),
"--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag,
)
}
startKavaCmd := exec.Command("kvtool", kvtoolArgs...) startKavaCmd := exec.Command("kvtool", kvtoolArgs...)
startKavaCmd.Env = os.Environ()
startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag))
startKavaCmd.Stdout = os.Stdout startKavaCmd.Stdout = os.Stdout
startKavaCmd.Stderr = os.Stderr startKavaCmd.Stderr = os.Stderr
log.Println(startKavaCmd.String())
if err := startKavaCmd.Run(); err != nil { if err := startKavaCmd.Run(); err != nil {
panic(fmt.Sprintf("failed to start kava: %s", err.Error())) panic(fmt.Sprintf("failed to start kava: %s", err.Error()))
} }
@ -74,6 +93,10 @@ func (k *KavaNodeRunner) StartChains() Chains {
} }
func (k *KavaNodeRunner) Shutdown() { func (k *KavaNodeRunner) Shutdown() {
if k.config.SkipShutdown {
log.Printf("would shut down but SkipShutdown is true")
return
}
log.Println("shutting down kava node") log.Println("shutting down kava node")
shutdownKavaCmd := exec.Command("kvtool", "testnet", "down") shutdownKavaCmd := exec.Command("kvtool", "testnet", "down")
shutdownKavaCmd.Stdout = os.Stdout shutdownKavaCmd.Stdout = os.Stdout

View File

@ -18,6 +18,8 @@ import (
kavaparams "github.com/kava-labs/kava/app/params" kavaparams "github.com/kava-labs/kava/app/params"
"github.com/kava-labs/kava/tests/e2e/runner" "github.com/kava-labs/kava/tests/e2e/runner"
"github.com/kava-labs/kava/tests/util" "github.com/kava-labs/kava/tests/util"
communitytypes "github.com/kava-labs/kava/x/community/types"
earntypes "github.com/kava-labs/kava/x/earn/types"
) )
// Chain wraps query clients & accounts for a network // Chain wraps query clients & accounts for a network
@ -30,8 +32,11 @@ type Chain struct {
ChainId string ChainId string
EvmClient *ethclient.Client EvmClient *ethclient.Client
Auth authtypes.QueryClient Auth authtypes.QueryClient
Bank banktypes.QueryClient Bank banktypes.QueryClient
Community communitytypes.QueryClient
Earn earntypes.QueryClient
Tm tmservice.ServiceClient Tm tmservice.ServiceClient
Tx txtypes.ServiceClient Tx txtypes.ServiceClient
} }
@ -61,6 +66,8 @@ func NewChain(t *testing.T, details *runner.ChainDetails, fundedAccountMnemonic
chain.Auth = authtypes.NewQueryClient(grpcConn) chain.Auth = authtypes.NewQueryClient(grpcConn)
chain.Bank = banktypes.NewQueryClient(grpcConn) chain.Bank = banktypes.NewQueryClient(grpcConn)
chain.Community = communitytypes.NewQueryClient(grpcConn)
chain.Earn = earntypes.NewQueryClient(grpcConn)
chain.Tm = tmservice.NewServiceClient(grpcConn) chain.Tm = tmservice.NewServiceClient(grpcConn)
chain.Tx = txtypes.NewServiceClient(grpcConn) chain.Tx = txtypes.NewServiceClient(grpcConn)

View File

@ -16,29 +16,51 @@ func init() {
type SuiteConfig struct { type SuiteConfig struct {
// A funded account used to fnd all other accounts. // A funded account used to fnd all other accounts.
FundedAccountMnemonic string FundedAccountMnemonic string
// The kava.configTemplate flag to be passed to kvtool, usually "master".
// This allows one to change the base genesis used to start the chain.
KavaConfigTemplate string
// Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests. // Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests.
IncludeIbcTests bool IncludeIbcTests bool
// Whether or not to run a chain upgrade & run post-upgrade tests. Use `suite.SkipIfUpgradeDisabled()` in post-upgrade tests.
IncludeAutomatedUpgrade bool
// Name of the upgrade, if upgrade is enabled.
KavaUpgradeName string
// Height upgrade will be applied to the test chain, if upgrade is enabled.
KavaUpgradeHeight int64
// Tag of kava docker image that will be upgraded to the current image before tests are run, if upgrade is enabled.
KavaUpgradeBaseImageTag string
// When true, the chains will remain running after tests complete (pass or fail) // When true, the chains will remain running after tests complete (pass or fail)
SkipShutdown bool SkipShutdown bool
} }
func ParseSuiteConfig() SuiteConfig { func ParseSuiteConfig() SuiteConfig {
// this mnemonic is expected to be a funded account that can seed the funds for all config := SuiteConfig{
// new accounts created during tests. it will be available under Accounts["whale"] // this mnemonic is expected to be a funded account that can seed the funds for all
fundedAccountMnemonic := os.Getenv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC") // new accounts created during tests. it will be available under Accounts["whale"]
if fundedAccountMnemonic == "" { FundedAccountMnemonic: nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"),
panic("no E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC provided") KavaConfigTemplate: nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"),
IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"),
IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"),
} }
var skipShutdown bool
skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN") skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN")
if skipShutdownEnv != "" { if skipShutdownEnv != "" {
skipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN") config.SkipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN")
} }
return SuiteConfig{
FundedAccountMnemonic: fundedAccountMnemonic, if config.IncludeAutomatedUpgrade {
IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"), config.KavaUpgradeName = nonemptyStringEnv("E2E_KAVA_UPGRADE_NAME")
SkipShutdown: skipShutdown, config.KavaUpgradeBaseImageTag = nonemptyStringEnv("E2E_KAVA_UPGRADE_BASE_IMAGE_TAG")
upgradeHeight, err := strconv.ParseInt(nonemptyStringEnv("E2E_KAVA_UPGRADE_HEIGHT"), 10, 64)
if err != nil {
panic(fmt.Sprintf("E2E_KAVA_UPGRADE_HEIGHT must be a number: %s", err))
}
config.KavaUpgradeHeight = upgradeHeight
} }
return config
} }
func mustParseBool(name string) bool { func mustParseBool(name string) bool {
@ -52,3 +74,11 @@ func mustParseBool(name string) bool {
} }
return value return value
} }
func nonemptyStringEnv(name string) string {
value := os.Getenv(name)
if value == "" {
panic(fmt.Sprintf("no %s env variable provided", name))
}
return value
}

View File

@ -28,6 +28,8 @@ type E2eTestSuite struct {
Kava *Chain Kava *Chain
Ibc *Chain Ibc *Chain
UpgradeHeight int64
} }
func (suite *E2eTestSuite) SetupSuite() { func (suite *E2eTestSuite) SetupSuite() {
@ -37,10 +39,20 @@ func (suite *E2eTestSuite) SetupSuite() {
suiteConfig := ParseSuiteConfig() suiteConfig := ParseSuiteConfig()
suite.config = suiteConfig suite.config = suiteConfig
suite.UpgradeHeight = suiteConfig.KavaUpgradeHeight
runnerConfig := runner.Config{ runnerConfig := runner.Config{
KavaConfigTemplate: suiteConfig.KavaConfigTemplate,
IncludeIBC: suiteConfig.IncludeIbcTests, IncludeIBC: suiteConfig.IncludeIbcTests,
ImageTag: "local", ImageTag: "local",
EnableAutomatedUpgrade: suiteConfig.IncludeAutomatedUpgrade,
KavaUpgradeName: suiteConfig.KavaUpgradeName,
KavaUpgradeHeight: suiteConfig.KavaUpgradeHeight,
KavaUpgradeBaseImageTag: suiteConfig.KavaUpgradeBaseImageTag,
SkipShutdown: suiteConfig.SkipShutdown,
} }
suite.runner = runner.NewKavaNode(runnerConfig) suite.runner = runner.NewKavaNode(runnerConfig)
@ -70,9 +82,7 @@ func (suite *E2eTestSuite) TearDownSuite() {
suite.Ibc.Shutdown() suite.Ibc.Shutdown()
} }
// gracefully shutdown docker container(s) // gracefully shutdown docker container(s)
if !suite.config.SkipShutdown { suite.runner.Shutdown()
suite.runner.Shutdown()
}
} }
func (suite *E2eTestSuite) SkipIfIbcDisabled() { func (suite *E2eTestSuite) SkipIfIbcDisabled() {
@ -80,3 +90,9 @@ func (suite *E2eTestSuite) SkipIfIbcDisabled() {
suite.T().SkipNow() suite.T().SkipNow()
} }
} }
func (suite *E2eTestSuite) SkipIfUpgradeDisabled() {
if !suite.config.IncludeAutomatedUpgrade {
suite.T().SkipNow()
}
}

View File

@ -1,12 +1,17 @@
package util package util
import ( import (
"context"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
"net/url" "net/url"
"strconv"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
grpctypes "github.com/cosmos/cosmos-sdk/types/grpc"
) )
// NewGrpcConnection parses a GRPC endpoint and creates a connection to it // NewGrpcConnection parses a GRPC endpoint and creates a connection to it
@ -34,3 +39,8 @@ func NewGrpcConnection(endpoint string) (*grpc.ClientConn, error) {
return grpcConn, nil return grpcConn, nil
} }
func CtxAtHeight(height int64) context.Context {
heightStr := strconv.FormatInt(height, 10)
return metadata.AppendToOutgoingContext(context.Background(), grpctypes.GRPCBlockHeightHeader, heightStr)
}