From 1a223bdce2602734eb9ee744f19b3ba938ec01ea Mon Sep 17 00:00:00 2001 From: Robert Pirtle Date: Tue, 20 Jun 2023 09:29:25 -0700 Subject: [PATCH] test(e2e): refactor NodeRunner in prep for live networks (#1627) * add KavaNodeRunner comments * update kvtool * refactor pingEvm() * refactor pingKava() * refactor EvmRpcPort -> EvmRpcUrl * refactor ChainDetails (g)rpcPort -> (g)rpcUrl * fixup chain details * extract kvtool options to separate config * refactor waitForChainStart() * pull out KavaNodeRunner into kvtool file * rename runner to KvtoolRunner * rename runner.Config to KvtoolRunnerConfig * prefix hardcoded chain details with "kvtool" --- tests/e2e/.env | 4 ++ tests/e2e/kvtool | 2 +- tests/e2e/runner/chain.go | 36 +++++------- tests/e2e/runner/kvtool.go | 104 +++++++++++++++++++++++++++++++++ tests/e2e/runner/main.go | 109 +++-------------------------------- tests/e2e/testutil/chain.go | 6 +- tests/e2e/testutil/config.go | 46 ++++++++++----- tests/e2e/testutil/suite.go | 27 +++++---- 8 files changed, 183 insertions(+), 151 deletions(-) create mode 100644 tests/e2e/runner/kvtool.go diff --git a/tests/e2e/.env b/tests/e2e/.env index 8c6d63bc..f3ddfb59 100644 --- a/tests/e2e/.env +++ b/tests/e2e/.env @@ -2,6 +2,10 @@ # Should be funded with KAVA and have an ERC20 balance E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold' +# E2E_RUN_KVTOOL_NETWORKS is the only currently supported option. +# It triggers tests to be run against local networks spun up with kvtool. +E2E_RUN_KVTOOL_NETWORKS=true + # E2E_KVTOOL_KAVA_CONFIG_TEMPLATE is the kvtool template used to start the chain. See the `kava.configTemplate` flag in kvtool. # Note that the config tempalte must support overriding the docker image tag via the KAVA_TAG variable. E2E_KVTOOL_KAVA_CONFIG_TEMPLATE="master" diff --git a/tests/e2e/kvtool b/tests/e2e/kvtool index 40f6311b..68f3a5ce 160000 --- a/tests/e2e/kvtool +++ b/tests/e2e/kvtool @@ -1 +1 @@ -Subproject commit 40f6311b6e92aa5b716d029da6047c2f3f22c882 +Subproject commit 68f3a5ce9e688028ab66cecdb583223f03e153fc diff --git a/tests/e2e/runner/chain.go b/tests/e2e/runner/chain.go index 7dcfd66f..f4a3efa1 100644 --- a/tests/e2e/runner/chain.go +++ b/tests/e2e/runner/chain.go @@ -14,25 +14,22 @@ var ( ErrChainAlreadyExists = errors.New("chain already exists") ) -// ChainDetails wraps information about the ports exposed to the host that endpoints could be access on. +// ChainDetails wraps information about the properties & endpoints of a chain. type ChainDetails struct { - RpcPort string - GrpcPort string - RestPort string - EvmPort string + RpcUrl string + GrpcUrl string + EvmRpcUrl string ChainId string StakingDenom string } func (c ChainDetails) EvmClient() (*ethclient.Client, error) { - evmRpcUrl := fmt.Sprintf("http://localhost:%s", c.EvmPort) - return ethclient.Dial(evmRpcUrl) + return ethclient.Dial(c.EvmRpcUrl) } func (c ChainDetails) GrpcConn() (*grpc.ClientConn, error) { - grpcUrl := fmt.Sprintf("http://localhost:%s", c.GrpcPort) - return util.NewGrpcConnection(grpcUrl) + return util.NewGrpcConnection(c.GrpcUrl) } type Chains struct { @@ -60,23 +57,20 @@ func (c *Chains) Register(name string, chain *ChainDetails) error { } // the Chain details are all hardcoded because they are currently fixed by kvtool. -// some day they may be configurable, at which point `runner` can determine the ports -// and generate these details dynamically +// someday they may be accepted as configurable parameters. var ( - kavaChain = ChainDetails{ - RpcPort: "26657", - RestPort: "1317", - GrpcPort: "9090", - EvmPort: "8545", + kvtoolKavaChain = ChainDetails{ + RpcUrl: "http://localhost:26657", + GrpcUrl: "http://localhost:9090", + EvmRpcUrl: "http://localhost:8545", ChainId: "kavalocalnet_8888-1", StakingDenom: "ukava", } - ibcChain = ChainDetails{ - RpcPort: "26658", - RestPort: "1318", - GrpcPort: "9092", - EvmPort: "8547", + kvtoolIbcChain = ChainDetails{ + RpcUrl: "http://localhost:26658", + GrpcUrl: "http://localhost:9092", + EvmRpcUrl: "http://localhost:8547", ChainId: "kavalocalnet_8889-2", StakingDenom: "uatom", diff --git a/tests/e2e/runner/kvtool.go b/tests/e2e/runner/kvtool.go new file mode 100644 index 00000000..93e5ee21 --- /dev/null +++ b/tests/e2e/runner/kvtool.go @@ -0,0 +1,104 @@ +package runner + +import ( + "fmt" + "log" + "os" + "os/exec" +) + +type KvtoolRunnerConfig struct { + KavaConfigTemplate string + + ImageTag string + IncludeIBC bool + + EnableAutomatedUpgrade bool + KavaUpgradeName string + KavaUpgradeHeight int64 + KavaUpgradeBaseImageTag string + + SkipShutdown bool +} + +// KvtoolRunner implements a NodeRunner that spins up local chains with kvtool. +// It has support for the following: +// - running a Kava node +// - optionally, running an IBC node with a channel opened to the Kava node +// - optionally, start the Kava node on one version and upgrade to another +type KvtoolRunner struct { + config KvtoolRunnerConfig +} + +var _ NodeRunner = &KvtoolRunner{} + +func NewKvtoolRunner(config KvtoolRunnerConfig) *KvtoolRunner { + return &KvtoolRunner{ + config: config, + } +} + +func (k *KvtoolRunner) StartChains() Chains { + // install kvtool if not already installed + installKvtoolCmd := exec.Command("./scripts/install-kvtool.sh") + installKvtoolCmd.Stdout = os.Stdout + installKvtoolCmd.Stderr = os.Stderr + if err := installKvtoolCmd.Run(); err != nil { + panic(fmt.Sprintf("failed to install kvtool: %s", err.Error())) + } + + // start local test network with kvtool + log.Println("starting kava node") + kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate} + // include an ibc chain if desired + if k.config.IncludeIBC { + kvtoolArgs = append(kvtoolArgs, "--ibc") + } + // handle automated upgrade functionality, if defined + if k.config.EnableAutomatedUpgrade { + kvtoolArgs = append(kvtoolArgs, + "--upgrade-name", k.config.KavaUpgradeName, + "--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight), + "--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag, + ) + } + // start the chain + startKavaCmd := exec.Command("kvtool", kvtoolArgs...) + startKavaCmd.Env = os.Environ() + startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag)) + startKavaCmd.Stdout = os.Stdout + startKavaCmd.Stderr = os.Stderr + log.Println(startKavaCmd.String()) + if err := startKavaCmd.Run(); err != nil { + panic(fmt.Sprintf("failed to start kava: %s", err.Error())) + } + + // wait for chain to be live. + // if an upgrade is defined, this waits for the upgrade to be completed. + if err := waitForChainStart(kvtoolKavaChain); err != nil { + k.Shutdown() + panic(err) + } + log.Println("kava is started!") + + chains := NewChains() + chains.Register("kava", &kvtoolKavaChain) + if k.config.IncludeIBC { + chains.Register("ibc", &kvtoolIbcChain) + } + return chains +} + +func (k *KvtoolRunner) Shutdown() { + if k.config.SkipShutdown { + log.Printf("would shut down but SkipShutdown is true") + return + } + log.Println("shutting down kava node") + shutdownKavaCmd := exec.Command("kvtool", "testnet", "down") + shutdownKavaCmd.Stdout = os.Stdout + shutdownKavaCmd.Stderr = os.Stderr + if err := shutdownKavaCmd.Run(); err != nil { + panic(fmt.Sprintf("failed to shutdown kvtool: %s", err.Error())) + } +} diff --git a/tests/e2e/runner/main.go b/tests/e2e/runner/main.go index 825b255d..b2ff8677 100644 --- a/tests/e2e/runner/main.go +++ b/tests/e2e/runner/main.go @@ -4,128 +4,38 @@ import ( "fmt" "log" "net/http" - "os" - "os/exec" "time" "github.com/cenkalti/backoff/v4" ) -type Config struct { - KavaConfigTemplate string - - ImageTag string - IncludeIBC bool - - EnableAutomatedUpgrade bool - KavaUpgradeName string - KavaUpgradeHeight int64 - KavaUpgradeBaseImageTag string - - SkipShutdown bool -} - // NodeRunner is responsible for starting and managing docker containers to run a node. type NodeRunner interface { StartChains() Chains Shutdown() } -// KavaNodeRunner manages and runs a single Kava node. -type KavaNodeRunner struct { - config Config - kavaChain *ChainDetails -} - -var _ NodeRunner = &KavaNodeRunner{} - -func NewKavaNode(config Config) *KavaNodeRunner { - return &KavaNodeRunner{ - config: config, - } -} - -func (k *KavaNodeRunner) StartChains() Chains { - installKvtoolCmd := exec.Command("./scripts/install-kvtool.sh") - installKvtoolCmd.Stdout = os.Stdout - installKvtoolCmd.Stderr = os.Stderr - if err := installKvtoolCmd.Run(); err != nil { - panic(fmt.Sprintf("failed to install kvtool: %s", err.Error())) - } - - log.Println("starting kava node") - kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate} - if k.config.IncludeIBC { - kvtoolArgs = append(kvtoolArgs, "--ibc") - } - if k.config.EnableAutomatedUpgrade { - kvtoolArgs = append(kvtoolArgs, - "--upgrade-name", k.config.KavaUpgradeName, - "--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight), - "--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag, - ) - } - startKavaCmd := exec.Command("kvtool", kvtoolArgs...) - startKavaCmd.Env = os.Environ() - startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag)) - startKavaCmd.Stdout = os.Stdout - startKavaCmd.Stderr = os.Stderr - log.Println(startKavaCmd.String()) - if err := startKavaCmd.Run(); err != nil { - panic(fmt.Sprintf("failed to start kava: %s", err.Error())) - } - - k.kavaChain = &kavaChain - - err := k.waitForChainStart() - if err != nil { - k.Shutdown() - panic(err) - } - log.Println("kava is started!") - - chains := NewChains() - chains.Register("kava", k.kavaChain) - if k.config.IncludeIBC { - chains.Register("ibc", &ibcChain) - } - return chains -} - -func (k *KavaNodeRunner) Shutdown() { - if k.config.SkipShutdown { - log.Printf("would shut down but SkipShutdown is true") - return - } - log.Println("shutting down kava node") - shutdownKavaCmd := exec.Command("kvtool", "testnet", "down") - shutdownKavaCmd.Stdout = os.Stdout - shutdownKavaCmd.Stderr = os.Stderr - if err := shutdownKavaCmd.Run(); err != nil { - panic(fmt.Sprintf("failed to shutdown kvtool: %s", err.Error())) - } -} - -func (k *KavaNodeRunner) waitForChainStart() error { +func waitForChainStart(chainDetails ChainDetails) error { // exponential backoff on trying to ping the node, timeout after 30 seconds b := backoff.NewExponentialBackOff() b.MaxInterval = 5 * time.Second b.MaxElapsedTime = 30 * time.Second - if err := backoff.Retry(k.pingKava, b); err != nil { + if err := backoff.Retry(func() error { return pingKava(chainDetails.RpcUrl) }, b); err != nil { return fmt.Errorf("failed to start & connect to chain: %s", err) } + b.Reset() // the evm takes a bit longer to start up. wait for it to start as well. - if err := backoff.Retry(k.pingEvm, b); err != nil { + if err := backoff.Retry(func() error { return pingEvm(chainDetails.EvmRpcUrl) }, b); err != nil { return fmt.Errorf("failed to start & connect to chain: %s", err) } return nil } -func (k *KavaNodeRunner) pingKava() error { +func pingKava(rpcUrl string) error { log.Println("pinging kava chain...") - url := fmt.Sprintf("http://localhost:%s/status", k.kavaChain.RpcPort) - res, err := http.Get(url) + statusUrl := fmt.Sprintf("%s/status", rpcUrl) + res, err := http.Get(statusUrl) if err != nil { return err } @@ -137,10 +47,9 @@ func (k *KavaNodeRunner) pingKava() error { return nil } -func (k *KavaNodeRunner) pingEvm() error { +func pingEvm(evmRpcUrl string) error { log.Println("pinging evm...") - url := fmt.Sprintf("http://localhost:%s", k.kavaChain.EvmPort) - res, err := http.Get(url) + res, err := http.Get(evmRpcUrl) if err != nil { return err } diff --git a/tests/e2e/testutil/chain.go b/tests/e2e/testutil/chain.go index 5444c7e4..cf3b3f8d 100644 --- a/tests/e2e/testutil/chain.go +++ b/tests/e2e/testutil/chain.go @@ -69,14 +69,12 @@ func NewChain(t *testing.T, details *runner.ChainDetails, fundedAccountMnemonic } chain.EncodingConfig = app.MakeEncodingConfig() - grpcUrl := fmt.Sprintf("http://localhost:%s", details.GrpcPort) - grpcConn, err := util.NewGrpcConnection(grpcUrl) + grpcConn, err := details.GrpcConn() if err != nil { return chain, err } - evmRpcUrl := fmt.Sprintf("http://localhost:%s", details.EvmPort) - chain.EvmClient, err = ethclient.Dial(evmRpcUrl) + chain.EvmClient, err = details.EvmClient() if err != nil { return chain, err } diff --git a/tests/e2e/testutil/config.go b/tests/e2e/testutil/config.go index ce7a82f7..10befaeb 100644 --- a/tests/e2e/testutil/config.go +++ b/tests/e2e/testutil/config.go @@ -16,11 +16,24 @@ func init() { type SuiteConfig struct { // A funded account used to fnd all other accounts. FundedAccountMnemonic string + + // A config for using kvtool local networks for the test run + Kvtool *KvtoolConfig + + // Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests. + IncludeIbcTests bool + + // The contract address of a deployed ERC-20 token + KavaErc20Address string + + // When true, the chains will remain running after tests complete (pass or fail) + SkipShutdown bool +} + +type KvtoolConfig struct { // The kava.configTemplate flag to be passed to kvtool, usually "master". // This allows one to change the base genesis used to start the chain. KavaConfigTemplate string - // Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests. - IncludeIbcTests bool // Whether or not to run a chain upgrade & run post-upgrade tests. Use `suite.SkipIfUpgradeDisabled()` in post-upgrade tests. IncludeAutomatedUpgrade bool @@ -30,23 +43,15 @@ type SuiteConfig struct { KavaUpgradeHeight int64 // Tag of kava docker image that will be upgraded to the current image before tests are run, if upgrade is enabled. KavaUpgradeBaseImageTag string - - // The contract address of a deployed ERC-20 token - KavaErc20Address string - - // When true, the chains will remain running after tests complete (pass or fail) - SkipShutdown bool } func ParseSuiteConfig() SuiteConfig { config := SuiteConfig{ // this mnemonic is expected to be a funded account that can seed the funds for all // new accounts created during tests. it will be available under Accounts["whale"] - FundedAccountMnemonic: nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"), - KavaConfigTemplate: nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"), - KavaErc20Address: nonemptyStringEnv("E2E_KAVA_ERC20_ADDRESS"), - IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"), - IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"), + FundedAccountMnemonic: nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"), + KavaErc20Address: nonemptyStringEnv("E2E_KAVA_ERC20_ADDRESS"), + IncludeIbcTests: mustParseBool("E2E_INCLUDE_IBC_TESTS"), } skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN") @@ -54,6 +59,21 @@ func ParseSuiteConfig() SuiteConfig { config.SkipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN") } + useKvtoolNetworks := mustParseBool("E2E_RUN_KVTOOL_NETWORKS") + if useKvtoolNetworks { + kvtoolConfig := ParseKvtoolConfig() + config.Kvtool = &kvtoolConfig + } + + return config +} + +func ParseKvtoolConfig() KvtoolConfig { + config := KvtoolConfig{ + KavaConfigTemplate: nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"), + IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"), + } + if config.IncludeAutomatedUpgrade { config.KavaUpgradeName = nonemptyStringEnv("E2E_KAVA_UPGRADE_NAME") config.KavaUpgradeBaseImageTag = nonemptyStringEnv("E2E_KAVA_UPGRADE_BASE_IMAGE_TAG") diff --git a/tests/e2e/testutil/suite.go b/tests/e2e/testutil/suite.go index a58cff61..e267cd0d 100644 --- a/tests/e2e/testutil/suite.go +++ b/tests/e2e/testutil/suite.go @@ -43,23 +43,26 @@ func (suite *E2eTestSuite) SetupSuite() { suiteConfig := ParseSuiteConfig() suite.config = suiteConfig - suite.UpgradeHeight = suiteConfig.KavaUpgradeHeight suite.DeployedErc20Address = common.HexToAddress(suiteConfig.KavaErc20Address) - runnerConfig := runner.Config{ - KavaConfigTemplate: suiteConfig.KavaConfigTemplate, + if suiteConfig.Kvtool != nil { + suite.UpgradeHeight = suiteConfig.Kvtool.KavaUpgradeHeight - IncludeIBC: suiteConfig.IncludeIbcTests, - ImageTag: "local", + runnerConfig := runner.KvtoolRunnerConfig{ + KavaConfigTemplate: suiteConfig.Kvtool.KavaConfigTemplate, - EnableAutomatedUpgrade: suiteConfig.IncludeAutomatedUpgrade, - KavaUpgradeName: suiteConfig.KavaUpgradeName, - KavaUpgradeHeight: suiteConfig.KavaUpgradeHeight, - KavaUpgradeBaseImageTag: suiteConfig.KavaUpgradeBaseImageTag, + IncludeIBC: suiteConfig.IncludeIbcTests, + ImageTag: "local", - SkipShutdown: suiteConfig.SkipShutdown, + EnableAutomatedUpgrade: suiteConfig.Kvtool.IncludeAutomatedUpgrade, + KavaUpgradeName: suiteConfig.Kvtool.KavaUpgradeName, + KavaUpgradeHeight: suiteConfig.Kvtool.KavaUpgradeHeight, + KavaUpgradeBaseImageTag: suiteConfig.Kvtool.KavaUpgradeBaseImageTag, + + SkipShutdown: suiteConfig.SkipShutdown, + } + suite.runner = runner.NewKvtoolRunner(runnerConfig) } - suite.runner = runner.NewKavaNode(runnerConfig) chains := suite.runner.StartChains() kavachain := chains.MustGetChain("kava") @@ -99,7 +102,7 @@ func (suite *E2eTestSuite) SkipIfIbcDisabled() { } func (suite *E2eTestSuite) SkipIfUpgradeDisabled() { - if !suite.config.IncludeAutomatedUpgrade { + if suite.config.Kvtool != nil && suite.config.Kvtool.IncludeAutomatedUpgrade { suite.T().SkipNow() } }