test(e2e): refactor NodeRunner in prep for live networks (#1627)

* add KavaNodeRunner comments

* update kvtool

* refactor pingEvm()

* refactor pingKava()

* refactor EvmRpcPort -> EvmRpcUrl

* refactor ChainDetails (g)rpcPort -> (g)rpcUrl

* fixup chain details

* extract kvtool options to separate config

* refactor waitForChainStart()

* pull out KavaNodeRunner into kvtool file

* rename runner to KvtoolRunner

* rename runner.Config to KvtoolRunnerConfig

* prefix hardcoded chain details with "kvtool"

tests/e2e/.env

@@ -2,6 +2,10 @@
 # Should be funded with KAVA and have an ERC20 balance
 E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC='tent fitness boat among census primary pipe nose dream glance cave turtle electric fabric jacket shaft easy myself genuine this sibling pulse word unfold'
 
+# E2E_RUN_KVTOOL_NETWORKS is the only currently supported option.
+# It triggers tests to be run against local networks spun up with kvtool.
+E2E_RUN_KVTOOL_NETWORKS=true
+
 # E2E_KVTOOL_KAVA_CONFIG_TEMPLATE is the kvtool template used to start the chain. See the `kava.configTemplate` flag in kvtool.
 # Note that the config tempalte must support overriding the docker image tag via the KAVA_TAG variable.
 E2E_KVTOOL_KAVA_CONFIG_TEMPLATE="master"

tests/e2e/kvtool

@@ -1 +1 @@
-Subproject commit 40f6311b6e92aa5b716d029da6047c2f3f22c882
+Subproject commit 68f3a5ce9e688028ab66cecdb583223f03e153fc

tests/e2e/runner/chain.go

@@ -14,25 +14,22 @@ var (
 	ErrChainAlreadyExists = errors.New("chain already exists")
 )
 
-// ChainDetails wraps information about the ports exposed to the host that endpoints could be access on.
+// ChainDetails wraps information about the properties & endpoints of a chain.
 type ChainDetails struct {
-	RpcPort  string
-	GrpcPort string
-	RestPort string
-	EvmPort  string
+	RpcUrl    string
+	GrpcUrl   string
+	EvmRpcUrl string
 
 	ChainId      string
 	StakingDenom string
 }
 
 func (c ChainDetails) EvmClient() (*ethclient.Client, error) {
-	evmRpcUrl := fmt.Sprintf("http://localhost:%s", c.EvmPort)
-	return ethclient.Dial(evmRpcUrl)
+	return ethclient.Dial(c.EvmRpcUrl)
 }
 
 func (c ChainDetails) GrpcConn() (*grpc.ClientConn, error) {
-	grpcUrl := fmt.Sprintf("http://localhost:%s", c.GrpcPort)
-	return util.NewGrpcConnection(grpcUrl)
+	return util.NewGrpcConnection(c.GrpcUrl)
 }
 
 type Chains struct {
@@ -60,23 +57,20 @@ func (c *Chains) Register(name string, chain *ChainDetails) error {
 }
 
 // the Chain details are all hardcoded because they are currently fixed by kvtool.
-// some day they may be configurable, at which point `runner` can determine the ports
-// and generate these details dynamically
+// someday they may be accepted as configurable parameters.
 var (
-	kavaChain = ChainDetails{
-		RpcPort:  "26657",
-		RestPort: "1317",
-		GrpcPort: "9090",
-		EvmPort:  "8545",
+	kvtoolKavaChain = ChainDetails{
+		RpcUrl:    "http://localhost:26657",
+		GrpcUrl:   "http://localhost:9090",
+		EvmRpcUrl: "http://localhost:8545",
 
 		ChainId:      "kavalocalnet_8888-1",
 		StakingDenom: "ukava",
 	}
-	ibcChain = ChainDetails{
-		RpcPort:  "26658",
-		RestPort: "1318",
-		GrpcPort: "9092",
-		EvmPort:  "8547",
+	kvtoolIbcChain = ChainDetails{
+		RpcUrl:    "http://localhost:26658",
+		GrpcUrl:   "http://localhost:9092",
+		EvmRpcUrl: "http://localhost:8547",
 
 		ChainId:      "kavalocalnet_8889-2",
 		StakingDenom: "uatom",

tests/e2e/runner/kvtool.go

@@ -0,0 +1,104 @@
+package runner
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+)
+
+type KvtoolRunnerConfig struct {
+	KavaConfigTemplate string
+
+	ImageTag   string
+	IncludeIBC bool
+
+	EnableAutomatedUpgrade  bool
+	KavaUpgradeName         string
+	KavaUpgradeHeight       int64
+	KavaUpgradeBaseImageTag string
+
+	SkipShutdown bool
+}
+
+// KvtoolRunner implements a NodeRunner that spins up local chains with kvtool.
+// It has support for the following:
+// - running a Kava node
+// - optionally, running an IBC node with a channel opened to the Kava node
+// - optionally, start the Kava node on one version and upgrade to another
+type KvtoolRunner struct {
+	config KvtoolRunnerConfig
+}
+
+var _ NodeRunner = &KvtoolRunner{}
+
+func NewKvtoolRunner(config KvtoolRunnerConfig) *KvtoolRunner {
+	return &KvtoolRunner{
+		config: config,
+	}
+}
+
+func (k *KvtoolRunner) StartChains() Chains {
+	// install kvtool if not already installed
+	installKvtoolCmd := exec.Command("./scripts/install-kvtool.sh")
+	installKvtoolCmd.Stdout = os.Stdout
+	installKvtoolCmd.Stderr = os.Stderr
+	if err := installKvtoolCmd.Run(); err != nil {
+		panic(fmt.Sprintf("failed to install kvtool: %s", err.Error()))
+	}
+
+	// start local test network with kvtool
+	log.Println("starting kava node")
+	kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate}
+	// include an ibc chain if desired
+	if k.config.IncludeIBC {
+		kvtoolArgs = append(kvtoolArgs, "--ibc")
+	}
+	// handle automated upgrade functionality, if defined
+	if k.config.EnableAutomatedUpgrade {
+		kvtoolArgs = append(kvtoolArgs,
+			"--upgrade-name", k.config.KavaUpgradeName,
+			"--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight),
+			"--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag,
+		)
+	}
+	// start the chain
+	startKavaCmd := exec.Command("kvtool", kvtoolArgs...)
+	startKavaCmd.Env = os.Environ()
+	startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag))
+	startKavaCmd.Stdout = os.Stdout
+	startKavaCmd.Stderr = os.Stderr
+	log.Println(startKavaCmd.String())
+	if err := startKavaCmd.Run(); err != nil {
+		panic(fmt.Sprintf("failed to start kava: %s", err.Error()))
+	}
+
+	// wait for chain to be live.
+	// if an upgrade is defined, this waits for the upgrade to be completed.
+	if err := waitForChainStart(kvtoolKavaChain); err != nil {
+		k.Shutdown()
+		panic(err)
+	}
+	log.Println("kava is started!")
+
+	chains := NewChains()
+	chains.Register("kava", &kvtoolKavaChain)
+	if k.config.IncludeIBC {
+		chains.Register("ibc", &kvtoolIbcChain)
+	}
+	return chains
+}
+
+func (k *KvtoolRunner) Shutdown() {
+	if k.config.SkipShutdown {
+		log.Printf("would shut down but SkipShutdown is true")
+		return
+	}
+	log.Println("shutting down kava node")
+	shutdownKavaCmd := exec.Command("kvtool", "testnet", "down")
+	shutdownKavaCmd.Stdout = os.Stdout
+	shutdownKavaCmd.Stderr = os.Stderr
+	if err := shutdownKavaCmd.Run(); err != nil {
+		panic(fmt.Sprintf("failed to shutdown kvtool: %s", err.Error()))
+	}
+}

tests/e2e/runner/main.go

@@ -4,128 +4,38 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"os"
-	"os/exec"
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
 )
 
-type Config struct {
-	KavaConfigTemplate string
-
-	ImageTag   string
-	IncludeIBC bool
-
-	EnableAutomatedUpgrade  bool
-	KavaUpgradeName         string
-	KavaUpgradeHeight       int64
-	KavaUpgradeBaseImageTag string
-
-	SkipShutdown bool
-}
-
 // NodeRunner is responsible for starting and managing docker containers to run a node.
 type NodeRunner interface {
 	StartChains() Chains
 	Shutdown()
 }
 
-// KavaNodeRunner manages and runs a single Kava node.
-type KavaNodeRunner struct {
-	config    Config
-	kavaChain *ChainDetails
-}
-
-var _ NodeRunner = &KavaNodeRunner{}
-
-func NewKavaNode(config Config) *KavaNodeRunner {
-	return &KavaNodeRunner{
-		config: config,
-	}
-}
-
-func (k *KavaNodeRunner) StartChains() Chains {
-	installKvtoolCmd := exec.Command("./scripts/install-kvtool.sh")
-	installKvtoolCmd.Stdout = os.Stdout
-	installKvtoolCmd.Stderr = os.Stderr
-	if err := installKvtoolCmd.Run(); err != nil {
-		panic(fmt.Sprintf("failed to install kvtool: %s", err.Error()))
-	}
-
-	log.Println("starting kava node")
-	kvtoolArgs := []string{"testnet", "bootstrap", "--kava.configTemplate", k.config.KavaConfigTemplate}
-	if k.config.IncludeIBC {
-		kvtoolArgs = append(kvtoolArgs, "--ibc")
-	}
-	if k.config.EnableAutomatedUpgrade {
-		kvtoolArgs = append(kvtoolArgs,
-			"--upgrade-name", k.config.KavaUpgradeName,
-			"--upgrade-height", fmt.Sprint(k.config.KavaUpgradeHeight),
-			"--upgrade-base-image-tag", k.config.KavaUpgradeBaseImageTag,
-		)
-	}
-	startKavaCmd := exec.Command("kvtool", kvtoolArgs...)
-	startKavaCmd.Env = os.Environ()
-	startKavaCmd.Env = append(startKavaCmd.Env, fmt.Sprintf("KAVA_TAG=%s", k.config.ImageTag))
-	startKavaCmd.Stdout = os.Stdout
-	startKavaCmd.Stderr = os.Stderr
-	log.Println(startKavaCmd.String())
-	if err := startKavaCmd.Run(); err != nil {
-		panic(fmt.Sprintf("failed to start kava: %s", err.Error()))
-	}
-
-	k.kavaChain = &kavaChain
-
-	err := k.waitForChainStart()
-	if err != nil {
-		k.Shutdown()
-		panic(err)
-	}
-	log.Println("kava is started!")
-
-	chains := NewChains()
-	chains.Register("kava", k.kavaChain)
-	if k.config.IncludeIBC {
-		chains.Register("ibc", &ibcChain)
-	}
-	return chains
-}
-
-func (k *KavaNodeRunner) Shutdown() {
-	if k.config.SkipShutdown {
-		log.Printf("would shut down but SkipShutdown is true")
-		return
-	}
-	log.Println("shutting down kava node")
-	shutdownKavaCmd := exec.Command("kvtool", "testnet", "down")
-	shutdownKavaCmd.Stdout = os.Stdout
-	shutdownKavaCmd.Stderr = os.Stderr
-	if err := shutdownKavaCmd.Run(); err != nil {
-		panic(fmt.Sprintf("failed to shutdown kvtool: %s", err.Error()))
-	}
-}
-
-func (k *KavaNodeRunner) waitForChainStart() error {
+func waitForChainStart(chainDetails ChainDetails) error {
 	// exponential backoff on trying to ping the node, timeout after 30 seconds
 	b := backoff.NewExponentialBackOff()
 	b.MaxInterval = 5 * time.Second
 	b.MaxElapsedTime = 30 * time.Second
-	if err := backoff.Retry(k.pingKava, b); err != nil {
+	if err := backoff.Retry(func() error { return pingKava(chainDetails.RpcUrl) }, b); err != nil {
 		return fmt.Errorf("failed to start & connect to chain: %s", err)
 	}
+
 	b.Reset()
 	// the evm takes a bit longer to start up. wait for it to start as well.
-	if err := backoff.Retry(k.pingEvm, b); err != nil {
+	if err := backoff.Retry(func() error { return pingEvm(chainDetails.EvmRpcUrl) }, b); err != nil {
 		return fmt.Errorf("failed to start & connect to chain: %s", err)
 	}
 	return nil
 }
 
-func (k *KavaNodeRunner) pingKava() error {
+func pingKava(rpcUrl string) error {
 	log.Println("pinging kava chain...")
-	url := fmt.Sprintf("http://localhost:%s/status", k.kavaChain.RpcPort)
-	res, err := http.Get(url)
+	statusUrl := fmt.Sprintf("%s/status", rpcUrl)
+	res, err := http.Get(statusUrl)
 	if err != nil {
 		return err
 	}
@@ -137,10 +47,9 @@ func (k *KavaNodeRunner) pingKava() error {
 	return nil
 }
 
-func (k *KavaNodeRunner) pingEvm() error {
+func pingEvm(evmRpcUrl string) error {
 	log.Println("pinging evm...")
-	url := fmt.Sprintf("http://localhost:%s", k.kavaChain.EvmPort)
-	res, err := http.Get(url)
+	res, err := http.Get(evmRpcUrl)
 	if err != nil {
 		return err
 	}

tests/e2e/testutil/chain.go

@@ -69,14 +69,12 @@ func NewChain(t *testing.T, details *runner.ChainDetails, fundedAccountMnemonic
 	}
 	chain.EncodingConfig = app.MakeEncodingConfig()
 
-	grpcUrl := fmt.Sprintf("http://localhost:%s", details.GrpcPort)
-	grpcConn, err := util.NewGrpcConnection(grpcUrl)
+	grpcConn, err := details.GrpcConn()
 	if err != nil {
 		return chain, err
 	}
 
-	evmRpcUrl := fmt.Sprintf("http://localhost:%s", details.EvmPort)
-	chain.EvmClient, err = ethclient.Dial(evmRpcUrl)
+	chain.EvmClient, err = details.EvmClient()
 	if err != nil {
 		return chain, err
 	}

tests/e2e/testutil/config.go

@@ -16,11 +16,24 @@ func init() {
 type SuiteConfig struct {
 	// A funded account used to fnd all other accounts.
 	FundedAccountMnemonic string
+
+	// A config for using kvtool local networks for the test run
+	Kvtool *KvtoolConfig
+
+	// Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests.
+	IncludeIbcTests bool
+
+	// The contract address of a deployed ERC-20 token
+	KavaErc20Address string
+
+	// When true, the chains will remain running after tests complete (pass or fail)
+	SkipShutdown bool
+}
+
+type KvtoolConfig struct {
 	// The kava.configTemplate flag to be passed to kvtool, usually "master".
 	// This allows one to change the base genesis used to start the chain.
 	KavaConfigTemplate string
-	// Whether or not to start an IBC chain. Use `suite.SkipIfIbcDisabled()` in IBC tests in IBC tests.
-	IncludeIbcTests bool
 
 	// Whether or not to run a chain upgrade & run post-upgrade tests. Use `suite.SkipIfUpgradeDisabled()` in post-upgrade tests.
 	IncludeAutomatedUpgrade bool
@@ -30,23 +43,15 @@ type SuiteConfig struct {
 	KavaUpgradeHeight int64
 	// Tag of kava docker image that will be upgraded to the current image before tests are run, if upgrade is enabled.
 	KavaUpgradeBaseImageTag string
-
-	// The contract address of a deployed ERC-20 token
-	KavaErc20Address string
-
-	// When true, the chains will remain running after tests complete (pass or fail)
-	SkipShutdown bool
 }
 
 func ParseSuiteConfig() SuiteConfig {
 	config := SuiteConfig{
 		// this mnemonic is expected to be a funded account that can seed the funds for all
 		// new accounts created during tests. it will be available under Accounts["whale"]
-		FundedAccountMnemonic:   nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"),
-		KavaConfigTemplate:      nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"),
-		KavaErc20Address:        nonemptyStringEnv("E2E_KAVA_ERC20_ADDRESS"),
-		IncludeIbcTests:         mustParseBool("E2E_INCLUDE_IBC_TESTS"),
-		IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"),
+		FundedAccountMnemonic: nonemptyStringEnv("E2E_KAVA_FUNDED_ACCOUNT_MNEMONIC"),
+		KavaErc20Address:      nonemptyStringEnv("E2E_KAVA_ERC20_ADDRESS"),
+		IncludeIbcTests:       mustParseBool("E2E_INCLUDE_IBC_TESTS"),
 	}
 
 	skipShutdownEnv := os.Getenv("E2E_SKIP_SHUTDOWN")
@@ -54,6 +59,21 @@ func ParseSuiteConfig() SuiteConfig {
 		config.SkipShutdown = mustParseBool("E2E_SKIP_SHUTDOWN")
 	}
 
+	useKvtoolNetworks := mustParseBool("E2E_RUN_KVTOOL_NETWORKS")
+	if useKvtoolNetworks {
+		kvtoolConfig := ParseKvtoolConfig()
+		config.Kvtool = &kvtoolConfig
+	}
+
+	return config
+}
+
+func ParseKvtoolConfig() KvtoolConfig {
+	config := KvtoolConfig{
+		KavaConfigTemplate:      nonemptyStringEnv("E2E_KVTOOL_KAVA_CONFIG_TEMPLATE"),
+		IncludeAutomatedUpgrade: mustParseBool("E2E_INCLUDE_AUTOMATED_UPGRADE"),
+	}
+
 	if config.IncludeAutomatedUpgrade {
 		config.KavaUpgradeName = nonemptyStringEnv("E2E_KAVA_UPGRADE_NAME")
 		config.KavaUpgradeBaseImageTag = nonemptyStringEnv("E2E_KAVA_UPGRADE_BASE_IMAGE_TAG")

tests/e2e/testutil/suite.go

@@ -43,23 +43,26 @@ func (suite *E2eTestSuite) SetupSuite() {
 
 	suiteConfig := ParseSuiteConfig()
 	suite.config = suiteConfig
-	suite.UpgradeHeight = suiteConfig.KavaUpgradeHeight
 	suite.DeployedErc20Address = common.HexToAddress(suiteConfig.KavaErc20Address)
 
-	runnerConfig := runner.Config{
-		KavaConfigTemplate: suiteConfig.KavaConfigTemplate,
+	if suiteConfig.Kvtool != nil {
+		suite.UpgradeHeight = suiteConfig.Kvtool.KavaUpgradeHeight
 
-		IncludeIBC: suiteConfig.IncludeIbcTests,
-		ImageTag:   "local",
+		runnerConfig := runner.KvtoolRunnerConfig{
+			KavaConfigTemplate: suiteConfig.Kvtool.KavaConfigTemplate,
 
-		EnableAutomatedUpgrade:  suiteConfig.IncludeAutomatedUpgrade,
-		KavaUpgradeName:         suiteConfig.KavaUpgradeName,
-		KavaUpgradeHeight:       suiteConfig.KavaUpgradeHeight,
-		KavaUpgradeBaseImageTag: suiteConfig.KavaUpgradeBaseImageTag,
+			IncludeIBC: suiteConfig.IncludeIbcTests,
+			ImageTag:   "local",
 
-		SkipShutdown: suiteConfig.SkipShutdown,
+			EnableAutomatedUpgrade:  suiteConfig.Kvtool.IncludeAutomatedUpgrade,
+			KavaUpgradeName:         suiteConfig.Kvtool.KavaUpgradeName,
+			KavaUpgradeHeight:       suiteConfig.Kvtool.KavaUpgradeHeight,
+			KavaUpgradeBaseImageTag: suiteConfig.Kvtool.KavaUpgradeBaseImageTag,
+
+			SkipShutdown: suiteConfig.SkipShutdown,
+		}
+		suite.runner = runner.NewKvtoolRunner(runnerConfig)
 	}
-	suite.runner = runner.NewKavaNode(runnerConfig)
 
 	chains := suite.runner.StartChains()
 	kavachain := chains.MustGetChain("kava")
@@ -99,7 +102,7 @@ func (suite *E2eTestSuite) SkipIfIbcDisabled() {
 }
 
 func (suite *E2eTestSuite) SkipIfUpgradeDisabled() {
-	if !suite.config.IncludeAutomatedUpgrade {
+	if suite.config.Kvtool != nil && suite.config.Kvtool.IncludeAutomatedUpgrade {
 		suite.T().SkipNow()
 	}
 }