Simulation fixes (#527)

* fix go version in dockerfile

* mix frequently occuring errors

* add missed import

* fix minor validator vesting sim bug

* fix auction sim bug

* fix docker build

* add todo from sim failure

* tidy up dockerfile

* update docs, add dockerignore to speed up builds

* Update simulations/README.md

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>

* add stack traces to logs for easier debugging

* replace root func with sdk version

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>
Co-authored-by: karzak <kjydavis3@gmail.com>

.dockerignore

@@ -0,0 +1,3 @@
+.git/
+build/
+docs/

app/sim_test.go

@@ -3,8 +3,11 @@ package app
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"math/rand"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -13,10 +16,12 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
+	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/simapp"
 	"github.com/cosmos/cosmos-sdk/simapp/helpers"
 	"github.com/cosmos/cosmos-sdk/store"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	distr "github.com/cosmos/cosmos-sdk/x/distribution"
 	"github.com/cosmos/cosmos-sdk/x/gov"
@@ -85,7 +90,7 @@ func TestFullAppSimulation(t *testing.T) {
 
 	// run randomized simulation
 	_, simParams, simErr := simulation.SimulateFromSeed(
-		t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
+		t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		simapp.SimulationOperations(app, app.Codec(), config),
 		app.ModuleAccountAddrs(), config,
 	)
@@ -117,7 +122,7 @@ func TestAppImportExport(t *testing.T) {
 
 	// Run randomized simulation
 	_, simParams, simErr := simulation.SimulateFromSeed(
-		t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
+		t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		simapp.SimulationOperations(app, app.Codec(), config),
 		app.ModuleAccountAddrs(), config,
 	)
@@ -212,7 +217,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 
 	// Run randomized simulation
 	stopEarly, simParams, simErr := simulation.SimulateFromSeed(
-		t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
+		t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		simapp.SimulationOperations(app, app.Codec(), config),
 		app.ModuleAccountAddrs(), config,
 	)
@@ -254,7 +259,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 	})
 
 	_, _, err = simulation.SimulateFromSeed(
-		t, os.Stdout, newApp.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
+		t, os.Stdout, newApp.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 		simapp.SimulationOperations(newApp, newApp.Codec(), config),
 		newApp.ModuleAccountAddrs(), config,
 	)
@@ -287,7 +292,7 @@ func TestAppStateDeterminism(t *testing.T) {
 		)
 
 		_, _, err := simulation.SimulateFromSeed(
-			t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
+			t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
 			simapp.SimulationOperations(app, app.Codec(), config),
 			app.ModuleAccountAddrs(), config,
 		)
@@ -304,3 +309,55 @@ func TestAppStateDeterminism(t *testing.T) {
 		}
 	}
 }
+
+// AppStateFn returns the initial application state using a genesis or the simulation parameters.
+// It panics if the user provides files for both of them.
+// If a file is not given for the genesis or the sim params, it creates a randomized one.
+// Note: this was copied in from the sdk/simapp/state.go, and modified to not generate genesis times too far in the future.
+// Dates greater than the year 9000 interfere with new auctions who's EndTime is set to 90000.
+func AppStateFn(cdc *codec.Codec, simManager *module.SimulationManager) simulation.AppStateFn {
+	return func(r *rand.Rand, accs []simulation.Account, config simulation.Config,
+	) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) {
+
+		if simapp.FlagGenesisTimeValue == 0 {
+			genesisTimestamp = time.Unix(r.Int63n(190288396800), 0) // 1st Jan year 8000
+		} else {
+			genesisTimestamp = time.Unix(simapp.FlagGenesisTimeValue, 0)
+		}
+
+		chainID = config.ChainID
+		switch {
+		case config.ParamsFile != "" && config.GenesisFile != "":
+			panic("cannot provide both a genesis file and a params file")
+
+		case config.GenesisFile != "":
+			// override the default chain-id from simapp to set it later to the config
+			genesisDoc, accounts := simapp.AppStateFromGenesisFileFn(r, cdc, config.GenesisFile)
+
+			if simapp.FlagGenesisTimeValue == 0 {
+				// use genesis timestamp if no custom timestamp is provided (i.e no random timestamp)
+				genesisTimestamp = genesisDoc.GenesisTime
+			}
+
+			appState = genesisDoc.AppState
+			chainID = genesisDoc.ChainID
+			simAccs = accounts
+
+		case config.ParamsFile != "":
+			appParams := make(simulation.AppParams)
+			bz, err := ioutil.ReadFile(config.ParamsFile)
+			if err != nil {
+				panic(err)
+			}
+
+			cdc.MustUnmarshalJSON(bz, &appParams)
+			appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams)
+
+		default:
+			appParams := make(simulation.AppParams)
+			appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams)
+		}
+
+		return appState, simAccs, chainID, genesisTimestamp
+	}
+}

simulations/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:alpine AS build-env
+FROM golang:1.13-alpine AS build-env
 
 # Set up dependencies
 # bash for debugging
@@ -14,13 +14,12 @@ RUN pip install awscli
 WORKDIR /root/kava
 # default home directory is /root
 
+# Download dependencies before adding source files to speed up build times
 COPY go.mod .
 COPY go.sum .
-
 RUN go mod download
 
 # Add source files
-COPY .git .git
 COPY app app
 COPY cli_test cli_test
 COPY cmd cmd
@@ -28,12 +27,6 @@ COPY app app
 COPY x x
 COPY Makefile .
 
-# Install kvd, kvcli
-ENV LEDGER_ENABLED False
-RUN make install
-
-# Copy in simulation script after to decrease image build time
 COPY simulations simulations
 
-# Run kvd by default, omit entrypoint to ease using container with kvcli
+# kvd and kcli binaries are not necessary for running the simulations
-CMD ["kvd"]

simulations/README.md

@@ -21,7 +21,7 @@ This can run sims but doesn't collect the results. This is handled by a custom s
 
 ## Running sims and uploading to S3
 
-The dockerfile in this repo defines the docker image to run sims. It's just a normal app, but with the aws cli included, and the custom script.
+The dockerfile in this repo defines the docker image to run sims. It includes the kava source code, aws cli, and the custom simulation script.
 
 The custom script reads some input args, runs a sim and uploads the stdout and stderr to a S3 bucket.
 
@@ -31,7 +31,7 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates
 
 - create and submit a new array job (based of the job definition) with
   - image `kava/kava-sim:<some-version>`
-  - command `run-then-upload.sh <starting-seed> <num-blocks> <block-size>`
+  - command `run-then-upload.sh TestFullAppSimulation <starting-seed> <num-blocks> <block-size>`
   - array size of how many sims you want to run
 - any changes needed to the code or script necessitates a rebuild:
   - `docker build -f simulations/Dockerfile -t kava/kava-sim:<some-version> .`
@@ -41,5 +41,3 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates
 
 - click on the compute environment name, to get details, then click the link ECS Cluster Name to get details on the actual machines running
 - for array jobs, click the job name to get details of the individual jobs
-
-## Sims - TODO

simulations/run-then-upload.sh

@@ -7,7 +7,7 @@
 
 
 # Parse Input Args
-# get simulation type
+# get simulation type, eg TestFullAppSimulation
 simType=$1
 # get seed
 startingSeed=$2

x/auction/simulation/operations.go

@@ -70,7 +70,7 @@ func SimulateMsgPlaceBid(ak auth.AccountKeeper, keeper keeper.Keeper) simulation
 			openAuctions[i], openAuctions[j] = openAuctions[j], openAuctions[i]
 		})
 
-		// search through auctions and an accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction)
+		// search through auctions and accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction)
 		blockTime := ctx.BlockHeader().Time
 		params := keeper.GetParams(ctx)
 		bidder, openAuction, found := findValidAccountAuctionPair(accs, openAuctions, func(acc simulation.Account, auc types.Auction) bool {

x/bep3/simulation/operations.go

@@ -183,7 +183,7 @@ func SimulateMsgCreateAtomicSwap(ak types.AccountKeeper, k keeper.Keeper) simula
 
 		_, result, err := app.Deliver(tx)
 		if err != nil {
-			return simulation.NoOpMsg(types.ModuleName), nil, err
+			return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 		}
 
 		// Construct a MsgClaimAtomicSwap or MsgRefundAtomicSwap future operation
@@ -274,7 +274,7 @@ func operationClaimAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID []
 		)
 		_, result, err := app.Deliver(tx)
 		if err != nil {
-			return simulation.NoOpMsg(types.ModuleName), nil, err
+			return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 		}
 		return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
 	}
@@ -322,7 +322,7 @@ func operationRefundAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID [
 
 		_, result, err := app.Deliver(tx)
 		if err != nil {
-			return simulation.NoOpMsg(types.ModuleName), nil, err
+			return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 		}
 		return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
 	}

x/cdp/simulation/operations.go

@@ -1,6 +1,7 @@
 package simulation
 
 import (
+	"fmt"
 	"math/rand"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
@@ -66,7 +67,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 
 		price, err := pfk.GetCurrentPrice(ctx, randCollateralParam.SpotMarketID)
 		if err != nil {
-			return simulation.NoOpMsg(types.ModuleName), nil, err
+			return simulation.NoOpMsg(types.ModuleName), nil, nil // pricefeed going down is an expected event
 		}
 		// convert the price to the same units as the debt param
 		priceShifted := ShiftDec(price.Price, debtParam.ConversionFactor)
@@ -125,7 +126,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 
 			_, _, err := app.Deliver(tx)
 			if err != nil {
-				return simulation.NoOpMsg(types.ModuleName), nil, err
+				// to aid debugging, add the stack trace to the comment field of the returned opMsg
+				return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 			}
 
 			return simulation.NewOperationMsg(msg, true, ""), nil, nil
@@ -149,7 +151,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 
 			_, _, err := app.Deliver(tx)
 			if err != nil {
-				return simulation.NoOpMsg(types.ModuleName), nil, err
+				return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 			}
 
 			return simulation.NewOperationMsg(msg, true, ""), nil, nil
@@ -172,7 +174,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 
 			_, _, err := app.Deliver(tx)
 			if err != nil {
-				return simulation.NoOpMsg(types.ModuleName), nil, err
+				return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 			}
 
 			return simulation.NewOperationMsg(msg, true, ""), nil, nil
@@ -214,9 +216,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 			)
 
 			_, _, err := app.Deliver(tx)
-
 			if err != nil {
-				return simulation.NoOpMsg(types.ModuleName), nil, err
+				return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 			}
 
 			return simulation.NewOperationMsg(msg, true, ""), nil, nil
@@ -254,7 +255,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
 
 			_, _, err := app.Deliver(tx)
 			if err != nil {
-				return simulation.NoOpMsg(types.ModuleName), nil, err
+				return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 			}
 
 			return simulation.NewOperationMsg(msg, true, ""), nil, nil

x/committee/simulation/operations.go

@@ -9,6 +9,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/simapp/helpers"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	authexported "github.com/cosmos/cosmos-sdk/x/auth/exported"
+	distsim "github.com/cosmos/cosmos-sdk/x/distribution/simulation"
 	"github.com/cosmos/cosmos-sdk/x/simulation"
 
 	"github.com/kava-labs/kava/x/committee/keeper"
@@ -32,8 +33,9 @@ func WeightedOperations(appParams simulation.AppParams, cdc *codec.Codec, ak Acc
 
 	for _, wContent := range wContents {
 		wContent := wContent // pin variable
-		if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal {
+		if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal || wContent.AppParamsKey == distsim.OpWeightSubmitCommunitySpendProposal {
 			// don't include committee change/delete proposals as they're not enabled for submission to committees
+			// don't include community pool proposals as the generator func sometimes returns nil // TODO replace generator with a better one
 			continue
 		}
 		var weight int

x/kavadist/simulation/genesis.go

@@ -69,7 +69,7 @@ func genRandomInflation(r *rand.Rand) sdk.Dec {
 	aprInflation := sdk.OneDec().Add(extraAprInflation)
 
 	// convert APR inflation to SPR (inflation per second)
-	inflationSpr, err := approxRoot(aprInflation, uint64(SecondsPerYear))
+	inflationSpr, err := aprInflation.ApproxRoot(uint64(SecondsPerYear))
 	if err != nil {
 		panic(fmt.Sprintf("error generating random inflation %v", err))
 	}

x/kavadist/simulation/utils.go

@@ -1,64 +0,0 @@
-package simulation
-
-import (
-	"errors"
-
-	sdk "github.com/cosmos/cosmos-sdk/types"
-)
-
-// TODO use the official version available in v0.38.2 of the sdk
-// https://github.com/cosmos/cosmos-sdk/blob/e8d89a2fe26175b73545a3e79ae783032b4e975e/types/decimal.go#L328
-func approxRoot(d sdk.Dec, root uint64) (guess sdk.Dec, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			var ok bool
-			err, ok = r.(error)
-			if !ok {
-				err = errors.New("out of bounds")
-			}
-		}
-	}()
-
-	if d.IsNegative() {
-		absRoot, err := approxRoot(d.MulInt64(-1), root)
-		return absRoot.MulInt64(-1), err
-	}
-	if root == 1 || d.IsZero() || d.Equal(sdk.OneDec()) {
-		return d, nil
-	}
-	if root == 0 {
-		return sdk.OneDec(), nil
-	}
-	rootInt := sdk.NewInt(int64(root))
-	guess, delta := sdk.OneDec(), sdk.OneDec()
-	for delta.Abs().GT(sdk.SmallestDec()) {
-		prev := power(guess, (root - 1))
-		if prev.IsZero() {
-			prev = sdk.SmallestDec()
-		}
-		delta = d.Quo(prev)
-		delta = delta.Sub(guess)
-		delta = delta.QuoInt(rootInt)
-
-		guess = guess.Add(delta)
-	}
-	return guess, nil
-}
-
-// Power returns a the result of raising to a positive integer power
-func power(d sdk.Dec, power uint64) sdk.Dec {
-	if power == 0 {
-		return sdk.OneDec()
-	}
-	tmp := sdk.OneDec()
-	for i := power; i > 1; {
-		if i%2 == 0 {
-			i /= 2
-		} else {
-			tmp = tmp.Mul(d)
-			i = (i - 1) / 2
-		}
-		d = d.Mul(d)
-	}
-	return d.Mul(tmp)
-}

x/kavadist/simulation/utils_test.go

@@ -1,31 +0,0 @@
-package simulation
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	sdk "github.com/cosmos/cosmos-sdk/types"
-)
-
-func TestApproxRoot(t *testing.T) {
-	testCases := []struct {
-		input    sdk.Dec
-		root     uint64
-		expected sdk.Dec
-	}{
-		{sdk.OneDec(), 10, sdk.OneDec()},                                                       // 1.0 ^ (0.1) => 1.0
-		{sdk.NewDecWithPrec(25, 2), 2, sdk.NewDecWithPrec(5, 1)},                               // 0.25 ^ (0.5) => 0.5
-		{sdk.NewDecWithPrec(4, 2), 2, sdk.NewDecWithPrec(2, 1)},                                // 0.04 => 0.2
-		{sdk.NewDecFromInt(sdk.NewInt(27)), 3, sdk.NewDecFromInt(sdk.NewInt(3))},               // 27 ^ (1/3) => 3
-		{sdk.NewDecFromInt(sdk.NewInt(-81)), 4, sdk.NewDecFromInt(sdk.NewInt(-3))},             // -81 ^ (0.25) => -3
-		{sdk.NewDecFromInt(sdk.NewInt(2)), 2, sdk.NewDecWithPrec(1414213562373095049, 18)},     // 2 ^ (0.5) => 1.414213562373095049
-		{sdk.NewDecWithPrec(1005, 3), 31536000, sdk.MustNewDecFromStr("1.000000000158153904")}, // 1.005 ^ (1/31536000) = 1.000000000158153904
-	}
-
-	for i, tc := range testCases {
-		res, err := approxRoot(tc.input, tc.root)
-		require.NoError(t, err)
-		require.True(t, tc.expected.Sub(res).Abs().LTE(sdk.SmallestDec()), "unexpected result for test case %d, input: %v", i, tc.input)
-	}
-}

x/pricefeed/simulation/operations.go

@@ -1,6 +1,7 @@
 package simulation
 
 import (
+	"fmt"
 	"math/rand"
 	"time"
 
@@ -19,6 +20,14 @@ import (
 // Simulation operation weights constants
 const (
 	OpWeightMsgUpdatePrices = "op_weight_msg_update_prices"
+
+	// Block time params are un-exported constants in cosmos-sdk/x/simulation.
+	// Copy them here in lieu of importing them.
+	minTimePerBlock time.Duration = (10000 / 2) * time.Second
+	maxTimePerBlock time.Duration = 10000 * time.Second
+
+	// Calculate the average block time
+	AverageBlockTime time.Duration = (maxTimePerBlock - minTimePerBlock) / 2
 )
 
 // WeightedOperations returns all the operations from the module with their respective weights
@@ -101,7 +110,8 @@ func SimulateMsgUpdatePrices(ak auth.AccountKeeper, keeper keeper.Keeper, blocks
 
 		_, result, err := app.Deliver(tx)
 		if err != nil {
-			return simulation.NoOpMsg(types.ModuleName), nil, err
+			// to aid debugging, add the stack trace to the comment field of the returned opMsg
+			return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
 		}
 		return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
 	}
@@ -126,5 +136,5 @@ func pickRandomAsset(ctx sdk.Context, keeper keeper.Keeper, r *rand.Rand) (marke
 // getExpiryTime gets a price expiry time by taking the current time and adding a delta to it
 func getExpiryTime(ctx sdk.Context) (t time.Time) {
 	// need to use the blocktime from the context as the context generates random start time when running simulations
-	return ctx.BlockTime().Add(time.Second * 1000000)
+	return ctx.BlockTime().Add(AverageBlockTime * 5000) // if blocks were 6 seconds, the expiry would be 8 hrs
 }

x/validator-vesting/simulation/genesis.go

@@ -85,10 +85,13 @@ func getRandomValidatorConsAddr(simState *module.SimulationState, rint int) sdk.
 
 func getRandomVestingPeriods(duration int64, r *rand.Rand, origCoins sdk.Coins) vestingtypes.Periods {
 	maxPeriods := int64(50)
+	if duration <= 0 {
+		panic("duration in getRandomVestingPeriods cannot be <= 0")
+	}
 	if duration < maxPeriods {
 		maxPeriods = duration
 	}
-	numPeriods := simulation.RandIntBetween(r, 1, int(maxPeriods))
+	numPeriods := r.Intn(int(maxPeriods)) + 1 // ensures >= 1, 0 check above prevents Intn panicking
 	lenPeriod := duration / int64(numPeriods)
 	periodLengths := make([]int64, numPeriods)
 	totalLength := int64(0)