From 2ffb1edd1f781efb205c0cf07a4e8cc9a502b2a1 Mon Sep 17 00:00:00 2001 From: Ruaridh Date: Tue, 1 Sep 2020 16:33:12 +0100 Subject: [PATCH] Simulation fixes (#527) * fix go version in dockerfile * mix frequently occuring errors * add missed import * fix minor validator vesting sim bug * fix auction sim bug * fix docker build * add todo from sim failure * tidy up dockerfile * update docs, add dockerignore to speed up builds * Update simulations/README.md Co-authored-by: Kevin Davis * add stack traces to logs for easier debugging * replace root func with sdk version Co-authored-by: Kevin Davis Co-authored-by: karzak --- .dockerignore | 3 + app/sim_test.go | 67 +++++++++++++++++++++-- simulations/Dockerfile | 13 +---- simulations/README.md | 6 +- simulations/run-then-upload.sh | 2 +- x/auction/simulation/operations.go | 2 +- x/bep3/simulation/operations.go | 6 +- x/cdp/simulation/operations.go | 15 ++--- x/committee/simulation/operations.go | 4 +- x/kavadist/simulation/genesis.go | 2 +- x/kavadist/simulation/utils.go | 64 ---------------------- x/kavadist/simulation/utils_test.go | 31 ----------- x/pricefeed/simulation/operations.go | 14 ++++- x/validator-vesting/simulation/genesis.go | 5 +- 14 files changed, 103 insertions(+), 131 deletions(-) create mode 100644 .dockerignore delete mode 100644 x/kavadist/simulation/utils.go delete mode 100644 x/kavadist/simulation/utils_test.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..29d822d0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git/ +build/ +docs/ diff --git a/app/sim_test.go b/app/sim_test.go index 8a42b28a..e49136db 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -3,8 +3,11 @@ package app import ( "encoding/json" "fmt" + "io/ioutil" + "math/rand" "os" "testing" + "time" "github.com/stretchr/testify/require" @@ -13,10 +16,12 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/simapp" "github.com/cosmos/cosmos-sdk/simapp/helpers" "github.com/cosmos/cosmos-sdk/store" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/module" "github.com/cosmos/cosmos-sdk/x/auth" distr "github.com/cosmos/cosmos-sdk/x/distribution" "github.com/cosmos/cosmos-sdk/x/gov" @@ -85,7 +90,7 @@ func TestFullAppSimulation(t *testing.T) { // run randomized simulation _, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()), simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) @@ -117,7 +122,7 @@ func TestAppImportExport(t *testing.T) { // Run randomized simulation _, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()), simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) @@ -212,7 +217,7 @@ func TestAppSimulationAfterImport(t *testing.T) { // Run randomized simulation stopEarly, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()), simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) @@ -254,7 +259,7 @@ func TestAppSimulationAfterImport(t *testing.T) { }) _, _, err = simulation.SimulateFromSeed( - t, os.Stdout, newApp.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + t, os.Stdout, newApp.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()), simapp.SimulationOperations(newApp, newApp.Codec(), config), newApp.ModuleAccountAddrs(), config, ) @@ -287,7 +292,7 @@ func TestAppStateDeterminism(t *testing.T) { ) _, _, err := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()), simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) @@ -304,3 +309,55 @@ func TestAppStateDeterminism(t *testing.T) { } } } + +// AppStateFn returns the initial application state using a genesis or the simulation parameters. +// It panics if the user provides files for both of them. +// If a file is not given for the genesis or the sim params, it creates a randomized one. +// Note: this was copied in from the sdk/simapp/state.go, and modified to not generate genesis times too far in the future. +// Dates greater than the year 9000 interfere with new auctions who's EndTime is set to 90000. +func AppStateFn(cdc *codec.Codec, simManager *module.SimulationManager) simulation.AppStateFn { + return func(r *rand.Rand, accs []simulation.Account, config simulation.Config, + ) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) { + + if simapp.FlagGenesisTimeValue == 0 { + genesisTimestamp = time.Unix(r.Int63n(190288396800), 0) // 1st Jan year 8000 + } else { + genesisTimestamp = time.Unix(simapp.FlagGenesisTimeValue, 0) + } + + chainID = config.ChainID + switch { + case config.ParamsFile != "" && config.GenesisFile != "": + panic("cannot provide both a genesis file and a params file") + + case config.GenesisFile != "": + // override the default chain-id from simapp to set it later to the config + genesisDoc, accounts := simapp.AppStateFromGenesisFileFn(r, cdc, config.GenesisFile) + + if simapp.FlagGenesisTimeValue == 0 { + // use genesis timestamp if no custom timestamp is provided (i.e no random timestamp) + genesisTimestamp = genesisDoc.GenesisTime + } + + appState = genesisDoc.AppState + chainID = genesisDoc.ChainID + simAccs = accounts + + case config.ParamsFile != "": + appParams := make(simulation.AppParams) + bz, err := ioutil.ReadFile(config.ParamsFile) + if err != nil { + panic(err) + } + + cdc.MustUnmarshalJSON(bz, &appParams) + appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams) + + default: + appParams := make(simulation.AppParams) + appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams) + } + + return appState, simAccs, chainID, genesisTimestamp + } +} diff --git a/simulations/Dockerfile b/simulations/Dockerfile index f07db3b7..42309096 100644 --- a/simulations/Dockerfile +++ b/simulations/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine AS build-env +FROM golang:1.13-alpine AS build-env # Set up dependencies # bash for debugging @@ -14,13 +14,12 @@ RUN pip install awscli WORKDIR /root/kava # default home directory is /root +# Download dependencies before adding source files to speed up build times COPY go.mod . COPY go.sum . - RUN go mod download # Add source files -COPY .git .git COPY app app COPY cli_test cli_test COPY cmd cmd @@ -28,12 +27,6 @@ COPY app app COPY x x COPY Makefile . -# Install kvd, kvcli -ENV LEDGER_ENABLED False -RUN make install - -# Copy in simulation script after to decrease image build time COPY simulations simulations -# Run kvd by default, omit entrypoint to ease using container with kvcli -CMD ["kvd"] +# kvd and kcli binaries are not necessary for running the simulations \ No newline at end of file diff --git a/simulations/README.md b/simulations/README.md index c4260146..4f48f16b 100644 --- a/simulations/README.md +++ b/simulations/README.md @@ -21,7 +21,7 @@ This can run sims but doesn't collect the results. This is handled by a custom s ## Running sims and uploading to S3 -The dockerfile in this repo defines the docker image to run sims. It's just a normal app, but with the aws cli included, and the custom script. +The dockerfile in this repo defines the docker image to run sims. It includes the kava source code, aws cli, and the custom simulation script. The custom script reads some input args, runs a sim and uploads the stdout and stderr to a S3 bucket. @@ -31,7 +31,7 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates - create and submit a new array job (based of the job definition) with - image `kava/kava-sim:` - - command `run-then-upload.sh ` + - command `run-then-upload.sh TestFullAppSimulation ` - array size of how many sims you want to run - any changes needed to the code or script necessitates a rebuild: - `docker build -f simulations/Dockerfile -t kava/kava-sim: .` @@ -41,5 +41,3 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates - click on the compute environment name, to get details, then click the link ECS Cluster Name to get details on the actual machines running - for array jobs, click the job name to get details of the individual jobs - -## Sims - TODO \ No newline at end of file diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index 39e45997..d44fd6cf 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -7,7 +7,7 @@ # Parse Input Args -# get simulation type +# get simulation type, eg TestFullAppSimulation simType=$1 # get seed startingSeed=$2 diff --git a/x/auction/simulation/operations.go b/x/auction/simulation/operations.go index b81b2340..1f2a9225 100644 --- a/x/auction/simulation/operations.go +++ b/x/auction/simulation/operations.go @@ -70,7 +70,7 @@ func SimulateMsgPlaceBid(ak auth.AccountKeeper, keeper keeper.Keeper) simulation openAuctions[i], openAuctions[j] = openAuctions[j], openAuctions[i] }) - // search through auctions and an accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction) + // search through auctions and accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction) blockTime := ctx.BlockHeader().Time params := keeper.GetParams(ctx) bidder, openAuction, found := findValidAccountAuctionPair(accs, openAuctions, func(acc simulation.Account, auc types.Auction) bool { diff --git a/x/bep3/simulation/operations.go b/x/bep3/simulation/operations.go index 9654f5e8..681512a9 100644 --- a/x/bep3/simulation/operations.go +++ b/x/bep3/simulation/operations.go @@ -183,7 +183,7 @@ func SimulateMsgCreateAtomicSwap(ak types.AccountKeeper, k keeper.Keeper) simula _, result, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } // Construct a MsgClaimAtomicSwap or MsgRefundAtomicSwap future operation @@ -274,7 +274,7 @@ func operationClaimAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID [] ) _, result, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, result.Log), nil, nil } @@ -322,7 +322,7 @@ func operationRefundAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID [ _, result, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, result.Log), nil, nil } diff --git a/x/cdp/simulation/operations.go b/x/cdp/simulation/operations.go index b16c653e..a88a0d15 100644 --- a/x/cdp/simulation/operations.go +++ b/x/cdp/simulation/operations.go @@ -1,6 +1,7 @@ package simulation import ( + "fmt" "math/rand" "github.com/cosmos/cosmos-sdk/baseapp" @@ -66,7 +67,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed price, err := pfk.GetCurrentPrice(ctx, randCollateralParam.SpotMarketID) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NoOpMsg(types.ModuleName), nil, nil // pricefeed going down is an expected event } // convert the price to the same units as the debt param priceShifted := ShiftDec(price.Price, debtParam.ConversionFactor) @@ -125,7 +126,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed _, _, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + // to aid debugging, add the stack trace to the comment field of the returned opMsg + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, ""), nil, nil @@ -149,7 +151,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed _, _, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, ""), nil, nil @@ -172,7 +174,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed _, _, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, ""), nil, nil @@ -214,9 +216,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed ) _, _, err := app.Deliver(tx) - if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, ""), nil, nil @@ -254,7 +255,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed _, _, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, ""), nil, nil diff --git a/x/committee/simulation/operations.go b/x/committee/simulation/operations.go index 466dfc82..4322b659 100644 --- a/x/committee/simulation/operations.go +++ b/x/committee/simulation/operations.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/simapp/helpers" sdk "github.com/cosmos/cosmos-sdk/types" authexported "github.com/cosmos/cosmos-sdk/x/auth/exported" + distsim "github.com/cosmos/cosmos-sdk/x/distribution/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" "github.com/kava-labs/kava/x/committee/keeper" @@ -32,8 +33,9 @@ func WeightedOperations(appParams simulation.AppParams, cdc *codec.Codec, ak Acc for _, wContent := range wContents { wContent := wContent // pin variable - if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal { + if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal || wContent.AppParamsKey == distsim.OpWeightSubmitCommunitySpendProposal { // don't include committee change/delete proposals as they're not enabled for submission to committees + // don't include community pool proposals as the generator func sometimes returns nil // TODO replace generator with a better one continue } var weight int diff --git a/x/kavadist/simulation/genesis.go b/x/kavadist/simulation/genesis.go index 5b34ecc6..43ff9736 100644 --- a/x/kavadist/simulation/genesis.go +++ b/x/kavadist/simulation/genesis.go @@ -69,7 +69,7 @@ func genRandomInflation(r *rand.Rand) sdk.Dec { aprInflation := sdk.OneDec().Add(extraAprInflation) // convert APR inflation to SPR (inflation per second) - inflationSpr, err := approxRoot(aprInflation, uint64(SecondsPerYear)) + inflationSpr, err := aprInflation.ApproxRoot(uint64(SecondsPerYear)) if err != nil { panic(fmt.Sprintf("error generating random inflation %v", err)) } diff --git a/x/kavadist/simulation/utils.go b/x/kavadist/simulation/utils.go deleted file mode 100644 index b6f771b7..00000000 --- a/x/kavadist/simulation/utils.go +++ /dev/null @@ -1,64 +0,0 @@ -package simulation - -import ( - "errors" - - sdk "github.com/cosmos/cosmos-sdk/types" -) - -// TODO use the official version available in v0.38.2 of the sdk -// https://github.com/cosmos/cosmos-sdk/blob/e8d89a2fe26175b73545a3e79ae783032b4e975e/types/decimal.go#L328 -func approxRoot(d sdk.Dec, root uint64) (guess sdk.Dec, err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - err, ok = r.(error) - if !ok { - err = errors.New("out of bounds") - } - } - }() - - if d.IsNegative() { - absRoot, err := approxRoot(d.MulInt64(-1), root) - return absRoot.MulInt64(-1), err - } - if root == 1 || d.IsZero() || d.Equal(sdk.OneDec()) { - return d, nil - } - if root == 0 { - return sdk.OneDec(), nil - } - rootInt := sdk.NewInt(int64(root)) - guess, delta := sdk.OneDec(), sdk.OneDec() - for delta.Abs().GT(sdk.SmallestDec()) { - prev := power(guess, (root - 1)) - if prev.IsZero() { - prev = sdk.SmallestDec() - } - delta = d.Quo(prev) - delta = delta.Sub(guess) - delta = delta.QuoInt(rootInt) - - guess = guess.Add(delta) - } - return guess, nil -} - -// Power returns a the result of raising to a positive integer power -func power(d sdk.Dec, power uint64) sdk.Dec { - if power == 0 { - return sdk.OneDec() - } - tmp := sdk.OneDec() - for i := power; i > 1; { - if i%2 == 0 { - i /= 2 - } else { - tmp = tmp.Mul(d) - i = (i - 1) / 2 - } - d = d.Mul(d) - } - return d.Mul(tmp) -} diff --git a/x/kavadist/simulation/utils_test.go b/x/kavadist/simulation/utils_test.go deleted file mode 100644 index 2521bb5f..00000000 --- a/x/kavadist/simulation/utils_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package simulation - -import ( - "testing" - - "github.com/stretchr/testify/require" - - sdk "github.com/cosmos/cosmos-sdk/types" -) - -func TestApproxRoot(t *testing.T) { - testCases := []struct { - input sdk.Dec - root uint64 - expected sdk.Dec - }{ - {sdk.OneDec(), 10, sdk.OneDec()}, // 1.0 ^ (0.1) => 1.0 - {sdk.NewDecWithPrec(25, 2), 2, sdk.NewDecWithPrec(5, 1)}, // 0.25 ^ (0.5) => 0.5 - {sdk.NewDecWithPrec(4, 2), 2, sdk.NewDecWithPrec(2, 1)}, // 0.04 => 0.2 - {sdk.NewDecFromInt(sdk.NewInt(27)), 3, sdk.NewDecFromInt(sdk.NewInt(3))}, // 27 ^ (1/3) => 3 - {sdk.NewDecFromInt(sdk.NewInt(-81)), 4, sdk.NewDecFromInt(sdk.NewInt(-3))}, // -81 ^ (0.25) => -3 - {sdk.NewDecFromInt(sdk.NewInt(2)), 2, sdk.NewDecWithPrec(1414213562373095049, 18)}, // 2 ^ (0.5) => 1.414213562373095049 - {sdk.NewDecWithPrec(1005, 3), 31536000, sdk.MustNewDecFromStr("1.000000000158153904")}, // 1.005 ^ (1/31536000) = 1.000000000158153904 - } - - for i, tc := range testCases { - res, err := approxRoot(tc.input, tc.root) - require.NoError(t, err) - require.True(t, tc.expected.Sub(res).Abs().LTE(sdk.SmallestDec()), "unexpected result for test case %d, input: %v", i, tc.input) - } -} diff --git a/x/pricefeed/simulation/operations.go b/x/pricefeed/simulation/operations.go index 5b58ca16..6a1d9b51 100644 --- a/x/pricefeed/simulation/operations.go +++ b/x/pricefeed/simulation/operations.go @@ -1,6 +1,7 @@ package simulation import ( + "fmt" "math/rand" "time" @@ -19,6 +20,14 @@ import ( // Simulation operation weights constants const ( OpWeightMsgUpdatePrices = "op_weight_msg_update_prices" + + // Block time params are un-exported constants in cosmos-sdk/x/simulation. + // Copy them here in lieu of importing them. + minTimePerBlock time.Duration = (10000 / 2) * time.Second + maxTimePerBlock time.Duration = 10000 * time.Second + + // Calculate the average block time + AverageBlockTime time.Duration = (maxTimePerBlock - minTimePerBlock) / 2 ) // WeightedOperations returns all the operations from the module with their respective weights @@ -101,7 +110,8 @@ func SimulateMsgUpdatePrices(ak auth.AccountKeeper, keeper keeper.Keeper, blocks _, result, err := app.Deliver(tx) if err != nil { - return simulation.NoOpMsg(types.ModuleName), nil, err + // to aid debugging, add the stack trace to the comment field of the returned opMsg + return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err } return simulation.NewOperationMsg(msg, true, result.Log), nil, nil } @@ -126,5 +136,5 @@ func pickRandomAsset(ctx sdk.Context, keeper keeper.Keeper, r *rand.Rand) (marke // getExpiryTime gets a price expiry time by taking the current time and adding a delta to it func getExpiryTime(ctx sdk.Context) (t time.Time) { // need to use the blocktime from the context as the context generates random start time when running simulations - return ctx.BlockTime().Add(time.Second * 1000000) + return ctx.BlockTime().Add(AverageBlockTime * 5000) // if blocks were 6 seconds, the expiry would be 8 hrs } diff --git a/x/validator-vesting/simulation/genesis.go b/x/validator-vesting/simulation/genesis.go index 9a39dd5b..4ab9915d 100644 --- a/x/validator-vesting/simulation/genesis.go +++ b/x/validator-vesting/simulation/genesis.go @@ -85,10 +85,13 @@ func getRandomValidatorConsAddr(simState *module.SimulationState, rint int) sdk. func getRandomVestingPeriods(duration int64, r *rand.Rand, origCoins sdk.Coins) vestingtypes.Periods { maxPeriods := int64(50) + if duration <= 0 { + panic("duration in getRandomVestingPeriods cannot be <= 0") + } if duration < maxPeriods { maxPeriods = duration } - numPeriods := simulation.RandIntBetween(r, 1, int(maxPeriods)) + numPeriods := r.Intn(int(maxPeriods)) + 1 // ensures >= 1, 0 check above prevents Intn panicking lenPeriod := duration / int64(numPeriods) periodLengths := make([]int64, numPeriods) totalLength := int64(0)