Simulation fixes (#527)

* fix go version in dockerfile

* mix frequently occuring errors

* add missed import

* fix minor validator vesting sim bug

* fix auction sim bug

* fix docker build

* add todo from sim failure

* tidy up dockerfile

* update docs, add dockerignore to speed up builds

* Update simulations/README.md

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>

* add stack traces to logs for easier debugging

* replace root func with sdk version

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>
Co-authored-by: karzak <kjydavis3@gmail.com>
This commit is contained in:
Ruaridh 2020-09-01 16:33:12 +01:00 committed by GitHub
parent f22139fcee
commit 2ffb1edd1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 103 additions and 131 deletions

3
.dockerignore Normal file
View File

@ -0,0 +1,3 @@
.git/
build/
docs/

View File

@ -3,8 +3,11 @@ package app
import (
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"testing"
"time"
"github.com/stretchr/testify/require"
@ -13,10 +16,12 @@ import (
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/codec"
"github.com/cosmos/cosmos-sdk/simapp"
"github.com/cosmos/cosmos-sdk/simapp/helpers"
"github.com/cosmos/cosmos-sdk/store"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/types/module"
"github.com/cosmos/cosmos-sdk/x/auth"
distr "github.com/cosmos/cosmos-sdk/x/distribution"
"github.com/cosmos/cosmos-sdk/x/gov"
@ -85,7 +90,7 @@ func TestFullAppSimulation(t *testing.T) {
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
simapp.SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
@ -117,7 +122,7 @@ func TestAppImportExport(t *testing.T) {
// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
simapp.SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
@ -212,7 +217,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
// Run randomized simulation
stopEarly, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
simapp.SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
@ -254,7 +259,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
})
_, _, err = simulation.SimulateFromSeed(
t, os.Stdout, newApp.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
t, os.Stdout, newApp.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
simapp.SimulationOperations(newApp, newApp.Codec(), config),
newApp.ModuleAccountAddrs(), config,
)
@ -287,7 +292,7 @@ func TestAppStateDeterminism(t *testing.T) {
)
_, _, err := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
simapp.SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
@ -304,3 +309,55 @@ func TestAppStateDeterminism(t *testing.T) {
}
}
}
// AppStateFn returns the initial application state using a genesis or the simulation parameters.
// It panics if the user provides files for both of them.
// If a file is not given for the genesis or the sim params, it creates a randomized one.
// Note: this was copied in from the sdk/simapp/state.go, and modified to not generate genesis times too far in the future.
// Dates greater than the year 9000 interfere with new auctions who's EndTime is set to 90000.
func AppStateFn(cdc *codec.Codec, simManager *module.SimulationManager) simulation.AppStateFn {
return func(r *rand.Rand, accs []simulation.Account, config simulation.Config,
) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) {
if simapp.FlagGenesisTimeValue == 0 {
genesisTimestamp = time.Unix(r.Int63n(190288396800), 0) // 1st Jan year 8000
} else {
genesisTimestamp = time.Unix(simapp.FlagGenesisTimeValue, 0)
}
chainID = config.ChainID
switch {
case config.ParamsFile != "" && config.GenesisFile != "":
panic("cannot provide both a genesis file and a params file")
case config.GenesisFile != "":
// override the default chain-id from simapp to set it later to the config
genesisDoc, accounts := simapp.AppStateFromGenesisFileFn(r, cdc, config.GenesisFile)
if simapp.FlagGenesisTimeValue == 0 {
// use genesis timestamp if no custom timestamp is provided (i.e no random timestamp)
genesisTimestamp = genesisDoc.GenesisTime
}
appState = genesisDoc.AppState
chainID = genesisDoc.ChainID
simAccs = accounts
case config.ParamsFile != "":
appParams := make(simulation.AppParams)
bz, err := ioutil.ReadFile(config.ParamsFile)
if err != nil {
panic(err)
}
cdc.MustUnmarshalJSON(bz, &appParams)
appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams)
default:
appParams := make(simulation.AppParams)
appState, simAccs = simapp.AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams)
}
return appState, simAccs, chainID, genesisTimestamp
}
}

View File

@ -1,4 +1,4 @@
FROM golang:alpine AS build-env
FROM golang:1.13-alpine AS build-env
# Set up dependencies
# bash for debugging
@ -14,13 +14,12 @@ RUN pip install awscli
WORKDIR /root/kava
# default home directory is /root
# Download dependencies before adding source files to speed up build times
COPY go.mod .
COPY go.sum .
RUN go mod download
# Add source files
COPY .git .git
COPY app app
COPY cli_test cli_test
COPY cmd cmd
@ -28,12 +27,6 @@ COPY app app
COPY x x
COPY Makefile .
# Install kvd, kvcli
ENV LEDGER_ENABLED False
RUN make install
# Copy in simulation script after to decrease image build time
COPY simulations simulations
# Run kvd by default, omit entrypoint to ease using container with kvcli
CMD ["kvd"]
# kvd and kcli binaries are not necessary for running the simulations

View File

@ -21,7 +21,7 @@ This can run sims but doesn't collect the results. This is handled by a custom s
## Running sims and uploading to S3
The dockerfile in this repo defines the docker image to run sims. It's just a normal app, but with the aws cli included, and the custom script.
The dockerfile in this repo defines the docker image to run sims. It includes the kava source code, aws cli, and the custom simulation script.
The custom script reads some input args, runs a sim and uploads the stdout and stderr to a S3 bucket.
@ -31,7 +31,7 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates
- create and submit a new array job (based of the job definition) with
- image `kava/kava-sim:<some-version>`
- command `run-then-upload.sh <starting-seed> <num-blocks> <block-size>`
- command `run-then-upload.sh TestFullAppSimulation <starting-seed> <num-blocks> <block-size>`
- array size of how many sims you want to run
- any changes needed to the code or script necessitates a rebuild:
- `docker build -f simulations/Dockerfile -t kava/kava-sim:<some-version> .`
@ -41,5 +41,3 @@ AWS Batch allows for "array jobs" which are a way of specifying many duplicates
- click on the compute environment name, to get details, then click the link ECS Cluster Name to get details on the actual machines running
- for array jobs, click the job name to get details of the individual jobs
## Sims - TODO

View File

@ -7,7 +7,7 @@
# Parse Input Args
# get simulation type
# get simulation type, eg TestFullAppSimulation
simType=$1
# get seed
startingSeed=$2

View File

@ -70,7 +70,7 @@ func SimulateMsgPlaceBid(ak auth.AccountKeeper, keeper keeper.Keeper) simulation
openAuctions[i], openAuctions[j] = openAuctions[j], openAuctions[i]
})
// search through auctions and an accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction)
// search through auctions and accounts to find a pair where a bid can be placed (ie account has enough coins to place bid on auction)
blockTime := ctx.BlockHeader().Time
params := keeper.GetParams(ctx)
bidder, openAuction, found := findValidAccountAuctionPair(accs, openAuctions, func(acc simulation.Account, auc types.Auction) bool {

View File

@ -183,7 +183,7 @@ func SimulateMsgCreateAtomicSwap(ak types.AccountKeeper, k keeper.Keeper) simula
_, result, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
// Construct a MsgClaimAtomicSwap or MsgRefundAtomicSwap future operation
@ -274,7 +274,7 @@ func operationClaimAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID []
)
_, result, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
}
@ -322,7 +322,7 @@ func operationRefundAtomicSwap(ak types.AccountKeeper, k keeper.Keeper, swapID [
_, result, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
}

View File

@ -1,6 +1,7 @@
package simulation
import (
"fmt"
"math/rand"
"github.com/cosmos/cosmos-sdk/baseapp"
@ -66,7 +67,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
price, err := pfk.GetCurrentPrice(ctx, randCollateralParam.SpotMarketID)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NoOpMsg(types.ModuleName), nil, nil // pricefeed going down is an expected event
}
// convert the price to the same units as the debt param
priceShifted := ShiftDec(price.Price, debtParam.ConversionFactor)
@ -125,7 +126,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
_, _, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
// to aid debugging, add the stack trace to the comment field of the returned opMsg
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, ""), nil, nil
@ -149,7 +151,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
_, _, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, ""), nil, nil
@ -172,7 +174,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
_, _, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, ""), nil, nil
@ -214,9 +216,8 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
)
_, _, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, ""), nil, nil
@ -254,7 +255,7 @@ func SimulateMsgCdp(ak types.AccountKeeper, k keeper.Keeper, pfk types.Pricefeed
_, _, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, ""), nil, nil

View File

@ -9,6 +9,7 @@ import (
"github.com/cosmos/cosmos-sdk/simapp/helpers"
sdk "github.com/cosmos/cosmos-sdk/types"
authexported "github.com/cosmos/cosmos-sdk/x/auth/exported"
distsim "github.com/cosmos/cosmos-sdk/x/distribution/simulation"
"github.com/cosmos/cosmos-sdk/x/simulation"
"github.com/kava-labs/kava/x/committee/keeper"
@ -32,8 +33,9 @@ func WeightedOperations(appParams simulation.AppParams, cdc *codec.Codec, ak Acc
for _, wContent := range wContents {
wContent := wContent // pin variable
if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal {
if wContent.AppParamsKey == OpWeightSubmitCommitteeChangeProposal || wContent.AppParamsKey == distsim.OpWeightSubmitCommunitySpendProposal {
// don't include committee change/delete proposals as they're not enabled for submission to committees
// don't include community pool proposals as the generator func sometimes returns nil // TODO replace generator with a better one
continue
}
var weight int

View File

@ -69,7 +69,7 @@ func genRandomInflation(r *rand.Rand) sdk.Dec {
aprInflation := sdk.OneDec().Add(extraAprInflation)
// convert APR inflation to SPR (inflation per second)
inflationSpr, err := approxRoot(aprInflation, uint64(SecondsPerYear))
inflationSpr, err := aprInflation.ApproxRoot(uint64(SecondsPerYear))
if err != nil {
panic(fmt.Sprintf("error generating random inflation %v", err))
}

View File

@ -1,64 +0,0 @@
package simulation
import (
"errors"
sdk "github.com/cosmos/cosmos-sdk/types"
)
// TODO use the official version available in v0.38.2 of the sdk
// https://github.com/cosmos/cosmos-sdk/blob/e8d89a2fe26175b73545a3e79ae783032b4e975e/types/decimal.go#L328
func approxRoot(d sdk.Dec, root uint64) (guess sdk.Dec, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = errors.New("out of bounds")
}
}
}()
if d.IsNegative() {
absRoot, err := approxRoot(d.MulInt64(-1), root)
return absRoot.MulInt64(-1), err
}
if root == 1 || d.IsZero() || d.Equal(sdk.OneDec()) {
return d, nil
}
if root == 0 {
return sdk.OneDec(), nil
}
rootInt := sdk.NewInt(int64(root))
guess, delta := sdk.OneDec(), sdk.OneDec()
for delta.Abs().GT(sdk.SmallestDec()) {
prev := power(guess, (root - 1))
if prev.IsZero() {
prev = sdk.SmallestDec()
}
delta = d.Quo(prev)
delta = delta.Sub(guess)
delta = delta.QuoInt(rootInt)
guess = guess.Add(delta)
}
return guess, nil
}
// Power returns a the result of raising to a positive integer power
func power(d sdk.Dec, power uint64) sdk.Dec {
if power == 0 {
return sdk.OneDec()
}
tmp := sdk.OneDec()
for i := power; i > 1; {
if i%2 == 0 {
i /= 2
} else {
tmp = tmp.Mul(d)
i = (i - 1) / 2
}
d = d.Mul(d)
}
return d.Mul(tmp)
}

View File

@ -1,31 +0,0 @@
package simulation
import (
"testing"
"github.com/stretchr/testify/require"
sdk "github.com/cosmos/cosmos-sdk/types"
)
func TestApproxRoot(t *testing.T) {
testCases := []struct {
input sdk.Dec
root uint64
expected sdk.Dec
}{
{sdk.OneDec(), 10, sdk.OneDec()}, // 1.0 ^ (0.1) => 1.0
{sdk.NewDecWithPrec(25, 2), 2, sdk.NewDecWithPrec(5, 1)}, // 0.25 ^ (0.5) => 0.5
{sdk.NewDecWithPrec(4, 2), 2, sdk.NewDecWithPrec(2, 1)}, // 0.04 => 0.2
{sdk.NewDecFromInt(sdk.NewInt(27)), 3, sdk.NewDecFromInt(sdk.NewInt(3))}, // 27 ^ (1/3) => 3
{sdk.NewDecFromInt(sdk.NewInt(-81)), 4, sdk.NewDecFromInt(sdk.NewInt(-3))}, // -81 ^ (0.25) => -3
{sdk.NewDecFromInt(sdk.NewInt(2)), 2, sdk.NewDecWithPrec(1414213562373095049, 18)}, // 2 ^ (0.5) => 1.414213562373095049
{sdk.NewDecWithPrec(1005, 3), 31536000, sdk.MustNewDecFromStr("1.000000000158153904")}, // 1.005 ^ (1/31536000) = 1.000000000158153904
}
for i, tc := range testCases {
res, err := approxRoot(tc.input, tc.root)
require.NoError(t, err)
require.True(t, tc.expected.Sub(res).Abs().LTE(sdk.SmallestDec()), "unexpected result for test case %d, input: %v", i, tc.input)
}
}

View File

@ -1,6 +1,7 @@
package simulation
import (
"fmt"
"math/rand"
"time"
@ -19,6 +20,14 @@ import (
// Simulation operation weights constants
const (
OpWeightMsgUpdatePrices = "op_weight_msg_update_prices"
// Block time params are un-exported constants in cosmos-sdk/x/simulation.
// Copy them here in lieu of importing them.
minTimePerBlock time.Duration = (10000 / 2) * time.Second
maxTimePerBlock time.Duration = 10000 * time.Second
// Calculate the average block time
AverageBlockTime time.Duration = (maxTimePerBlock - minTimePerBlock) / 2
)
// WeightedOperations returns all the operations from the module with their respective weights
@ -101,7 +110,8 @@ func SimulateMsgUpdatePrices(ak auth.AccountKeeper, keeper keeper.Keeper, blocks
_, result, err := app.Deliver(tx)
if err != nil {
return simulation.NoOpMsg(types.ModuleName), nil, err
// to aid debugging, add the stack trace to the comment field of the returned opMsg
return simulation.NewOperationMsg(msg, false, fmt.Sprintf("%+v", err)), nil, err
}
return simulation.NewOperationMsg(msg, true, result.Log), nil, nil
}
@ -126,5 +136,5 @@ func pickRandomAsset(ctx sdk.Context, keeper keeper.Keeper, r *rand.Rand) (marke
// getExpiryTime gets a price expiry time by taking the current time and adding a delta to it
func getExpiryTime(ctx sdk.Context) (t time.Time) {
// need to use the blocktime from the context as the context generates random start time when running simulations
return ctx.BlockTime().Add(time.Second * 1000000)
return ctx.BlockTime().Add(AverageBlockTime * 5000) // if blocks were 6 seconds, the expiry would be 8 hrs
}

View File

@ -85,10 +85,13 @@ func getRandomValidatorConsAddr(simState *module.SimulationState, rint int) sdk.
func getRandomVestingPeriods(duration int64, r *rand.Rand, origCoins sdk.Coins) vestingtypes.Periods {
maxPeriods := int64(50)
if duration <= 0 {
panic("duration in getRandomVestingPeriods cannot be <= 0")
}
if duration < maxPeriods {
maxPeriods = duration
}
numPeriods := simulation.RandIntBetween(r, 1, int(maxPeriods))
numPeriods := r.Intn(int(maxPeriods)) + 1 // ensures >= 1, 0 check above prevents Intn panicking
lenPeriod := duration / int64(numPeriods)
periodLengths := make([]int64, numPeriods)
totalLength := int64(0)