From a2572fad0f1d0d5514228da6afe5db9b1bf92268 Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Thu, 10 Oct 2019 12:36:19 -0400 Subject: [PATCH 1/8] feat: pin to sdk 64a2741 --- go.mod | 4 +--- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5e39708d..e7a3803c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/kava-labs/kava go 1.13 require ( - github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e + github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c github.com/gorilla/mux v1.7.3 github.com/spf13/cobra v0.0.5 github.com/spf13/viper v1.4.0 @@ -13,5 +13,3 @@ require ( github.com/tendermint/tm-db v0.2.0 gopkg.in/yaml.v2 v2.2.4 ) - -replace github.com/cosmos/cosmos-sdk => ../../cosmos/cosmos-sdk diff --git a/go.sum b/go.sum index 2b1c40e9..7c04b8a0 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e h1:V8WpJTIAjajE2PE+1wWCG5LUYkWQal+aH6uqPUiZ9Qc= github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e/go.mod h1:gwKdI16dOjylNYJkaHbcx0TcEIHyRs1xyc5qROmjCJE= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c h1:LM81MVa0CG0Q118dOp6f2Q0MoYIMRTi26zoyqZiL8sY= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c/go.mod h1:FxjFo2Y2ZuqUczTcVfnDUG413OgofBbxFNXA72eaUR4= github.com/cosmos/cosmos-sdk v0.37.1 h1:mz5W3Au32VIPPtrY65dheVYeVDSFfS3eSSmuIj+cXsI= github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8 h1:Iwin12wRQtyZhH6FV3ykFcdGNlYEzoeR0jN8Vn+JWsI= github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= @@ -154,6 +156,8 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -304,6 +308,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From bbf42c7c29c94c6410b2770678e62334715e3933 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:06 -0400 Subject: [PATCH 2/8] add vesting to dockerfile --- simulations/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/simulations/Dockerfile b/simulations/Dockerfile index ff843e68..f07db3b7 100644 --- a/simulations/Dockerfile +++ b/simulations/Dockerfile @@ -25,6 +25,7 @@ COPY app app COPY cli_test cli_test COPY cmd cmd COPY app app +COPY x x COPY Makefile . # Install kvd, kvcli From 6f1cdeb336117fad0d29a5eaaf1eb98719b134c7 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:28 -0400 Subject: [PATCH 3/8] set address prefixes for tests --- app/sim_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 298acccc..387e1477 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -58,8 +58,18 @@ const ( OpWeightMsgUnjail = "op_weight_msg_unjail" ) -func init() { - simapp.GetSimulatorFlags() +// TestMain runs setup and teardown code before all tests. +func TestMain(m *testing.M) { + // set prefixes + config := sdk.GetConfig() + SetBech32AddressPrefixes(config) + config.Seal() + // load the values from simulation specific flags + simapp.GetSimulatorFlags() + + // run tests + exitCode := m.Run() + os.Exit(exitCode) } func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation { From 5befb6a9546c154eb723f93fd1402ab3d7e1837d Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:51 -0400 Subject: [PATCH 4/8] sim script UX improvements --- simulations/run-then-upload.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index d862adf3..3bf54c3e 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -7,24 +7,27 @@ # Parse Input Args +# get simulation type +simType=$1 # get seed -startingSeed=$1 +startingSeed=$2 # compute the seed from the starting and the job index # add two nums together, hence the $(()), and use 0 as the default value for array index, hence the ${:-} syntax seed=$(($startingSeed+${AWS_BATCH_JOB_ARRAY_INDEX:-0})) -echo "seed: " $seed # get sim parameters -numBlocks=$2 -blockSize=$3 +numBlocks=$3 +blockSize=$4 # Run The Sim -# redirect stdout and stderr to a file -go test ./app -run TestFullAppSimulation -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h > out.log 2>&1 +# record cli arguments in the log file (in json in case we need to parse this) and stdout (https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout) +printf "{\"simType\": \"%s\", \"startingSeed\": %s, \"seed\": %s, \"numBlocks\": %s, \"blockSize\": %s}\n" $simType $startingSeed $seed $numBlocks $blockSize | tee out.log +# run sim and redirect stdout and stderr to a file) +go test ./app -run $simType -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h >> out.log 2>&1 # get the exit code to determine how to upload results simExitStatus=$? if [ $simExitStatus -eq 0 ];then - echo "simulations passed" + echo "simulation passed" simResult="pass" else echo "simulation failed" @@ -39,5 +42,5 @@ jobID=${AWS_BATCH_JOB_ID:-"testJobID:"} jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/') # create the filename from the array job index (which won't be set if this is a normal job) -fileName=out$AWS_BATCH_JOB_ARRAY_INDEX.log +fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX) aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName \ No newline at end of file From 11c1d1b369fd27b015352d372fcd7ed7f2c1aa25 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Mon, 7 Oct 2019 17:44:30 -0400 Subject: [PATCH 5/8] add "sim name" env var for clarity --- simulations/run-then-upload.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index 3bf54c3e..39e45997 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -43,4 +43,5 @@ jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/') # create the filename from the array job index (which won't be set if this is a normal job) fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX) -aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName \ No newline at end of file + +aws s3 cp out.log s3://simulations-1/$SIM_NAME$jobID/$simResult/$fileName \ No newline at end of file From 154af93476f243e755440a644ec0f171a7f890b5 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Thu, 10 Oct 2019 12:03:26 -0400 Subject: [PATCH 6/8] add sim test comments --- app/sim_test.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 387e1477..ede0aaf1 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -60,16 +60,16 @@ const ( // TestMain runs setup and teardown code before all tests. func TestMain(m *testing.M) { - // set prefixes - config := sdk.GetConfig() - SetBech32AddressPrefixes(config) - config.Seal() - // load the values from simulation specific flags - simapp.GetSimulatorFlags() + // set prefixes + config := sdk.GetConfig() + SetBech32AddressPrefixes(config) + config.Seal() + // load the values from simulation specific flags + simapp.GetSimulatorFlags() - // run tests - exitCode := m.Run() - os.Exit(exitCode) + // run tests + exitCode := m.Run() + os.Exit(exitCode) } func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation { @@ -329,6 +329,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { } } +// TestFullAppSimulation runs a standard simulation of the app, modified by cmd line flag values. func TestFullAppSimulation(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") @@ -383,6 +384,7 @@ func TestFullAppSimulation(t *testing.T) { } } +// TestAppImportExport runs a simulation, exports the state, imports it, then checks the db state is same after import as it was before export. func TestAppImportExport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application import/export simulation") @@ -500,6 +502,7 @@ func TestAppImportExport(t *testing.T) { } } +// TestAppSimulationAfterImport runs a simulation, exports it, imports it and runs another simulation. func TestAppSimulationAfterImport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation after import") @@ -594,6 +597,7 @@ func TestAppSimulationAfterImport(t *testing.T) { // TODO: Make another test for the fuzzer itself, which just has noOp txs // and doesn't depend on the application. +// TestAppStateDeterminism runs several sims with the same seed and checks the states are equal. func TestAppStateDeterminism(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") From 887f062674d6171e60bc9733518362da504e4183 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 11 Oct 2019 11:38:34 -0400 Subject: [PATCH 7/8] comment typo --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ed080a2a..f77d9037 100644 --- a/Makefile +++ b/Makefile @@ -105,8 +105,8 @@ test-all: build # other sim tests @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h - @# AppStateDeterminism does use Seed flag - @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h + @# AppStateDeterminism does not use Seed flag + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h .PHONY: all build-linux install clean build test-all \ No newline at end of file From b8857bad428a1d6a3736c1529397ba6a66d9aecc Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 11 Oct 2019 14:53:38 -0400 Subject: [PATCH 8/8] add remote sim script --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f77d9037..c448f11e 100644 --- a/Makefile +++ b/Makefile @@ -108,5 +108,19 @@ test-all: build @# AppStateDeterminism does not use Seed flag @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h +# Kick start lots of sims on an AWS cluster. +# This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3 +start-remote-sims: + # build the image used for running sims in, and tag it + docker build -f simulations/Dockerfile -t kava/kava-sim:master . + # push that image to the hub + docker push kava/kava-sim:master + # submit an array job on AWS Batch, using 1000 seeds, spot instances + aws batch submit-job \ + -—job-name "master-$(VERSION)" \ + -—job-queue “simulation-1-queue-spot" \ + -—array-properties size=1000 \ + -—job-definition kava-sim-master \ + -—container-override environment=[{SIM_NAME=master-$(VERSION)}] -.PHONY: all build-linux install clean build test-all \ No newline at end of file +.PHONY: all build-linux install clean build test-all start-remote-sims \ No newline at end of file