From bbf42c7c29c94c6410b2770678e62334715e3933 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:06 -0400 Subject: [PATCH 1/7] add vesting to dockerfile --- simulations/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/simulations/Dockerfile b/simulations/Dockerfile index ff843e68..f07db3b7 100644 --- a/simulations/Dockerfile +++ b/simulations/Dockerfile @@ -25,6 +25,7 @@ COPY app app COPY cli_test cli_test COPY cmd cmd COPY app app +COPY x x COPY Makefile . # Install kvd, kvcli From 6f1cdeb336117fad0d29a5eaaf1eb98719b134c7 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:28 -0400 Subject: [PATCH 2/7] set address prefixes for tests --- app/sim_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 298acccc..387e1477 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -58,8 +58,18 @@ const ( OpWeightMsgUnjail = "op_weight_msg_unjail" ) -func init() { - simapp.GetSimulatorFlags() +// TestMain runs setup and teardown code before all tests. +func TestMain(m *testing.M) { + // set prefixes + config := sdk.GetConfig() + SetBech32AddressPrefixes(config) + config.Seal() + // load the values from simulation specific flags + simapp.GetSimulatorFlags() + + // run tests + exitCode := m.Run() + os.Exit(exitCode) } func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation { From 5befb6a9546c154eb723f93fd1402ab3d7e1837d Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 4 Oct 2019 13:44:51 -0400 Subject: [PATCH 3/7] sim script UX improvements --- simulations/run-then-upload.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index d862adf3..3bf54c3e 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -7,24 +7,27 @@ # Parse Input Args +# get simulation type +simType=$1 # get seed -startingSeed=$1 +startingSeed=$2 # compute the seed from the starting and the job index # add two nums together, hence the $(()), and use 0 as the default value for array index, hence the ${:-} syntax seed=$(($startingSeed+${AWS_BATCH_JOB_ARRAY_INDEX:-0})) -echo "seed: " $seed # get sim parameters -numBlocks=$2 -blockSize=$3 +numBlocks=$3 +blockSize=$4 # Run The Sim -# redirect stdout and stderr to a file -go test ./app -run TestFullAppSimulation -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h > out.log 2>&1 +# record cli arguments in the log file (in json in case we need to parse this) and stdout (https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout) +printf "{\"simType\": \"%s\", \"startingSeed\": %s, \"seed\": %s, \"numBlocks\": %s, \"blockSize\": %s}\n" $simType $startingSeed $seed $numBlocks $blockSize | tee out.log +# run sim and redirect stdout and stderr to a file) +go test ./app -run $simType -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h >> out.log 2>&1 # get the exit code to determine how to upload results simExitStatus=$? if [ $simExitStatus -eq 0 ];then - echo "simulations passed" + echo "simulation passed" simResult="pass" else echo "simulation failed" @@ -39,5 +42,5 @@ jobID=${AWS_BATCH_JOB_ID:-"testJobID:"} jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/') # create the filename from the array job index (which won't be set if this is a normal job) -fileName=out$AWS_BATCH_JOB_ARRAY_INDEX.log +fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX) aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName \ No newline at end of file From 11c1d1b369fd27b015352d372fcd7ed7f2c1aa25 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Mon, 7 Oct 2019 17:44:30 -0400 Subject: [PATCH 4/7] add "sim name" env var for clarity --- simulations/run-then-upload.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index 3bf54c3e..39e45997 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -43,4 +43,5 @@ jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/') # create the filename from the array job index (which won't be set if this is a normal job) fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX) -aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName \ No newline at end of file + +aws s3 cp out.log s3://simulations-1/$SIM_NAME$jobID/$simResult/$fileName \ No newline at end of file From 154af93476f243e755440a644ec0f171a7f890b5 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Thu, 10 Oct 2019 12:03:26 -0400 Subject: [PATCH 5/7] add sim test comments --- app/sim_test.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 387e1477..ede0aaf1 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -60,16 +60,16 @@ const ( // TestMain runs setup and teardown code before all tests. func TestMain(m *testing.M) { - // set prefixes - config := sdk.GetConfig() - SetBech32AddressPrefixes(config) - config.Seal() - // load the values from simulation specific flags - simapp.GetSimulatorFlags() + // set prefixes + config := sdk.GetConfig() + SetBech32AddressPrefixes(config) + config.Seal() + // load the values from simulation specific flags + simapp.GetSimulatorFlags() - // run tests - exitCode := m.Run() - os.Exit(exitCode) + // run tests + exitCode := m.Run() + os.Exit(exitCode) } func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation { @@ -329,6 +329,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { } } +// TestFullAppSimulation runs a standard simulation of the app, modified by cmd line flag values. func TestFullAppSimulation(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") @@ -383,6 +384,7 @@ func TestFullAppSimulation(t *testing.T) { } } +// TestAppImportExport runs a simulation, exports the state, imports it, then checks the db state is same after import as it was before export. func TestAppImportExport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application import/export simulation") @@ -500,6 +502,7 @@ func TestAppImportExport(t *testing.T) { } } +// TestAppSimulationAfterImport runs a simulation, exports it, imports it and runs another simulation. func TestAppSimulationAfterImport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation after import") @@ -594,6 +597,7 @@ func TestAppSimulationAfterImport(t *testing.T) { // TODO: Make another test for the fuzzer itself, which just has noOp txs // and doesn't depend on the application. +// TestAppStateDeterminism runs several sims with the same seed and checks the states are equal. func TestAppStateDeterminism(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") From 887f062674d6171e60bc9733518362da504e4183 Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 11 Oct 2019 11:38:34 -0400 Subject: [PATCH 6/7] comment typo --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ed080a2a..f77d9037 100644 --- a/Makefile +++ b/Makefile @@ -105,8 +105,8 @@ test-all: build # other sim tests @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h - @# AppStateDeterminism does use Seed flag - @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h + @# AppStateDeterminism does not use Seed flag + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h .PHONY: all build-linux install clean build test-all \ No newline at end of file From b8857bad428a1d6a3736c1529397ba6a66d9aecc Mon Sep 17 00:00:00 2001 From: rhuairahrighairigh Date: Fri, 11 Oct 2019 14:53:38 -0400 Subject: [PATCH 7/7] add remote sim script --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f77d9037..c448f11e 100644 --- a/Makefile +++ b/Makefile @@ -108,5 +108,19 @@ test-all: build @# AppStateDeterminism does not use Seed flag @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h +# Kick start lots of sims on an AWS cluster. +# This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3 +start-remote-sims: + # build the image used for running sims in, and tag it + docker build -f simulations/Dockerfile -t kava/kava-sim:master . + # push that image to the hub + docker push kava/kava-sim:master + # submit an array job on AWS Batch, using 1000 seeds, spot instances + aws batch submit-job \ + -—job-name "master-$(VERSION)" \ + -—job-queue “simulation-1-queue-spot" \ + -—array-properties size=1000 \ + -—job-definition kava-sim-master \ + -—container-override environment=[{SIM_NAME=master-$(VERSION)}] -.PHONY: all build-linux install clean build test-all \ No newline at end of file +.PHONY: all build-linux install clean build test-all start-remote-sims \ No newline at end of file