Merge pull request #257 from Kava-Labs/ro/sim-improvements

Sim Improvements

Makefile

@@ -105,8 +105,22 @@ test-all: build
 	# other sim tests
 	@go test ./app -run TestAppImportExport          -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
 	@go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
-	@# AppStateDeterminism does use Seed flag
+	@# AppStateDeterminism does not use Seed flag
 	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=100 -BlockSize=200         -v -timeout 24h
 
+# Kick start lots of sims on an AWS cluster.
+# This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3
+start-remote-sims:
+	# build the image used for running sims in, and tag it
+	docker build -f simulations/Dockerfile -t kava/kava-sim:master .
+	# push that image to the hub
+	docker push kava/kava-sim:master
+	# submit an array job on AWS Batch, using 1000 seeds, spot instances
+	aws batch submit-job \
+		-—job-name "master-$(VERSION)" \
+		-—job-queue “simulation-1-queue-spot" \
+		-—array-properties size=1000 \
+		-—job-definition kava-sim-master \
+		-—container-override environment=[{SIM_NAME=master-$(VERSION)}]
 
-.PHONY: all build-linux install clean build test-all
+.PHONY: all build-linux install clean build test-all start-remote-sims

app/sim_test.go

@@ -58,8 +58,18 @@ const (
 	OpWeightMsgUnjail                                  = "op_weight_msg_unjail"
 )
 
-func init() {
+// TestMain runs setup and teardown code before all tests.
+func TestMain(m *testing.M) {
+	// set prefixes
+	config := sdk.GetConfig()
+	SetBech32AddressPrefixes(config)
+	config.Seal()
+	// load the values from simulation specific flags
 	simapp.GetSimulatorFlags()
+
+	// run tests
+	exitCode := m.Run()
+	os.Exit(exitCode)
 }
 
 func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation {
@@ -319,6 +329,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
 	}
 }
 
+// TestFullAppSimulation runs a standard simulation of the app, modified by cmd line flag values.
 func TestFullAppSimulation(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation")
@@ -373,6 +384,7 @@ func TestFullAppSimulation(t *testing.T) {
 	}
 }
 
+// TestAppImportExport runs a simulation, exports the state, imports it, then checks the db state is same after import as it was before export.
 func TestAppImportExport(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application import/export simulation")
@@ -490,6 +502,7 @@ func TestAppImportExport(t *testing.T) {
 	}
 }
 
+// TestAppSimulationAfterImport runs a simulation, exports it, imports it and runs another simulation.
 func TestAppSimulationAfterImport(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation after import")
@@ -584,6 +597,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 
 // TODO: Make another test for the fuzzer itself, which just has noOp txs
 // and doesn't depend on the application.
+// TestAppStateDeterminism runs several sims with the same seed and checks the states are equal.
 func TestAppStateDeterminism(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation")

go.mod

@@ -3,7 +3,7 @@ module github.com/kava-labs/kava
 go 1.13
 
 require (
-	github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e
+	github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c
 	github.com/gorilla/mux v1.7.3
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.4.0
@@ -13,5 +13,3 @@ require (
 	github.com/tendermint/tm-db v0.2.0
 	gopkg.in/yaml.v2 v2.2.4
 )
-
-replace github.com/cosmos/cosmos-sdk => ../../cosmos/cosmos-sdk

go.sum

@@ -38,6 +38,8 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e h1:V8WpJTIAjajE2PE+1wWCG5LUYkWQal+aH6uqPUiZ9Qc=
 github.com/cosmos/cosmos-sdk v0.34.4-0.20190925161702-9d0bed8f4f4e/go.mod h1:gwKdI16dOjylNYJkaHbcx0TcEIHyRs1xyc5qROmjCJE=
+github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c h1:LM81MVa0CG0Q118dOp6f2Q0MoYIMRTi26zoyqZiL8sY=
+github.com/cosmos/cosmos-sdk v0.34.4-0.20191010155330-64a27412505c/go.mod h1:FxjFo2Y2ZuqUczTcVfnDUG413OgofBbxFNXA72eaUR4=
 github.com/cosmos/cosmos-sdk v0.37.1 h1:mz5W3Au32VIPPtrY65dheVYeVDSFfS3eSSmuIj+cXsI=
 github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8 h1:Iwin12wRQtyZhH6FV3ykFcdGNlYEzoeR0jN8Vn+JWsI=
 github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
@@ -154,6 +156,8 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -304,6 +308,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

simulations/Dockerfile

@@ -25,6 +25,7 @@ COPY app app
 COPY cli_test cli_test
 COPY cmd cmd
 COPY app app
+COPY x x
 COPY Makefile .
 
 # Install kvd, kvcli

simulations/run-then-upload.sh

@@ -7,24 +7,27 @@
 
 
 # Parse Input Args
+# get simulation type
+simType=$1
 # get seed
-startingSeed=$1
+startingSeed=$2
 # compute the seed from the starting and the job index
 # add two nums together, hence the $(()), and use 0 as the default value for array index, hence the ${:-} syntax
 seed=$(($startingSeed+${AWS_BATCH_JOB_ARRAY_INDEX:-0}))
-echo "seed: " $seed
 # get sim parameters
-numBlocks=$2
+numBlocks=$3
-blockSize=$3
+blockSize=$4
 
 
 # Run The Sim
-# redirect stdout and stderr to a file
+# record cli arguments in the log file (in json in case we need to parse this) and stdout (https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout)
-go test ./app -run TestFullAppSimulation -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h > out.log 2>&1
+printf "{\"simType\": \"%s\", \"startingSeed\": %s, \"seed\": %s, \"numBlocks\": %s, \"blockSize\": %s}\n" $simType $startingSeed $seed $numBlocks $blockSize | tee out.log
+# run sim and redirect stdout and stderr to a file)
+go test ./app -run $simType -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h >> out.log 2>&1
 # get the exit code to determine how to upload results
 simExitStatus=$?
 if [ $simExitStatus -eq 0 ];then
-   echo "simulations passed"
+   echo "simulation passed"
    simResult="pass"
 else
    echo "simulation failed"
@@ -39,5 +42,6 @@ jobID=${AWS_BATCH_JOB_ID:-"testJobID:"}
 jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/')
 
 # create the filename from the array job index (which won't be set if this is a normal job)
-fileName=out$AWS_BATCH_JOB_ARRAY_INDEX.log
+fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX)
-aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName
+
+aws s3 cp out.log s3://simulations-1/$SIM_NAME$jobID/$simResult/$fileName