Merge remote-tracking branch 'origin' into kd-pin-to-sdk

Makefile

@@ -105,9 +105,25 @@ test-all: build
 	# other sim tests
 	@go test ./app -run TestAppImportExport          -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
 	@go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
-	@# AppStateDeterminism does use Seed flag
-	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h
+	@# AppStateDeterminism does not use Seed flag
+	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=100 -BlockSize=200         -v -timeout 24h
 
 test:
 	@go test ./...
-.PHONY: all build-linux install clean build test-all test
+
+# Kick start lots of sims on an AWS cluster.
+# This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3
+start-remote-sims:
+	# build the image used for running sims in, and tag it
+	docker build -f simulations/Dockerfile -t kava/kava-sim:master .
+	# push that image to the hub
+	docker push kava/kava-sim:master
+	# submit an array job on AWS Batch, using 1000 seeds, spot instances
+	aws batch submit-job \
+		-—job-name "master-$(VERSION)" \
+		-—job-queue “simulation-1-queue-spot" \
+		-—array-properties size=1000 \
+		-—job-definition kava-sim-master \
+		-—container-override environment=[{SIM_NAME=master-$(VERSION)}]
+
+.PHONY: all build-linux install clean build test test-all start-remote-sims

app/sim_test.go

@@ -58,8 +58,18 @@ const (
 	OpWeightMsgUnjail                                  = "op_weight_msg_unjail"
 )
 
-func init() {
+// TestMain runs setup and teardown code before all tests.
+func TestMain(m *testing.M) {
+	// set prefixes
+	config := sdk.GetConfig()
+	SetBech32AddressPrefixes(config)
+	config.Seal()
+	// load the values from simulation specific flags
 	simapp.GetSimulatorFlags()
+
+	// run tests
+	exitCode := m.Run()
+	os.Exit(exitCode)
 }
 
 func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation {
@@ -319,6 +329,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
 	}
 }
 
+// TestFullAppSimulation runs a standard simulation of the app, modified by cmd line flag values.
 func TestFullAppSimulation(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation")
@@ -373,6 +384,7 @@ func TestFullAppSimulation(t *testing.T) {
 	}
 }
 
+// TestAppImportExport runs a simulation, exports the state, imports it, then checks the db state is same after import as it was before export.
 func TestAppImportExport(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application import/export simulation")
@@ -490,6 +502,7 @@ func TestAppImportExport(t *testing.T) {
 	}
 }
 
+// TestAppSimulationAfterImport runs a simulation, exports it, imports it and runs another simulation.
 func TestAppSimulationAfterImport(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation after import")
@@ -584,6 +597,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 
 // TODO: Make another test for the fuzzer itself, which just has noOp txs
 // and doesn't depend on the application.
+// TestAppStateDeterminism runs several sims with the same seed and checks the states are equal.
 func TestAppStateDeterminism(t *testing.T) {
 	if !simapp.FlagEnabledValue {
 		t.Skip("skipping application simulation")

simulations/Dockerfile

@@ -25,6 +25,7 @@ COPY app app
 COPY cli_test cli_test
 COPY cmd cmd
 COPY app app
+COPY x x
 COPY Makefile .
 
 # Install kvd, kvcli

simulations/run-then-upload.sh

@@ -7,24 +7,27 @@
 
 
 # Parse Input Args
+# get simulation type
+simType=$1
 # get seed
-startingSeed=$1
+startingSeed=$2
 # compute the seed from the starting and the job index
 # add two nums together, hence the $(()), and use 0 as the default value for array index, hence the ${:-} syntax
 seed=$(($startingSeed+${AWS_BATCH_JOB_ARRAY_INDEX:-0}))
-echo "seed: " $seed
 # get sim parameters
-numBlocks=$2
-blockSize=$3
+numBlocks=$3
+blockSize=$4
 
 
 # Run The Sim
-# redirect stdout and stderr to a file
-go test ./app -run TestFullAppSimulation -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h > out.log 2>&1
+# record cli arguments in the log file (in json in case we need to parse this) and stdout (https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout)
+printf "{\"simType\": \"%s\", \"startingSeed\": %s, \"seed\": %s, \"numBlocks\": %s, \"blockSize\": %s}\n" $simType $startingSeed $seed $numBlocks $blockSize | tee out.log
+# run sim and redirect stdout and stderr to a file)
+go test ./app -run $simType -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h >> out.log 2>&1
 # get the exit code to determine how to upload results
 simExitStatus=$?
 if [ $simExitStatus -eq 0 ];then
-   echo "simulations passed"
+   echo "simulation passed"
    simResult="pass"
 else
    echo "simulation failed"
@@ -39,5 +42,6 @@ jobID=${AWS_BATCH_JOB_ID:-"testJobID:"}
 jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/')
 
 # create the filename from the array job index (which won't be set if this is a normal job)
-fileName=out$AWS_BATCH_JOB_ARRAY_INDEX.log
-aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName
+fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX)
+
+aws s3 cp out.log s3://simulations-1/$SIM_NAME$jobID/$simResult/$fileName