diff --git a/Makefile b/Makefile index 146a1d9e..114bad36 100644 --- a/Makefile +++ b/Makefile @@ -105,9 +105,25 @@ test-all: build # other sim tests @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h - @# AppStateDeterminism does use Seed flag - @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h + @# AppStateDeterminism does not use Seed flag + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h test: @go test ./... -.PHONY: all build-linux install clean build test-all test \ No newline at end of file + +# Kick start lots of sims on an AWS cluster. +# This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3 +start-remote-sims: + # build the image used for running sims in, and tag it + docker build -f simulations/Dockerfile -t kava/kava-sim:master . + # push that image to the hub + docker push kava/kava-sim:master + # submit an array job on AWS Batch, using 1000 seeds, spot instances + aws batch submit-job \ + -—job-name "master-$(VERSION)" \ + -—job-queue “simulation-1-queue-spot" \ + -—array-properties size=1000 \ + -—job-definition kava-sim-master \ + -—container-override environment=[{SIM_NAME=master-$(VERSION)}] + +.PHONY: all build-linux install clean build test test-all start-remote-sims diff --git a/app/sim_test.go b/app/sim_test.go index 298acccc..ede0aaf1 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -58,8 +58,18 @@ const ( OpWeightMsgUnjail = "op_weight_msg_unjail" ) -func init() { +// TestMain runs setup and teardown code before all tests. +func TestMain(m *testing.M) { + // set prefixes + config := sdk.GetConfig() + SetBech32AddressPrefixes(config) + config.Seal() + // load the values from simulation specific flags simapp.GetSimulatorFlags() + + // run tests + exitCode := m.Run() + os.Exit(exitCode) } func testAndRunTxs(app *App, config simulation.Config) []simulation.WeightedOperation { @@ -319,6 +329,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { } } +// TestFullAppSimulation runs a standard simulation of the app, modified by cmd line flag values. func TestFullAppSimulation(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") @@ -373,6 +384,7 @@ func TestFullAppSimulation(t *testing.T) { } } +// TestAppImportExport runs a simulation, exports the state, imports it, then checks the db state is same after import as it was before export. func TestAppImportExport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application import/export simulation") @@ -490,6 +502,7 @@ func TestAppImportExport(t *testing.T) { } } +// TestAppSimulationAfterImport runs a simulation, exports it, imports it and runs another simulation. func TestAppSimulationAfterImport(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation after import") @@ -584,6 +597,7 @@ func TestAppSimulationAfterImport(t *testing.T) { // TODO: Make another test for the fuzzer itself, which just has noOp txs // and doesn't depend on the application. +// TestAppStateDeterminism runs several sims with the same seed and checks the states are equal. func TestAppStateDeterminism(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") diff --git a/simulations/Dockerfile b/simulations/Dockerfile index ff843e68..f07db3b7 100644 --- a/simulations/Dockerfile +++ b/simulations/Dockerfile @@ -25,6 +25,7 @@ COPY app app COPY cli_test cli_test COPY cmd cmd COPY app app +COPY x x COPY Makefile . # Install kvd, kvcli diff --git a/simulations/run-then-upload.sh b/simulations/run-then-upload.sh index d862adf3..39e45997 100755 --- a/simulations/run-then-upload.sh +++ b/simulations/run-then-upload.sh @@ -7,24 +7,27 @@ # Parse Input Args +# get simulation type +simType=$1 # get seed -startingSeed=$1 +startingSeed=$2 # compute the seed from the starting and the job index # add two nums together, hence the $(()), and use 0 as the default value for array index, hence the ${:-} syntax seed=$(($startingSeed+${AWS_BATCH_JOB_ARRAY_INDEX:-0})) -echo "seed: " $seed # get sim parameters -numBlocks=$2 -blockSize=$3 +numBlocks=$3 +blockSize=$4 # Run The Sim -# redirect stdout and stderr to a file -go test ./app -run TestFullAppSimulation -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h > out.log 2>&1 +# record cli arguments in the log file (in json in case we need to parse this) and stdout (https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout) +printf "{\"simType\": \"%s\", \"startingSeed\": %s, \"seed\": %s, \"numBlocks\": %s, \"blockSize\": %s}\n" $simType $startingSeed $seed $numBlocks $blockSize | tee out.log +# run sim and redirect stdout and stderr to a file) +go test ./app -run $simType -Enabled=true -NumBlocks=$numBlocks -BlockSize=$blockSize -Commit=true -Period=5 -Seed=$seed -v -timeout 24h >> out.log 2>&1 # get the exit code to determine how to upload results simExitStatus=$? if [ $simExitStatus -eq 0 ];then - echo "simulations passed" + echo "simulation passed" simResult="pass" else echo "simulation failed" @@ -39,5 +42,6 @@ jobID=${AWS_BATCH_JOB_ID:-"testJobID:"} jobID=$(echo $jobID | sed 's/\(.*\):\d*/\1/') # create the filename from the array job index (which won't be set if this is a normal job) -fileName=out$AWS_BATCH_JOB_ARRAY_INDEX.log -aws s3 cp out.log s3://simulations-1/$jobID/$simResult/$fileName \ No newline at end of file +fileName=$(printf "out%05d.log" $AWS_BATCH_JOB_ARRAY_INDEX) + +aws s3 cp out.log s3://simulations-1/$SIM_NAME$jobID/$simResult/$fileName \ No newline at end of file