From 162a47343d3b50012294717e5a38af9d25965c4c Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Wed, 15 Apr 2020 22:37:22 -0400 Subject: [PATCH 1/5] feat: reduce determinism sim runs --- app/sim_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 37ad51a3..1d0bd12e 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -662,12 +662,12 @@ func TestAppStateDeterminism(t *testing.T) { config.OnOperation = false config.AllInvariants = false - numSeeds := 3 - numTimesToRunPerSeed := 5 + numSeeds := 1 + numTimesToRunPerSeed := 2 appHashList := make([]json.RawMessage, numTimesToRunPerSeed) for i := 0; i < numSeeds; i++ { - config.Seed = rand.Int63() + config.Seed = 8674665223082153551 for j := 0; j < numTimesToRunPerSeed; j++ { logger := log.NewNopLogger() From 3ab264b5b7064a0e8827010195c9f86c2ca743a6 Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Wed, 15 Apr 2020 22:44:08 -0400 Subject: [PATCH 2/5] fix: remove hardcoded value --- app/sim_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sim_test.go b/app/sim_test.go index 1d0bd12e..3a00315b 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -667,7 +667,7 @@ func TestAppStateDeterminism(t *testing.T) { appHashList := make([]json.RawMessage, numTimesToRunPerSeed) for i := 0; i < numSeeds; i++ { - config.Seed = 8674665223082153551 + config.Seed = rand.Int63() for j := 0; j < numTimesToRunPerSeed; j++ { logger := log.NewNopLogger() From decbbd4c00e61f32c9472a23a2d9e6bd4baa7f6c Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Thu, 16 Apr 2020 16:58:00 -0400 Subject: [PATCH 3/5] feat: only run one seed in determinism tests --- app/sim_test.go | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/app/sim_test.go b/app/sim_test.go index 3a00315b..ee0a5395 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -662,38 +662,33 @@ func TestAppStateDeterminism(t *testing.T) { config.OnOperation = false config.AllInvariants = false - numSeeds := 1 numTimesToRunPerSeed := 2 appHashList := make([]json.RawMessage, numTimesToRunPerSeed) - for i := 0; i < numSeeds; i++ { - config.Seed = rand.Int63() + for j := 0; j < numTimesToRunPerSeed; j++ { + logger := log.NewNopLogger() + db := dbm.NewMemDB() + app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) - for j := 0; j < numTimesToRunPerSeed; j++ { - logger := log.NewNopLogger() - db := dbm.NewMemDB() - app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) + fmt.Printf( + "running non-determinism simulation; seed %d: attempt: %d/%d\n", + config.Seed, j+1, numTimesToRunPerSeed, + ) - fmt.Printf( - "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", - config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, + _, _, err := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), + testAndRunTxs(app, config), app.ModuleAccountAddrs(), config, + ) + require.NoError(t, err) + + appHash := app.LastCommitID().Hash + appHashList[j] = appHash + + if j != 0 { + require.Equal( + t, appHashList[0], appHashList[j], + "non-determinism in seed %d: attempt: %d/%d\n", config.Seed, j+1, numTimesToRunPerSeed, ) - - _, _, err := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - testAndRunTxs(app, config), app.ModuleAccountAddrs(), config, - ) - require.NoError(t, err) - - appHash := app.LastCommitID().Hash - appHashList[j] = appHash - - if j != 0 { - require.Equal( - t, appHashList[0], appHashList[j], - "non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, - ) - } } } } From be5b12d9d73919d83000567d33f506cc5965fa33 Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Thu, 16 Apr 2020 16:58:26 -0400 Subject: [PATCH 4/5] add make method for faster testing and sims --- Makefile | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 11c1c996..6a4624e9 100644 --- a/Makefile +++ b/Makefile @@ -109,18 +109,26 @@ link-check: # TODO tidy up cli tests to use same -Enable flag as simulations, or the other way round # TODO -mod=readonly ? # build dependency needed for cli tests -test-all: build +test-all: build test test-cli # basic app tests @go test ./app -v - # cli tests - @go test ./cli_test -tags cli_test -v -p 4 # basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks) @go test ./app -run TestFullAppSimulation -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h # other sim tests @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @# AppStateDeterminism does not use Seed flag - @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h + +# run module tests and short simulations +test-basic: test + # basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks) + @go test ./app -run TestFullAppSimulation -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + # other sim tests + @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + @# AppStateDeterminism does not use Seed flag + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m test: @go test ./... @@ -128,6 +136,9 @@ test: test_dredd: rest_test/./run_all_tests_from_make.sh +test-cli: + @go test ./cli_test -tags cli_test -v -p 4 + # Kick start lots of sims on an AWS cluster. # This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3 start-remote-sims: @@ -143,4 +154,4 @@ start-remote-sims: -—job-definition kava-sim-master \ -—container-override environment=[{SIM_NAME=master-$(VERSION)}] -.PHONY: all build-linux install clean build test test-all start-remote-sims +.PHONY: all build-linux install clean build test test-cli test-all test_dredd test-basic start-remote-sims From 5ac44ff86e2c54c387f7b65ba73177bf5f7936ab Mon Sep 17 00:00:00 2001 From: Kevin Davis Date: Thu, 16 Apr 2020 17:05:54 -0400 Subject: [PATCH 5/5] fix: run cli tests before module tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a4624e9..ab532ba0 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ link-check: # TODO tidy up cli tests to use same -Enable flag as simulations, or the other way round # TODO -mod=readonly ? # build dependency needed for cli tests -test-all: build test test-cli +test-all: build test-cli test # basic app tests @go test ./app -v # basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks)