Make rocksdb configurable (#1658)

* Make rocksdb configurable

* Make sure rocksdb tests are running in CI

* Updating ci-rocksdb-build workflow

* Remove test.sh

* Update tm-db dependency

.github/workflows/ci-rocksdb-build.yml

@@ -1,5 +1,8 @@
 name: Continuous Integration (Rocksdb Build)
 
+env:
+  ROCKSDB_VERSION: v8.1.1
+
 on:
   workflow_call:
 jobs:
@@ -16,7 +19,29 @@ jobs:
           cache: true
       - name: build rocksdb dependency
         run: bash ${GITHUB_WORKSPACE}/.github/scripts/install-rocksdb.sh
-        env:
-          ROCKSDB_VERSION: v7.10.2
       - name: build application
         run: make build COSMOS_BUILD_OPTIONS=rocksdb
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: install RocksDB dependencies
+        run: sudo apt-get update
+          && sudo apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+      - name: install RocksDB as shared library
+        run: git clone https://github.com/facebook/rocksdb.git
+          && cd rocksdb
+          && git checkout $ROCKSDB_VERSION
+          && sudo make -j$(nproc) install-shared
+          && sudo ldconfig
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: run unit tests
+        run: make test-rocksdb

Dockerfile-rocksdb

@@ -10,7 +10,7 @@ WORKDIR /root
 # default home directory is /root
 
 # install rocksdb
-ARG rocksdb_version=v7.10.2
+ARG rocksdb_version=v8.1.1
 ENV ROCKSDB_VERSION=$rocksdb_version
 
 RUN git clone https://github.com/facebook/rocksdb.git \

Makefile

@@ -301,6 +301,9 @@ test-e2e: docker-build
 test:
 	@$(GO_BIN) test $$($(GO_BIN) list ./... | grep -v 'contrib' | grep -v 'tests/e2e')
 
+test-rocksdb:
+	@go test -tags=rocksdb ./cmd/kava/opendb
+
 # Run cli integration tests
 # `-p 4` to use 4 cores, `-tags cli_test` to tell $(GO_BIN) not to ignore the cli package
 # These tests use the `kvd` or `kvcli` binaries in the build dir, or in `$BUILDDIR` if that env var is set.

cmd/kava/cmd/root.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kava-labs/kava/app"
 	"github.com/kava-labs/kava/app/params"
 	kavaclient "github.com/kava-labs/kava/client"
+	"github.com/kava-labs/kava/cmd/kava/opendb"
 )
 
 // EnvPrefix is the prefix environment variables must have to configure the app.
@@ -105,13 +106,15 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de
 		encodingConfig: encodingConfig,
 	}
 
+	opts := ethermintserver.StartOptions{
+		AppCreator:      ac.newApp,
+		DefaultNodeHome: app.DefaultNodeHome,
+		DBOpener:        opendb.OpenDB,
+	}
 	// ethermintserver adds additional flags to start the JSON-RPC server for evm support
 	ethermintserver.AddCommands(
 		rootCmd,
-		ethermintserver.NewDefaultStartOptions(
-			ac.newApp,
-			app.DefaultNodeHome,
-		),
+		opts,
 		ac.appExport,
 		ac.addStartCmdFlags,
 	)

cmd/kava/opendb/opendb.go

@@ -0,0 +1,18 @@
+//go:build !rocksdb
+// +build !rocksdb
+
+package opendb
+
+import (
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+// OpenDB is a copy of default DBOpener function used by ethermint, see for details:
+// https://github.com/evmos/ethermint/blob/07cf2bd2b1ce9bdb2e44ec42a39e7239292a14af/server/start.go#L647
+func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	return dbm.NewDB("application", backendType, dataDir)
+}

cmd/kava/opendb/opendb_rocksdb.go

@@ -0,0 +1,170 @@
+//go:build rocksdb
+// +build rocksdb
+
+// Copyright 2023 Kava Labs, Inc.
+// Copyright 2023 Cronos Labs, Inc.
+//
+// Derived from https://github.com/crypto-org-chain/cronos@496ce7e
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package opendb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/linxGnu/grocksdb"
+	"github.com/spf13/cast"
+	dbm "github.com/tendermint/tm-db"
+)
+
+var ErrUnexpectedConfiguration = errors.New("unexpected rocksdb configuration, rocksdb should have only one column family named default")
+
+const (
+	// default tm-db block cache size for RocksDB
+	blockCacheSize = 1 << 30
+
+	defaultColumnFamilyName = "default"
+
+	maxOpenFilesDBOptName          = "max_open_files"
+	maxFileOpeningThreadsDBOptName = "max_file_opening_threads"
+
+	writeBufferSizeCFOptName = "write_buffer_size"
+	numLevelsCFOptName       = "num_levels"
+)
+
+func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	if backendType == dbm.RocksDBBackend {
+		return openRocksdb(filepath.Join(dataDir, "application.db"), appOpts)
+	}
+
+	return dbm.NewDB("application", backendType, dataDir)
+}
+
+// openRocksdb loads existing options, overrides some of them with appOpts and opens database
+// option will be overridden only in case if it explicitly specified in appOpts
+func openRocksdb(dir string, appOpts types.AppOptions) (dbm.DB, error) {
+	dbOpts, cfOpts, err := loadLatestOptions(dir)
+	if err != nil {
+		return nil, err
+	}
+	// customize rocksdb options
+	dbOpts = overrideDBOpts(dbOpts, appOpts)
+	cfOpts = overrideCFOpts(cfOpts, appOpts)
+
+	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts)
+}
+
+// loadLatestOptions loads and returns database and column family options
+// if options file not found, it means database isn't created yet, in such case default tm-db options will be returned
+// if database exists it should have only one column family named default
+func loadLatestOptions(dir string) (*grocksdb.Options, *grocksdb.Options, error) {
+	latestOpts, err := grocksdb.LoadLatestOptions(dir, grocksdb.NewDefaultEnv(), true, grocksdb.NewLRUCache(blockCacheSize))
+	if err != nil && strings.HasPrefix(err.Error(), "NotFound: ") {
+		return newDefaultOptions(), newDefaultOptions(), nil
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfNames := latestOpts.ColumnFamilyNames()
+	cfOpts := latestOpts.ColumnFamilyOpts()
+	// db should have only one column family named default
+	ok := len(cfNames) == 1 && cfNames[0] == defaultColumnFamilyName
+	if !ok {
+		return nil, nil, ErrUnexpectedConfiguration
+	}
+
+	// return db and cf opts
+	return latestOpts.Options(), &cfOpts[0], nil
+}
+
+// overrideDBOpts merges dbOpts and appOpts, appOpts takes precedence
+func overrideDBOpts(dbOpts *grocksdb.Options, appOpts types.AppOptions) *grocksdb.Options {
+	maxOpenFiles := appOpts.Get(maxOpenFilesDBOptName)
+	if maxOpenFiles != nil {
+		dbOpts.SetMaxOpenFiles(cast.ToInt(maxOpenFiles))
+	}
+
+	maxFileOpeningThreads := appOpts.Get(maxFileOpeningThreadsDBOptName)
+	if maxFileOpeningThreads != nil {
+		dbOpts.SetMaxFileOpeningThreads(cast.ToInt(maxFileOpeningThreads))
+	}
+
+	return dbOpts
+}
+
+// overrideCFOpts merges cfOpts and appOpts, appOpts takes precedence
+func overrideCFOpts(cfOpts *grocksdb.Options, appOpts types.AppOptions) *grocksdb.Options {
+	writeBufferSize := appOpts.Get(writeBufferSizeCFOptName)
+	if writeBufferSize != nil {
+		cfOpts.SetWriteBufferSize(cast.ToUint64(writeBufferSize))
+	}
+
+	numLevels := appOpts.Get(numLevelsCFOptName)
+	if numLevels != nil {
+		cfOpts.SetNumLevels(cast.ToInt(numLevels))
+	}
+
+	return cfOpts
+}
+
+// newRocksDBWithOptions opens rocksdb with provided database and column family options
+// newRocksDBWithOptions expects that db has only one column family named default
+func newRocksDBWithOptions(name string, dir string, dbOpts, cfOpts *grocksdb.Options) (*dbm.RocksDB, error) {
+	dbPath := filepath.Join(dir, name+".db")
+
+	// Ensure path exists
+	if err := os.MkdirAll(dbPath, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create db path: %w", err)
+	}
+
+	db, _, err := grocksdb.OpenDbColumnFamilies(dbOpts, dbPath, []string{defaultColumnFamilyName}, []*grocksdb.Options{cfOpts})
+	if err != nil {
+		return nil, err
+	}
+	ro := grocksdb.NewDefaultReadOptions()
+	wo := grocksdb.NewDefaultWriteOptions()
+	woSync := grocksdb.NewDefaultWriteOptions()
+	woSync.SetSync(true)
+	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+}
+
+// newDefaultOptions returns default tm-db options for RocksDB, see for details:
+// https://github.com/Kava-Labs/tm-db/blob/94ff76d31724965f8883cddebabe91e0d01bc03f/rocksdb.go#L30
+func newDefaultOptions() *grocksdb.Options {
+	// default rocksdb option, good enough for most cases, including heavy workloads.
+	// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
+	// compression: snappy as default, need to -lsnappy to enable.
+	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
+	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
+	bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10))
+
+	opts := grocksdb.NewDefaultOptions()
+	opts.SetBlockBasedTableFactory(bbto)
+	// SetMaxOpenFiles to 4096 seems to provide a reliable performance boost
+	opts.SetMaxOpenFiles(4096)
+	opts.SetCreateIfMissing(true)
+	opts.IncreaseParallelism(runtime.NumCPU())
+	// 1.5GB maximum memory use for writebuffer.
+	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
+
+	return opts
+}

cmd/kava/opendb/opendb_rocksdb_test.go

@@ -0,0 +1,357 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/linxGnu/grocksdb"
+	"github.com/stretchr/testify/require"
+)
+
+type mockAppOptions struct {
+	opts map[string]interface{}
+}
+
+func newMockAppOptions(opts map[string]interface{}) *mockAppOptions {
+	return &mockAppOptions{
+		opts: opts,
+	}
+}
+
+func (m *mockAppOptions) Get(key string) interface{} {
+	return m.opts[key]
+}
+
+func TestOpenRocksdb(t *testing.T) {
+	t.Run("db already exists", func(t *testing.T) {
+		defaultOpts := newDefaultOptions()
+
+		for _, tc := range []struct {
+			desc                  string
+			mockAppOptions        *mockAppOptions
+			maxOpenFiles          int
+			maxFileOpeningThreads int
+			writeBufferSize       uint64
+			numLevels             int
+		}{
+			{
+				desc:                  "default options",
+				mockAppOptions:        newMockAppOptions(map[string]interface{}{}),
+				maxOpenFiles:          defaultOpts.GetMaxOpenFiles(),
+				maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+				writeBufferSize:       defaultOpts.GetWriteBufferSize(),
+				numLevels:             defaultOpts.GetNumLevels(),
+			},
+			{
+				desc: "change 2 options",
+				mockAppOptions: newMockAppOptions(map[string]interface{}{
+					maxOpenFilesDBOptName:    999,
+					writeBufferSizeCFOptName: 999_999,
+				}),
+				maxOpenFiles:          999,
+				maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+				writeBufferSize:       999_999,
+				numLevels:             defaultOpts.GetNumLevels(),
+			},
+			{
+				desc: "change 4 options",
+				mockAppOptions: newMockAppOptions(map[string]interface{}{
+					maxOpenFilesDBOptName:          999,
+					maxFileOpeningThreadsDBOptName: 9,
+					writeBufferSizeCFOptName:       999_999,
+					numLevelsCFOptName:             9,
+				}),
+				maxOpenFiles:          999,
+				maxFileOpeningThreads: 9,
+				writeBufferSize:       999_999,
+				numLevels:             9,
+			},
+		} {
+			t.Run(tc.desc, func(t *testing.T) {
+				dir, err := os.MkdirTemp("", "rocksdb")
+				require.NoError(t, err)
+				defer func() {
+					err := os.RemoveAll(dir)
+					require.NoError(t, err)
+				}()
+
+				db, err := openRocksdb(dir, tc.mockAppOptions)
+				require.NoError(t, err)
+				require.NoError(t, db.Close())
+
+				dbOpts, cfOpts, err := loadLatestOptions(filepath.Join(dir, "application.db"))
+				require.NoError(t, err)
+				require.Equal(t, tc.maxOpenFiles, dbOpts.GetMaxOpenFiles())
+				require.Equal(t, tc.maxFileOpeningThreads, dbOpts.GetMaxFileOpeningThreads())
+				require.Equal(t, tc.writeBufferSize, cfOpts.GetWriteBufferSize())
+				require.Equal(t, tc.numLevels, cfOpts.GetNumLevels())
+			})
+		}
+	})
+
+	t.Run("db doesn't exist yet", func(t *testing.T) {
+		defaultOpts := newDefaultOptions()
+
+		dir, err := os.MkdirTemp("", "rocksdb")
+		require.NoError(t, err)
+		defer func() {
+			err := os.RemoveAll(dir)
+			require.NoError(t, err)
+		}()
+
+		mockAppOpts := newMockAppOptions(map[string]interface{}{})
+		db, err := openRocksdb(dir, mockAppOpts)
+		require.NoError(t, err)
+		require.NoError(t, db.Close())
+
+		dbOpts, cfOpts, err := loadLatestOptions(filepath.Join(dir, "application.db"))
+		require.NoError(t, err)
+		require.Equal(t, defaultOpts.GetMaxOpenFiles(), dbOpts.GetMaxOpenFiles())
+		require.Equal(t, defaultOpts.GetMaxFileOpeningThreads(), dbOpts.GetMaxFileOpeningThreads())
+		require.Equal(t, defaultOpts.GetWriteBufferSize(), cfOpts.GetWriteBufferSize())
+		require.Equal(t, defaultOpts.GetNumLevels(), cfOpts.GetNumLevels())
+	})
+}
+
+func TestLoadLatestOptions(t *testing.T) {
+	t.Run("db already exists", func(t *testing.T) {
+		defaultOpts := newDefaultOptions()
+
+		const testCasesNum = 3
+		dbOptsList := make([]*grocksdb.Options, testCasesNum)
+		cfOptsList := make([]*grocksdb.Options, testCasesNum)
+
+		dbOptsList[0] = newDefaultOptions()
+		cfOptsList[0] = newDefaultOptions()
+
+		dbOptsList[1] = newDefaultOptions()
+		dbOptsList[1].SetMaxOpenFiles(999)
+		cfOptsList[1] = newDefaultOptions()
+		cfOptsList[1].SetWriteBufferSize(999_999)
+
+		dbOptsList[2] = newDefaultOptions()
+		dbOptsList[2].SetMaxOpenFiles(999)
+		dbOptsList[2].SetMaxFileOpeningThreads(9)
+		cfOptsList[2] = newDefaultOptions()
+		cfOptsList[2].SetWriteBufferSize(999_999)
+		cfOptsList[2].SetNumLevels(9)
+
+		for _, tc := range []struct {
+			desc                  string
+			dbOpts                *grocksdb.Options
+			cfOpts                *grocksdb.Options
+			maxOpenFiles          int
+			maxFileOpeningThreads int
+			writeBufferSize       uint64
+			numLevels             int
+		}{
+			{
+				desc:                  "default options",
+				dbOpts:                dbOptsList[0],
+				cfOpts:                cfOptsList[0],
+				maxOpenFiles:          defaultOpts.GetMaxOpenFiles(),
+				maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+				writeBufferSize:       defaultOpts.GetWriteBufferSize(),
+				numLevels:             defaultOpts.GetNumLevels(),
+			},
+			{
+				desc:                  "change 2 options",
+				dbOpts:                dbOptsList[1],
+				cfOpts:                cfOptsList[1],
+				maxOpenFiles:          999,
+				maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+				writeBufferSize:       999_999,
+				numLevels:             defaultOpts.GetNumLevels(),
+			},
+			{
+				desc:                  "change 4 options",
+				dbOpts:                dbOptsList[2],
+				cfOpts:                cfOptsList[2],
+				maxOpenFiles:          999,
+				maxFileOpeningThreads: 9,
+				writeBufferSize:       999_999,
+				numLevels:             9,
+			},
+		} {
+			t.Run(tc.desc, func(t *testing.T) {
+				name := "application"
+				dir, err := os.MkdirTemp("", "rocksdb")
+				require.NoError(t, err)
+				defer func() {
+					err := os.RemoveAll(dir)
+					require.NoError(t, err)
+				}()
+
+				db, err := newRocksDBWithOptions(name, dir, tc.dbOpts, tc.cfOpts)
+				require.NoError(t, err)
+				require.NoError(t, db.Close())
+
+				dbOpts, cfOpts, err := loadLatestOptions(filepath.Join(dir, "application.db"))
+				require.NoError(t, err)
+				require.Equal(t, tc.maxOpenFiles, dbOpts.GetMaxOpenFiles())
+				require.Equal(t, tc.maxFileOpeningThreads, dbOpts.GetMaxFileOpeningThreads())
+				require.Equal(t, tc.writeBufferSize, cfOpts.GetWriteBufferSize())
+				require.Equal(t, tc.numLevels, cfOpts.GetNumLevels())
+			})
+		}
+	})
+
+	t.Run("db doesn't exist yet", func(t *testing.T) {
+		defaultOpts := newDefaultOptions()
+
+		dir, err := os.MkdirTemp("", "rocksdb")
+		require.NoError(t, err)
+		defer func() {
+			err := os.RemoveAll(dir)
+			require.NoError(t, err)
+		}()
+
+		dbOpts, cfOpts, err := loadLatestOptions(filepath.Join(dir, "application.db"))
+		require.NoError(t, err)
+		require.Equal(t, defaultOpts.GetMaxOpenFiles(), dbOpts.GetMaxOpenFiles())
+		require.Equal(t, defaultOpts.GetMaxFileOpeningThreads(), dbOpts.GetMaxFileOpeningThreads())
+		require.Equal(t, defaultOpts.GetWriteBufferSize(), cfOpts.GetWriteBufferSize())
+		require.Equal(t, defaultOpts.GetNumLevels(), cfOpts.GetNumLevels())
+	})
+}
+
+func TestOverrideDBOpts(t *testing.T) {
+	defaultOpts := newDefaultOptions()
+
+	for _, tc := range []struct {
+		desc                  string
+		mockAppOptions        *mockAppOptions
+		maxOpenFiles          int
+		maxFileOpeningThreads int
+	}{
+		{
+			desc:                  "override nothing",
+			mockAppOptions:        newMockAppOptions(map[string]interface{}{}),
+			maxOpenFiles:          defaultOpts.GetMaxOpenFiles(),
+			maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+		},
+		{
+			desc: "override max-open-files",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				maxOpenFilesDBOptName: 999,
+			}),
+			maxOpenFiles:          999,
+			maxFileOpeningThreads: defaultOpts.GetMaxFileOpeningThreads(),
+		},
+		{
+			desc: "override max-file-opening-threads",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				maxFileOpeningThreadsDBOptName: 9,
+			}),
+			maxOpenFiles:          defaultOpts.GetMaxOpenFiles(),
+			maxFileOpeningThreads: 9,
+		},
+		{
+			desc: "override max-open-files and max-file-opening-threads",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				maxOpenFilesDBOptName:          999,
+				maxFileOpeningThreadsDBOptName: 9,
+			}),
+			maxOpenFiles:          999,
+			maxFileOpeningThreads: 9,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			dbOpts := newDefaultOptions()
+			dbOpts = overrideDBOpts(dbOpts, tc.mockAppOptions)
+
+			require.Equal(t, tc.maxOpenFiles, dbOpts.GetMaxOpenFiles())
+			require.Equal(t, tc.maxFileOpeningThreads, dbOpts.GetMaxFileOpeningThreads())
+		})
+	}
+}
+
+func TestOverrideCFOpts(t *testing.T) {
+	defaultOpts := newDefaultOptions()
+
+	for _, tc := range []struct {
+		desc            string
+		mockAppOptions  *mockAppOptions
+		writeBufferSize uint64
+		numLevels       int
+	}{
+		{
+			desc:            "override nothing",
+			mockAppOptions:  newMockAppOptions(map[string]interface{}{}),
+			writeBufferSize: defaultOpts.GetWriteBufferSize(),
+			numLevels:       defaultOpts.GetNumLevels(),
+		},
+		{
+			desc: "override write-buffer-size",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				writeBufferSizeCFOptName: 999_999,
+			}),
+			writeBufferSize: 999_999,
+			numLevels:       defaultOpts.GetNumLevels(),
+		},
+		{
+			desc: "override num-levels",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				numLevelsCFOptName: 9,
+			}),
+			writeBufferSize: defaultOpts.GetWriteBufferSize(),
+			numLevels:       9,
+		},
+		{
+			desc: "override write-buffer-size and num-levels",
+			mockAppOptions: newMockAppOptions(map[string]interface{}{
+				writeBufferSizeCFOptName: 999_999,
+				numLevelsCFOptName:       9,
+			}),
+			writeBufferSize: 999_999,
+			numLevels:       9,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			cfOpts := newDefaultOptions()
+			cfOpts = overrideCFOpts(cfOpts, tc.mockAppOptions)
+
+			require.Equal(t, tc.writeBufferSize, cfOpts.GetWriteBufferSize())
+			require.Equal(t, tc.numLevels, cfOpts.GetNumLevels())
+		})
+	}
+}
+
+func TestNewRocksDBWithOptions(t *testing.T) {
+	defaultOpts := newDefaultOptions()
+
+	name := "application"
+	dir, err := os.MkdirTemp("", "rocksdb")
+	require.NoError(t, err)
+	defer func() {
+		err := os.RemoveAll(dir)
+		require.NoError(t, err)
+	}()
+
+	dbOpts := newDefaultOptions()
+	dbOpts.SetMaxOpenFiles(999)
+	cfOpts := newDefaultOptions()
+	cfOpts.SetWriteBufferSize(999_999)
+
+	db, err := newRocksDBWithOptions(name, dir, dbOpts, cfOpts)
+	require.NoError(t, err)
+	require.NoError(t, db.Close())
+
+	dbOpts, cfOpts, err = loadLatestOptions(filepath.Join(dir, "application.db"))
+	require.NoError(t, err)
+	require.Equal(t, 999, dbOpts.GetMaxOpenFiles())
+	require.Equal(t, defaultOpts.GetMaxFileOpeningThreads(), dbOpts.GetMaxFileOpeningThreads())
+	require.Equal(t, uint64(999_999), cfOpts.GetWriteBufferSize())
+	require.Equal(t, defaultOpts.GetNumLevels(), dbOpts.GetNumLevels())
+}
+
+func TestNewDefaultOptions(t *testing.T) {
+	defaultOpts := newDefaultOptions()
+
+	maxOpenFiles := defaultOpts.GetMaxOpenFiles()
+	require.Equal(t, 4096, maxOpenFiles)
+}

go.mod

@@ -16,11 +16,12 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/linxGnu/grocksdb v1.8.0
 	github.com/pelletier/go-toml/v2 v2.0.6
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.15.0
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.3
 	github.com/subosito/gotenv v1.4.2
 	github.com/tendermint/tendermint v0.34.27
 	github.com/tendermint/tm-db v0.6.7
@@ -129,7 +130,6 @@ require (
 	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/lib/pq v1.10.7 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
-	github.com/linxGnu/grocksdb v1.7.15 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -216,5 +216,5 @@ replace (
 	// Use cometbft fork of tendermint
 	github.com/tendermint/tendermint => github.com/kava-labs/cometbft v0.34.27-kava.0
 	// Indirect dependencies still use tendermint/tm-db
-	github.com/tendermint/tm-db => github.com/kava-labs/tm-db v0.6.7-kava.3
+	github.com/tendermint/tm-db => github.com/kava-labs/tm-db v0.6.7-kava.4
 )

go.sum

@@ -808,8 +808,8 @@ github.com/kava-labs/cosmos-sdk v0.46.11-kava.1 h1:3VRpm4zf/gQgmpRVd1p99/2P8ZecA
 github.com/kava-labs/cosmos-sdk v0.46.11-kava.1/go.mod h1:bG4AkW9bqc8ycrryyKGQEl3YV9BY2wr6HggGq8kvcgM=
 github.com/kava-labs/ethermint v0.21.0-kava-v23-1 h1:5TSyCtPvFdMuSe8p2iMVqXmFBlK3lHyjaT9EqN752aI=
 github.com/kava-labs/ethermint v0.21.0-kava-v23-1/go.mod h1:rdm6AinxZ4dzPEv/cjH+/AGyTbKufJ3RE7M2MDyklH0=
-github.com/kava-labs/tm-db v0.6.7-kava.3 h1:4vyAh+NyZ1xTjCt0utNT6FJHnsZK1I19xwZeJttdRXQ=
-github.com/kava-labs/tm-db v0.6.7-kava.3/go.mod h1:70tpLhNfwCP64nAlq+bU+rOiVfWr3Nnju1D1nhGDGKs=
+github.com/kava-labs/tm-db v0.6.7-kava.4 h1:M2RibOKmbi+k2OhAFry8z9+RJF0CYuDETB7/PrSdoro=
+github.com/kava-labs/tm-db v0.6.7-kava.4/go.mod h1:70tpLhNfwCP64nAlq+bU+rOiVfWr3Nnju1D1nhGDGKs=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
@@ -849,8 +849,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.7.15 h1:AEhP28lkeAybv5UYNYviYISpR6bJejEnKuYbnWAnxx0=
-github.com/linxGnu/grocksdb v1.7.15/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
+github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
+github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -1130,8 +1130,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=