Rocksdb Metrics (#1692)

* Rocksdb Metrics

* Add rocksdb namespace for options

* Adding help to the metrics

* CR's fixes

* CR's fixes

* CR's fixes

cmd/kava/opendb/metrics.go

@@ -0,0 +1,166 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+// rocksdbMetrics will be initialized in registerMetrics() if enableRocksdbMetrics flag set to true
+var rocksdbMetrics *Metrics
+
+// Metrics contains all rocksdb metrics which will be reported to prometheus
+type Metrics struct {
+	// Keys
+	NumberKeysWritten metrics.Gauge
+	NumberKeysRead    metrics.Gauge
+	NumberKeysUpdated metrics.Gauge
+	EstimateNumKeys   metrics.Gauge
+
+	// Files
+	NumberFileOpens  metrics.Gauge
+	NumberFileErrors metrics.Gauge
+
+	// Memory
+	BlockCacheUsage         metrics.Gauge
+	EstimateTableReadersMem metrics.Gauge
+	CurSizeAllMemTables     metrics.Gauge
+	BlockCachePinnedUsage   metrics.Gauge
+
+	// Cache
+	BlockCacheMiss        metrics.Gauge
+	BlockCacheHit         metrics.Gauge
+	BlockCacheAdd         metrics.Gauge
+	BlockCacheAddFailures metrics.Gauge
+}
+
+// registerMetrics registers metrics in prometheus and initializes rocksdbMetrics variable
+func registerMetrics() {
+	if rocksdbMetrics != nil {
+		// metrics already registered
+		return
+	}
+
+	labels := make([]string, 0)
+	rocksdbMetrics = &Metrics{
+		// Keys
+		NumberKeysWritten: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_written",
+			Help:      "",
+		}, labels),
+		NumberKeysRead: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_read",
+			Help:      "",
+		}, labels),
+		NumberKeysUpdated: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_updated",
+			Help:      "",
+		}, labels),
+		EstimateNumKeys: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "estimate_num_keys",
+			Help:      "estimated number of total keys in the active and unflushed immutable memtables and storage",
+		}, labels),
+
+		// Files
+		NumberFileOpens: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_opens",
+			Help:      "",
+		}, labels),
+		NumberFileErrors: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_errors",
+			Help:      "",
+		}, labels),
+
+		// Memory
+		BlockCacheUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_usage",
+			Help:      "memory size for the entries residing in block cache",
+		}, labels),
+		EstimateTableReadersMem: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "estimate_table_readers_mem",
+			Help:      "estimated memory used for reading SST tables, excluding memory used in block cache (e.g., filter and index blocks)",
+		}, labels),
+		CurSizeAllMemTables: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "cur_size_all_mem_tables",
+			Help:      "approximate size of active and unflushed immutable memtables (bytes)",
+		}, labels),
+		BlockCachePinnedUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_pinned_usage",
+			Help:      "returns the memory size for the entries being pinned",
+		}, labels),
+
+		// Cache
+		BlockCacheMiss: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_miss",
+			Help:      "block_cache_miss == block_cache_index_miss + block_cache_filter_miss + block_cache_data_miss",
+		}, labels),
+		BlockCacheHit: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_hit",
+			Help:      "block_cache_hit == block_cache_index_hit + block_cache_filter_hit + block_cache_data_hit",
+		}, labels),
+		BlockCacheAdd: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add",
+			Help:      "number of blocks added to block cache",
+		}, labels),
+		BlockCacheAddFailures: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add_failures",
+			Help:      "number of failures when adding blocks to block cache",
+		}, labels),
+	}
+}
+
+// report reports metrics to prometheus based on rocksdb props and stats
+func (m *Metrics) report(props *properties, stats *stats) {
+	// Keys
+	m.NumberKeysWritten.Set(float64(stats.NumberKeysWritten))
+	m.NumberKeysRead.Set(float64(stats.NumberKeysRead))
+	m.NumberKeysUpdated.Set(float64(stats.NumberKeysUpdated))
+	m.EstimateNumKeys.Set(float64(props.EstimateNumKeys))
+
+	// Files
+	m.NumberFileOpens.Set(float64(stats.NumberFileOpens))
+	m.NumberFileErrors.Set(float64(stats.NumberFileErrors))
+
+	// Memory
+	m.BlockCacheUsage.Set(float64(props.BlockCacheUsage))
+	m.EstimateTableReadersMem.Set(float64(props.EstimateTableReadersMem))
+	m.CurSizeAllMemTables.Set(float64(props.CurSizeAllMemTables))
+	m.BlockCachePinnedUsage.Set(float64(props.BlockCachePinnedUsage))
+
+	// Cache
+	m.BlockCacheMiss.Set(float64(stats.BlockCacheMiss))
+	m.BlockCacheHit.Set(float64(stats.BlockCacheHit))
+	m.BlockCacheAdd.Set(float64(stats.BlockCacheAdd))
+	m.BlockCacheAddFailures.Set(float64(stats.BlockCacheAddFailures))
+}

cmd/kava/opendb/opendb_rocksdb.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/cosmos/cosmos-sdk/server/types"
 	"github.com/linxGnu/grocksdb"
@@ -42,11 +43,15 @@ const (
 
 	defaultColumnFamilyName = "default"
 
-	maxOpenFilesDBOptName          = "max_open_files"
+	enableMetricsOptName             = "rocksdb.enable-metrics"
-	maxFileOpeningThreadsDBOptName = "max_file_opening_threads"
+	reportMetricsIntervalSecsOptName = "rocksdb.report-metrics-interval-secs"
+	defaultReportMetricsIntervalSecs = 15
 
-	writeBufferSizeCFOptName = "write_buffer_size"
+	maxOpenFilesDBOptName          = "rocksdb.max-open-files"
-	numLevelsCFOptName       = "num_levels"
+	maxFileOpeningThreadsDBOptName = "rocksdb.max-file-opening-threads"
+
+	writeBufferSizeCFOptName = "rocksdb.write-buffer-size"
+	numLevelsCFOptName       = "rocksdb.num-levels"
 )
 
 func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
@@ -69,7 +74,13 @@ func openRocksdb(dir string, appOpts types.AppOptions) (dbm.DB, error) {
 	dbOpts = overrideDBOpts(dbOpts, appOpts)
 	cfOpts = overrideCFOpts(cfOpts, appOpts)
 
-	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts)
+	enableMetrics := cast.ToBool(appOpts.Get(enableMetricsOptName))
+	reportMetricsIntervalSecs := cast.ToInt64(appOpts.Get(reportMetricsIntervalSecsOptName))
+	if reportMetricsIntervalSecs == 0 {
+		reportMetricsIntervalSecs = defaultReportMetricsIntervalSecs
+	}
+
+	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts, enableMetrics, reportMetricsIntervalSecs)
 }
 
 // loadLatestOptions loads and returns database and column family options
@@ -128,7 +139,14 @@ func overrideCFOpts(cfOpts *grocksdb.Options, appOpts types.AppOptions) *grocksd
 
 // newRocksDBWithOptions opens rocksdb with provided database and column family options
 // newRocksDBWithOptions expects that db has only one column family named default
-func newRocksDBWithOptions(name string, dir string, dbOpts, cfOpts *grocksdb.Options) (*dbm.RocksDB, error) {
+func newRocksDBWithOptions(
+	name string,
+	dir string,
+	dbOpts *grocksdb.Options,
+	cfOpts *grocksdb.Options,
+	enableMetrics bool,
+	reportMetricsIntervalSecs int64,
+) (*dbm.RocksDB, error) {
 	dbPath := filepath.Join(dir, name+".db")
 
 	// Ensure path exists
@@ -136,10 +154,21 @@ func newRocksDBWithOptions(name string, dir string, dbOpts, cfOpts *grocksdb.Opt
 		return nil, fmt.Errorf("failed to create db path: %w", err)
 	}
 
+	// EnableStatistics adds overhead so shouldn't be enabled in production
+	if enableMetrics {
+		dbOpts.EnableStatistics()
+	}
+
 	db, _, err := grocksdb.OpenDbColumnFamilies(dbOpts, dbPath, []string{defaultColumnFamilyName}, []*grocksdb.Options{cfOpts})
 	if err != nil {
 		return nil, err
 	}
+
+	if enableMetrics {
+		registerMetrics()
+		go reportMetrics(db, time.Second*time.Duration(reportMetricsIntervalSecs))
+	}
+
 	ro := grocksdb.NewDefaultReadOptions()
 	wo := grocksdb.NewDefaultWriteOptions()
 	woSync := grocksdb.NewDefaultWriteOptions()
@@ -168,3 +197,42 @@ func newDefaultOptions() *grocksdb.Options {
 
 	return opts
 }
+
+// reportMetrics periodically requests stats from rocksdb and reports to prometheus
+// NOTE: should be launched as a goroutine
+func reportMetrics(db *grocksdb.DB, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for {
+		select {
+		case <-ticker.C:
+			props, stats, err := getPropsAndStats(db)
+			if err != nil {
+				continue
+			}
+
+			rocksdbMetrics.report(props, stats)
+		}
+	}
+}
+
+// getPropsAndStats gets statistics from rocksdb
+func getPropsAndStats(db *grocksdb.DB) (*properties, *stats, error) {
+	propsLoader := newPropsLoader(db)
+	props, err := propsLoader.load()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	statMap, err := parseSerializedStats(props.OptionsStatistics)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	statLoader := newStatLoader(statMap)
+	stats, err := statLoader.load()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return props, stats, nil
+}

cmd/kava/opendb/opendb_rocksdb_test.go

@@ -186,7 +186,7 @@ func TestLoadLatestOptions(t *testing.T) {
 					require.NoError(t, err)
 				}()
 
-				db, err := newRocksDBWithOptions(name, dir, tc.dbOpts, tc.cfOpts)
+				db, err := newRocksDBWithOptions(name, dir, tc.dbOpts, tc.cfOpts, true, defaultReportMetricsIntervalSecs)
 				require.NoError(t, err)
 				require.NoError(t, db.Close())
 
@@ -337,7 +337,7 @@ func TestNewRocksDBWithOptions(t *testing.T) {
 	cfOpts := newDefaultOptions()
 	cfOpts.SetWriteBufferSize(999_999)
 
-	db, err := newRocksDBWithOptions(name, dir, dbOpts, cfOpts)
+	db, err := newRocksDBWithOptions(name, dir, dbOpts, cfOpts, true, defaultReportMetricsIntervalSecs)
 	require.NoError(t, err)
 	require.NoError(t, db.Close())
 

cmd/kava/opendb/props_loader.go

@@ -0,0 +1,87 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"fmt"
+	"strings"
+
+	"errors"
+)
+
+type propsGetter interface {
+	GetProperty(propName string) (value string)
+	GetIntProperty(propName string) (value uint64, success bool)
+}
+
+type propsLoader struct {
+	db        propsGetter
+	errorMsgs []string
+}
+
+func newPropsLoader(db propsGetter) *propsLoader {
+	return &propsLoader{
+		db:        db,
+		errorMsgs: make([]string, 0),
+	}
+}
+
+func (l *propsLoader) load() (*properties, error) {
+	props := &properties{
+		BaseLevel:               l.getIntProperty("rocksdb.base-level"),
+		BlockCacheCapacity:      l.getIntProperty("rocksdb.block-cache-capacity"),
+		BlockCachePinnedUsage:   l.getIntProperty("rocksdb.block-cache-pinned-usage"),
+		BlockCacheUsage:         l.getIntProperty("rocksdb.block-cache-usage"),
+		CurSizeActiveMemTable:   l.getIntProperty("rocksdb.cur-size-active-mem-table"),
+		CurSizeAllMemTables:     l.getIntProperty("rocksdb.cur-size-all-mem-tables"),
+		EstimateLiveDataSize:    l.getIntProperty("rocksdb.estimate-live-data-size"),
+		EstimateNumKeys:         l.getIntProperty("rocksdb.estimate-num-keys"),
+		EstimateTableReadersMem: l.getIntProperty("rocksdb.estimate-table-readers-mem"),
+		LiveSSTFilesSize:        l.getIntProperty("rocksdb.live-sst-files-size"),
+		SizeAllMemTables:        l.getIntProperty("rocksdb.size-all-mem-tables"),
+		OptionsStatistics:       l.getProperty("rocksdb.options-statistics"),
+	}
+
+	if len(l.errorMsgs) != 0 {
+		errorMsg := strings.Join(l.errorMsgs, ";")
+		return nil, errors.New(errorMsg)
+	}
+
+	return props, nil
+}
+
+func (l *propsLoader) getProperty(propName string) string {
+	value := l.db.GetProperty(propName)
+	if value == "" {
+		l.errorMsgs = append(l.errorMsgs, fmt.Sprintf("property %v is empty", propName))
+		return ""
+	}
+
+	return value
+}
+
+func (l *propsLoader) getIntProperty(propName string) uint64 {
+	value, ok := l.db.GetIntProperty(propName)
+	if !ok {
+		l.errorMsgs = append(l.errorMsgs, fmt.Sprintf("can't get %v int property", propName))
+		return 0
+	}
+
+	return value
+}
+
+type properties struct {
+	BaseLevel               uint64
+	BlockCacheCapacity      uint64
+	BlockCachePinnedUsage   uint64
+	BlockCacheUsage         uint64
+	CurSizeActiveMemTable   uint64
+	CurSizeAllMemTables     uint64
+	EstimateLiveDataSize    uint64
+	EstimateNumKeys         uint64
+	EstimateTableReadersMem uint64
+	LiveSSTFilesSize        uint64
+	SizeAllMemTables        uint64
+	OptionsStatistics       string
+}

cmd/kava/opendb/props_loader_test.go

@@ -0,0 +1,112 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type mockPropsGetter struct {
+	props    map[string]string
+	intProps map[string]uint64
+}
+
+func newMockPropsGetter(
+	props map[string]string,
+	intProps map[string]uint64,
+) *mockPropsGetter {
+	return &mockPropsGetter{
+		props:    props,
+		intProps: intProps,
+	}
+}
+
+func (m *mockPropsGetter) GetProperty(propName string) string {
+	return m.props[propName]
+}
+
+func (m *mockPropsGetter) GetIntProperty(propName string) (uint64, bool) {
+	prop, ok := m.intProps[propName]
+	return prop, ok
+}
+
+func TestPropsLoader(t *testing.T) {
+	defaultProps := map[string]string{
+		"rocksdb.options-statistics": "1",
+	}
+	defaultIntProps := map[string]uint64{
+		"rocksdb.base-level":                 1,
+		"rocksdb.block-cache-capacity":       2,
+		"rocksdb.block-cache-pinned-usage":   3,
+		"rocksdb.block-cache-usage":          4,
+		"rocksdb.cur-size-active-mem-table":  5,
+		"rocksdb.cur-size-all-mem-tables":    6,
+		"rocksdb.estimate-live-data-size":    7,
+		"rocksdb.estimate-num-keys":          8,
+		"rocksdb.estimate-table-readers-mem": 9,
+		"rocksdb.live-sst-files-size":        10,
+		"rocksdb.size-all-mem-tables":        11,
+	}
+	missingProps := make(map[string]string)
+	missingIntProps := make(map[string]uint64)
+	defaultExpectedProps := properties{
+		BaseLevel:               1,
+		BlockCacheCapacity:      2,
+		BlockCachePinnedUsage:   3,
+		BlockCacheUsage:         4,
+		CurSizeActiveMemTable:   5,
+		CurSizeAllMemTables:     6,
+		EstimateLiveDataSize:    7,
+		EstimateNumKeys:         8,
+		EstimateTableReadersMem: 9,
+		LiveSSTFilesSize:        10,
+		SizeAllMemTables:        11,
+		OptionsStatistics:       "1",
+	}
+
+	for _, tc := range []struct {
+		desc          string
+		props         map[string]string
+		intProps      map[string]uint64
+		expectedProps *properties
+		success       bool
+	}{
+		{
+			desc:          "success case",
+			props:         defaultProps,
+			intProps:      defaultIntProps,
+			expectedProps: &defaultExpectedProps,
+			success:       true,
+		},
+		{
+			desc:          "missing props",
+			props:         missingProps,
+			intProps:      defaultIntProps,
+			expectedProps: nil,
+			success:       false,
+		},
+		{
+			desc:          "missing integer props",
+			props:         defaultProps,
+			intProps:      missingIntProps,
+			expectedProps: nil,
+			success:       false,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			mockPropsGetter := newMockPropsGetter(tc.props, tc.intProps)
+
+			propsLoader := newPropsLoader(mockPropsGetter)
+			actualProps, err := propsLoader.load()
+			if tc.success {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+			}
+			require.Equal(t, tc.expectedProps, actualProps)
+		})
+	}
+}

cmd/kava/opendb/stat_parser.go

@@ -0,0 +1,111 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"fmt"
+	"strings"
+
+	"errors"
+)
+
+// stat represents one line from rocksdb statistics data, stat may have one or more properties
+// examples:
+// - rocksdb.block.cache.miss COUNT : 5
+// - rocksdb.compaction.times.micros P50 : 21112 P95 : 21112 P99 : 21112 P100 : 21112 COUNT : 1 SUM : 21112
+// `rocksdb.compaction.times.micros` is name of stat, P50, COUNT, SUM, etc... are props of stat
+type stat struct {
+	name  string
+	props map[string]string
+}
+
+// parseSerializedStats parses serialisedStats into map of stat objects
+// example of serializedStats:
+// rocksdb.block.cache.miss COUNT : 5
+// rocksdb.compaction.times.micros P50 : 21112 P95 : 21112 P99 : 21112 P100 : 21112 COUNT : 1 SUM : 21112
+func parseSerializedStats(serializedStats string) (map[string]*stat, error) {
+	stats := make(map[string]*stat, 0)
+
+	serializedStatList := strings.Split(serializedStats, "\n")
+	if len(serializedStatList) == 0 {
+		return nil, errors.New("serializedStats is empty")
+	}
+	serializedStatList = serializedStatList[:len(serializedStatList)-1]
+	// iterate over stats line by line
+	for _, serializedStat := range serializedStatList {
+		stat, err := parseSerializedStat(serializedStat)
+		if err != nil {
+			return nil, err
+		}
+
+		stats[stat.name] = stat
+	}
+
+	return stats, nil
+}
+
+// parseSerializedStat parses serialisedStat into stat object
+// example of serializedStat:
+// rocksdb.block.cache.miss COUNT : 5
+func parseSerializedStat(serializedStat string) (*stat, error) {
+	tokens := strings.Split(serializedStat, " ")
+	tokensNum := len(tokens)
+	if err := validateTokens(tokens); err != nil {
+		return nil, fmt.Errorf("tokens are invalid: %v", err)
+	}
+
+	props := make(map[string]string)
+	for idx := 1; idx < tokensNum; idx += 3 {
+		// never should happen, but double check to avoid unexpected panic
+		if idx+2 >= tokensNum {
+			break
+		}
+
+		key := tokens[idx]
+		sep := tokens[idx+1]
+		value := tokens[idx+2]
+
+		if err := validateStatProperty(key, value, sep); err != nil {
+			return nil, fmt.Errorf("invalid stat property: %v", err)
+		}
+
+		props[key] = value
+	}
+
+	return &stat{
+		name:  tokens[0],
+		props: props,
+	}, nil
+}
+
+// validateTokens validates that tokens contains name + N triples (key, sep, value)
+func validateTokens(tokens []string) error {
+	tokensNum := len(tokens)
+	if tokensNum < 4 {
+		return fmt.Errorf("invalid number of tokens: %v, tokens: %v", tokensNum, tokens)
+	}
+	if (tokensNum-1)%3 != 0 {
+		return fmt.Errorf("invalid number of tokens: %v, tokens: %v", tokensNum, tokens)
+	}
+	if tokens[0] == "" {
+		return fmt.Errorf("stat name shouldn't be empty")
+	}
+
+	return nil
+}
+
+// validateStatProperty validates that key and value are divided by separator and aren't empty
+func validateStatProperty(key, value, sep string) error {
+	if key == "" {
+		return fmt.Errorf("key shouldn't be empty")
+	}
+	if sep != ":" {
+		return fmt.Errorf("separator should be :")
+	}
+	if value == "" {
+		return fmt.Errorf("value shouldn't be empty")
+	}
+
+	return nil
+}

cmd/kava/opendb/stat_parser_test.go

@@ -0,0 +1,208 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseSerializedStats(t *testing.T) {
+	defaultSerializedStats := `rocksdb.block.cache.miss COUNT : 1
+rocksdb.block.cache.hit COUNT : 2
+rocksdb.block.cache.add COUNT : 3
+rocksdb.block.cache.add.failures COUNT : 4
+rocksdb.compaction.times.micros P50 : 1 P95 : 2 P99 : 3 P100 : 4 COUNT : 5 SUM : 6
+rocksdb.compaction.times.cpu_micros P50 : 7 P95 : 8 P99 : 9 P100 : 10 COUNT : 11 SUM : 12
+`
+	defaultExpectedStatMap := map[string]*stat{
+		"rocksdb.block.cache.miss": {
+			name: "rocksdb.block.cache.miss",
+			props: map[string]string{
+				"COUNT": "1",
+			},
+		},
+		"rocksdb.block.cache.hit": {
+			name: "rocksdb.block.cache.hit",
+			props: map[string]string{
+				"COUNT": "2",
+			},
+		},
+		"rocksdb.block.cache.add": {
+			name: "rocksdb.block.cache.add",
+			props: map[string]string{
+				"COUNT": "3",
+			},
+		},
+		"rocksdb.block.cache.add.failures": {
+			name: "rocksdb.block.cache.add.failures",
+			props: map[string]string{
+				"COUNT": "4",
+			},
+		},
+		"rocksdb.compaction.times.micros": {
+			name: "rocksdb.compaction.times.micros",
+			props: map[string]string{
+				"P50":   "1",
+				"P95":   "2",
+				"P99":   "3",
+				"P100":  "4",
+				"COUNT": "5",
+				"SUM":   "6",
+			},
+		},
+		"rocksdb.compaction.times.cpu_micros": {
+			name: "rocksdb.compaction.times.cpu_micros",
+			props: map[string]string{
+				"P50":   "7",
+				"P95":   "8",
+				"P99":   "9",
+				"P100":  "10",
+				"COUNT": "11",
+				"SUM":   "12",
+			},
+		},
+	}
+
+	for _, tc := range []struct {
+		desc            string
+		serializedStats string
+		expectedStatMap map[string]*stat
+		errMsg          string
+	}{
+		{
+			desc:            "success case",
+			serializedStats: defaultSerializedStats,
+			expectedStatMap: defaultExpectedStatMap,
+			errMsg:          "",
+		},
+		{
+			desc: "missing value #1",
+			serializedStats: `rocksdb.block.cache.miss COUNT :
+`,
+			expectedStatMap: nil,
+			errMsg:          "invalid number of tokens",
+		},
+		{
+			desc: "missing value #2",
+			serializedStats: `rocksdb.compaction.times.micros P50 : 1 P95 :
+`,
+			expectedStatMap: nil,
+			errMsg:          "invalid number of tokens",
+		},
+		{
+			desc: "missing stat name",
+			serializedStats: ` COUNT : 1
+`,
+			expectedStatMap: nil,
+			errMsg:          "stat name shouldn't be empty",
+		},
+		{
+			desc:            "empty stat",
+			serializedStats: ``,
+			expectedStatMap: make(map[string]*stat),
+			errMsg:          "",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			actualStatMap, err := parseSerializedStats(tc.serializedStats)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errMsg)
+			}
+			require.Equal(t, tc.expectedStatMap, actualStatMap)
+		})
+	}
+}
+
+func TestValidateTokens(t *testing.T) {
+	for _, tc := range []struct {
+		desc   string
+		tokens []string
+		errMsg string
+	}{
+		{
+			desc:   "success case",
+			tokens: []string{"name", "key", ":", "value"},
+			errMsg: "",
+		},
+		{
+			desc:   "missing value #1",
+			tokens: []string{"name", "key", ":"},
+			errMsg: "invalid number of tokens",
+		},
+		{
+			desc:   "missing value #2",
+			tokens: []string{"name", "key", ":", "value", "key2", ":"},
+			errMsg: "invalid number of tokens",
+		},
+		{
+			desc:   "empty stat name",
+			tokens: []string{"", "key", ":", "value"},
+			errMsg: "stat name shouldn't be empty",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			err := validateTokens(tc.tokens)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errMsg)
+			}
+		})
+	}
+}
+
+func TestValidateStatProperty(t *testing.T) {
+	for _, tc := range []struct {
+		desc   string
+		key    string
+		value  string
+		sep    string
+		errMsg string
+	}{
+		{
+			desc:   "success case",
+			key:    "key",
+			value:  "value",
+			sep:    ":",
+			errMsg: "",
+		},
+		{
+			desc:   "missing key",
+			key:    "",
+			value:  "value",
+			sep:    ":",
+			errMsg: "key shouldn't be empty",
+		},
+		{
+			desc:   "missing value",
+			key:    "key",
+			value:  "",
+			sep:    ":",
+			errMsg: "value shouldn't be empty",
+		},
+		{
+			desc:   "invalid separator",
+			key:    "key",
+			value:  "value",
+			sep:    "#",
+			errMsg: "separator should be :",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			err := validateStatProperty(tc.key, tc.value, tc.sep)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errMsg)
+			}
+		})
+	}
+}

cmd/kava/opendb/stats_loader.go

@@ -0,0 +1,263 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"fmt"
+	"strconv"
+)
+
+const (
+	sum   = "SUM"
+	count = "COUNT"
+	p50   = "P50"
+	p95   = "P95"
+	p99   = "P99"
+	p100  = "P100"
+)
+
+type statLoader struct {
+	// statMap contains map of stat objects returned by parseSerializedStats function
+	// example of stats:
+	// #1: rocksdb.block.cache.miss COUNT : 5
+	// #2: rocksdb.compaction.times.micros P50 : 21112 P95 : 21112 P99 : 21112 P100 : 21112 COUNT : 1 SUM : 21112
+	// #1 case will be cast into int64
+	// #2 case will be cast into float64Histogram
+	statMap map[string]*stat
+
+	// NOTE: some methods accumulate errors instead of returning them, these methods are private and not intended to use outside
+	errors []error
+}
+
+func newStatLoader(statMap map[string]*stat) *statLoader {
+	return &statLoader{
+		statMap: statMap,
+		errors:  make([]error, 0),
+	}
+}
+
+type stats struct {
+	NumberKeysWritten int64
+	NumberKeysRead    int64
+	NumberKeysUpdated int64
+
+	// total block cache misses
+	// BLOCK_CACHE_MISS == BLOCK_CACHE_INDEX_MISS +
+	//                     BLOCK_CACHE_FILTER_MISS +
+	//                     BLOCK_CACHE_DATA_MISS;
+	// BLOCK_CACHE_INDEX_MISS: # of times cache miss when accessing index block from block cache.
+	// BLOCK_CACHE_FILTER_MISS: # of times cache miss when accessing filter block from block cache.
+	// BLOCK_CACHE_DATA_MISS: # of times cache miss when accessing data block from block cache.
+	BlockCacheMiss int64
+
+	// total block cache hit
+	// BLOCK_CACHE_HIT == BLOCK_CACHE_INDEX_HIT +
+	//                    BLOCK_CACHE_FILTER_HIT +
+	//                    BLOCK_CACHE_DATA_HIT;
+	// BLOCK_CACHE_INDEX_HIT: # of times cache hit when accessing index block from block cache.
+	// BLOCK_CACHE_FILTER_HIT: # of times cache hit when accessing filter block from block cache.
+	// BLOCK_CACHE_DATA_HIT: # of times cache hit when accessing data block from block cache.
+	BlockCacheHit int64
+
+	// # of blocks added to block cache.
+	BlockCacheAdd int64
+	// # of failures when adding blocks to block cache.
+	BlockCacheAddFailures int64
+
+	CompactReadBytes  int64 // Bytes read during compaction
+	CompactWriteBytes int64 // Bytes written during compaction
+
+	CompactionTimesMicros      *float64Histogram
+	CompactionTimesCPUMicros   *float64Histogram
+	NumFilesInSingleCompaction *float64Histogram
+
+	// Read amplification statistics.
+	// Read amplification can be calculated using this formula
+	// (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES)
+	//
+	// REQUIRES: ReadOptions::read_amp_bytes_per_bit to be enabled
+	// TODO(yevhenii): seems not working?
+	ReadAmpEstimateUsefulBytes int64 // Estimate of total bytes actually used.
+	ReadAmpTotalReadBytes      int64 // Total size of loaded data blocks.
+
+	NumberFileOpens  int64
+	NumberFileErrors int64
+
+	// # of times bloom filter has avoided file reads, i.e., negatives.
+	BloomFilterUseful int64
+	// # of times bloom FullFilter has not avoided the reads.
+	BloomFilterFullPositive int64
+	// # of times bloom FullFilter has not avoided the reads and data actually
+	// exist.
+	BloomFilterFullTruePositive int64
+
+	// # of memtable hits.
+	MemtableHit int64
+	// # of memtable misses.
+	MemtableMiss int64
+
+	// # of Get() queries served by L0
+	GetHitL0 int64
+	// # of Get() queries served by L1
+	GetHitL1 int64
+	// # of Get() queries served by L2 and up
+	GetHitL2AndUp int64
+
+	// The number of uncompressed bytes issued by DB::Put(), DB::Delete(),
+	// DB::Merge(), and DB::Write().
+	BytesWritten int64
+	// The number of uncompressed bytes read from DB::Get().  It could be
+	// either from memtables, cache, or table files.
+	// For the number of logical bytes read from DB::MultiGet(),
+	// please use NUMBER_MULTIGET_BYTES_READ.
+	BytesRead int64
+
+	// Writer has to wait for compaction or flush to finish.
+	StallMicros int64
+
+	// Last level and non-last level read statistics
+	LastLevelReadBytes    int64
+	LastLevelReadCount    int64
+	NonLastLevelReadBytes int64
+	NonLastLevelReadCount int64
+
+	DBGetMicros   *float64Histogram
+	DBWriteMicros *float64Histogram
+
+	// Value size distribution in each operation
+	BytesPerRead     *float64Histogram
+	BytesPerWrite    *float64Histogram
+	BytesPerMultiget *float64Histogram
+
+	// Time spent flushing memtable to disk
+	FlushMicros *float64Histogram
+}
+
+type float64Histogram struct {
+	Sum   float64
+	Count float64
+	P50   float64
+	P95   float64
+	P99   float64
+	P100  float64
+}
+
+func (l *statLoader) error() error {
+	if len(l.errors) != 0 {
+		return fmt.Errorf("%v", l.errors)
+	}
+
+	return nil
+}
+
+func (l *statLoader) load() (*stats, error) {
+	stats := &stats{
+		NumberKeysWritten:           l.getInt64StatValue("rocksdb.number.keys.written", count),
+		NumberKeysRead:              l.getInt64StatValue("rocksdb.number.keys.read", count),
+		NumberKeysUpdated:           l.getInt64StatValue("rocksdb.number.keys.updated", count),
+		BlockCacheMiss:              l.getInt64StatValue("rocksdb.block.cache.miss", count),
+		BlockCacheHit:               l.getInt64StatValue("rocksdb.block.cache.hit", count),
+		BlockCacheAdd:               l.getInt64StatValue("rocksdb.block.cache.add", count),
+		BlockCacheAddFailures:       l.getInt64StatValue("rocksdb.block.cache.add.failures", count),
+		CompactReadBytes:            l.getInt64StatValue("rocksdb.compact.read.bytes", count),
+		CompactWriteBytes:           l.getInt64StatValue("rocksdb.compact.write.bytes", count),
+		CompactionTimesMicros:       l.getFloat64HistogramStatValue("rocksdb.compaction.times.micros"),
+		CompactionTimesCPUMicros:    l.getFloat64HistogramStatValue("rocksdb.compaction.times.cpu_micros"),
+		NumFilesInSingleCompaction:  l.getFloat64HistogramStatValue("rocksdb.numfiles.in.singlecompaction"),
+		ReadAmpEstimateUsefulBytes:  l.getInt64StatValue("rocksdb.read.amp.estimate.useful.bytes", count),
+		ReadAmpTotalReadBytes:       l.getInt64StatValue("rocksdb.read.amp.total.read.bytes", count),
+		NumberFileOpens:             l.getInt64StatValue("rocksdb.no.file.opens", count),
+		NumberFileErrors:            l.getInt64StatValue("rocksdb.no.file.errors", count),
+		BloomFilterUseful:           l.getInt64StatValue("rocksdb.bloom.filter.useful", count),
+		BloomFilterFullPositive:     l.getInt64StatValue("rocksdb.bloom.filter.full.positive", count),
+		BloomFilterFullTruePositive: l.getInt64StatValue("rocksdb.bloom.filter.full.true.positive", count),
+		MemtableHit:                 l.getInt64StatValue("rocksdb.memtable.hit", count),
+		MemtableMiss:                l.getInt64StatValue("rocksdb.memtable.miss", count),
+		GetHitL0:                    l.getInt64StatValue("rocksdb.l0.hit", count),
+		GetHitL1:                    l.getInt64StatValue("rocksdb.l1.hit", count),
+		GetHitL2AndUp:               l.getInt64StatValue("rocksdb.l2andup.hit", count),
+		BytesWritten:                l.getInt64StatValue("rocksdb.bytes.written", count),
+		BytesRead:                   l.getInt64StatValue("rocksdb.bytes.read", count),
+		StallMicros:                 l.getInt64StatValue("rocksdb.stall.micros", count),
+		LastLevelReadBytes:          l.getInt64StatValue("rocksdb.last.level.read.bytes", count),
+		LastLevelReadCount:          l.getInt64StatValue("rocksdb.last.level.read.count", count),
+		NonLastLevelReadBytes:       l.getInt64StatValue("rocksdb.non.last.level.read.bytes", count),
+		NonLastLevelReadCount:       l.getInt64StatValue("rocksdb.non.last.level.read.count", count),
+		DBGetMicros:                 l.getFloat64HistogramStatValue("rocksdb.db.get.micros"),
+		DBWriteMicros:               l.getFloat64HistogramStatValue("rocksdb.db.write.micros"),
+		BytesPerRead:                l.getFloat64HistogramStatValue("rocksdb.bytes.per.read"),
+		BytesPerWrite:               l.getFloat64HistogramStatValue("rocksdb.bytes.per.write"),
+		BytesPerMultiget:            l.getFloat64HistogramStatValue("rocksdb.bytes.per.multiget"),
+		FlushMicros:                 l.getFloat64HistogramStatValue("rocksdb.db.flush.micros"),
+	}
+
+	err := l.error()
+	if err != nil {
+		return nil, err
+	}
+
+	return stats, nil
+}
+
+// getFloat64HistogramStatValue converts stat object into float64Histogram
+func (l *statLoader) getFloat64HistogramStatValue(statName string) *float64Histogram {
+	return &float64Histogram{
+		Sum:   l.getFloat64StatValue(statName, sum),
+		Count: l.getFloat64StatValue(statName, count),
+		P50:   l.getFloat64StatValue(statName, p50),
+		P95:   l.getFloat64StatValue(statName, p95),
+		P99:   l.getFloat64StatValue(statName, p99),
+		P100:  l.getFloat64StatValue(statName, p100),
+	}
+}
+
+// getInt64StatValue converts property of stat object into int64
+func (l *statLoader) getInt64StatValue(statName, propName string) int64 {
+	stringVal := l.getStatValue(statName, propName)
+	if stringVal == "" {
+		l.errors = append(l.errors, fmt.Errorf("can't get stat by name: %v", statName))
+		return 0
+	}
+
+	intVal, err := strconv.ParseInt(stringVal, 10, 64)
+	if err != nil {
+		l.errors = append(l.errors, fmt.Errorf("can't parse int: %v", err))
+		return 0
+	}
+
+	return intVal
+}
+
+// getFloat64StatValue converts property of stat object into float64
+func (l *statLoader) getFloat64StatValue(statName, propName string) float64 {
+	stringVal := l.getStatValue(statName, propName)
+	if stringVal == "" {
+		l.errors = append(l.errors, fmt.Errorf("can't get stat by name: %v", statName))
+		return 0
+	}
+
+	floatVal, err := strconv.ParseFloat(stringVal, 64)
+	if err != nil {
+		l.errors = append(l.errors, fmt.Errorf("can't parse float: %v", err))
+		return 0
+	}
+
+	return floatVal
+}
+
+// getStatValue gets property of stat object
+func (l *statLoader) getStatValue(statName, propName string) string {
+	stat, ok := l.statMap[statName]
+	if !ok {
+		l.errors = append(l.errors, fmt.Errorf("stat %v doesn't exist", statName))
+		return ""
+	}
+	prop, ok := stat.props[propName]
+	if !ok {
+		l.errors = append(l.errors, fmt.Errorf("stat %v doesn't have %v property", statName, propName))
+		return ""
+	}
+
+	return prop
+}

cmd/kava/opendb/stats_loader_test.go

@@ -0,0 +1,80 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestStatsLoader(t *testing.T) {
+	defaultStat := stat{
+		props: map[string]string{
+			"COUNT": "1",
+		},
+	}
+	defaultHistogramStat := stat{
+		props: map[string]string{
+			"P50":   "1",
+			"P95":   "2",
+			"P99":   "3",
+			"P100":  "4",
+			"COUNT": "5",
+			"SUM":   "6",
+		},
+	}
+	defaultStatMap := map[string]*stat{
+		"rocksdb.number.keys.written":             &defaultStat,
+		"rocksdb.number.keys.read":                &defaultStat,
+		"rocksdb.number.keys.updated":             &defaultStat,
+		"rocksdb.block.cache.miss":                &defaultStat,
+		"rocksdb.block.cache.hit":                 &defaultStat,
+		"rocksdb.block.cache.add":                 &defaultStat,
+		"rocksdb.block.cache.add.failures":        &defaultStat,
+		"rocksdb.compact.read.bytes":              &defaultStat,
+		"rocksdb.compact.write.bytes":             &defaultStat,
+		"rocksdb.compaction.times.micros":         &defaultHistogramStat,
+		"rocksdb.compaction.times.cpu_micros":     &defaultHistogramStat,
+		"rocksdb.numfiles.in.singlecompaction":    &defaultHistogramStat,
+		"rocksdb.read.amp.estimate.useful.bytes":  &defaultStat,
+		"rocksdb.read.amp.total.read.bytes":       &defaultStat,
+		"rocksdb.no.file.opens":                   &defaultStat,
+		"rocksdb.no.file.errors":                  &defaultStat,
+		"rocksdb.bloom.filter.useful":             &defaultStat,
+		"rocksdb.bloom.filter.full.positive":      &defaultStat,
+		"rocksdb.bloom.filter.full.true.positive": &defaultStat,
+		"rocksdb.memtable.hit":                    &defaultStat,
+		"rocksdb.memtable.miss":                   &defaultStat,
+		"rocksdb.l0.hit":                          &defaultStat,
+		"rocksdb.l1.hit":                          &defaultStat,
+		"rocksdb.l2andup.hit":                     &defaultStat,
+		"rocksdb.bytes.written":                   &defaultStat,
+		"rocksdb.bytes.read":                      &defaultStat,
+		"rocksdb.stall.micros":                    &defaultStat,
+		"rocksdb.last.level.read.bytes":           &defaultStat,
+		"rocksdb.last.level.read.count":           &defaultStat,
+		"rocksdb.non.last.level.read.bytes":       &defaultStat,
+		"rocksdb.non.last.level.read.count":       &defaultStat,
+		"rocksdb.db.get.micros":                   &defaultHistogramStat,
+		"rocksdb.db.write.micros":                 &defaultHistogramStat,
+		"rocksdb.bytes.per.read":                  &defaultHistogramStat,
+		"rocksdb.bytes.per.write":                 &defaultHistogramStat,
+		"rocksdb.bytes.per.multiget":              &defaultHistogramStat,
+		"rocksdb.db.flush.micros":                 &defaultHistogramStat,
+	}
+
+	statLoader := newStatLoader(defaultStatMap)
+	stats, err := statLoader.load()
+	require.NoError(t, err)
+
+	require.Equal(t, stats.NumberKeysWritten, int64(1))
+	require.Equal(t, stats.NumberKeysRead, int64(1))
+	require.Equal(t, stats.CompactionTimesMicros.P50, float64(1))
+	require.Equal(t, stats.CompactionTimesMicros.P95, float64(2))
+	require.Equal(t, stats.CompactionTimesMicros.P99, float64(3))
+	require.Equal(t, stats.CompactionTimesMicros.P100, float64(4))
+	require.Equal(t, stats.CompactionTimesMicros.Count, float64(5))
+	require.Equal(t, stats.CompactionTimesMicros.Sum, float64(6))
+}