feat(x/metrics): add module for emiting custom chain metrics (#1668)

* initialize x/metrics with metrics collection

* include global labels in x/metrics metrics

* add x/metrics spec

* add x/metrics test coverage

* update changelog
This commit is contained in:
Robert Pirtle 2023-08-25 12:23:53 -07:00 committed by GitHub
parent 0ddc87ad1d
commit 9a0aed7626
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 401 additions and 2 deletions

View File

@ -36,6 +36,9 @@ Ref: https://keepachangelog.com/en/1.0.0/
## [Unreleased]
### Features
- (metrics) [#1668] Adds non-state breaking x/metrics module for custom telemetry.
### Bug Fixes
- (evmutil) [#1655] Initialize x/evmutil module account in InitGenesis
@ -276,6 +279,7 @@ the [changelog](https://github.com/cosmos/cosmos-sdk/blob/v0.38.4/CHANGELOG.md).
- [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run
large-scale simulations remotely using aws-batch
[#1668]: https://github.com/Kava-Labs/kava/pull/1668
[#1655]: https://github.com/Kava-Labs/kava/pull/1655
[#1624]: https://github.com/Kava-Labs/kava/pull/1624
[#1631]: https://github.com/Kava-Labs/kava/pull/1631

View File

@ -139,6 +139,8 @@ import (
"github.com/kava-labs/kava/x/liquid"
liquidkeeper "github.com/kava-labs/kava/x/liquid/keeper"
liquidtypes "github.com/kava-labs/kava/x/liquid/types"
metrics "github.com/kava-labs/kava/x/metrics"
metricstypes "github.com/kava-labs/kava/x/metrics/types"
pricefeed "github.com/kava-labs/kava/x/pricefeed"
pricefeedkeeper "github.com/kava-labs/kava/x/pricefeed/keeper"
pricefeedtypes "github.com/kava-labs/kava/x/pricefeed/types"
@ -216,6 +218,7 @@ var (
router.AppModuleBasic{},
mint.AppModuleBasic{},
community.AppModuleBasic{},
metrics.AppModuleBasic{},
)
// module account permissions
@ -261,6 +264,7 @@ type Options struct {
MempoolAuthAddresses []sdk.AccAddress
EVMTrace string
EVMMaxGasWanted uint64
TelemetryOptions metricstypes.TelemetryOptions
}
// DefaultOptions is a sensible default Options value.
@ -790,10 +794,12 @@ func NewApp(
// nil InflationCalculationFn, use SDK's default inflation function
mint.NewAppModule(appCodec, app.mintKeeper, app.accountKeeper, nil),
community.NewAppModule(app.communityKeeper, app.accountKeeper),
metrics.NewAppModule(options.TelemetryOptions),
)
// Warning: Some begin blockers must run before others. Ensure the dependencies are understood before modifying this list.
app.mm.SetOrderBeginBlockers(
metricstypes.ModuleName,
// Upgrade begin blocker runs migrations on the first block after an upgrade. It should run before any other module.
upgradetypes.ModuleName,
// Capability begin blocker runs non state changing initialization.
@ -882,6 +888,7 @@ func NewApp(
routertypes.ModuleName,
minttypes.ModuleName,
communitytypes.ModuleName,
metricstypes.ModuleName,
)
// Warning: Some init genesis methods must run before others. Ensure the dependencies are understood before modifying this list
@ -923,6 +930,7 @@ func NewApp(
validatorvestingtypes.ModuleName,
liquidtypes.ModuleName,
routertypes.ModuleName,
metricstypes.ModuleName,
)
app.mm.RegisterInvariants(&app.crisisKeeper)

View File

@ -24,6 +24,7 @@ import (
"github.com/kava-labs/kava/app"
"github.com/kava-labs/kava/app/params"
metricstypes "github.com/kava-labs/kava/x/metrics/types"
)
const (
@ -99,6 +100,7 @@ func (ac appCreator) newApp(
MempoolAuthAddresses: mempoolAuthAddresses,
EVMTrace: cast.ToString(appOpts.Get(ethermintflags.EVMTracer)),
EVMMaxGasWanted: cast.ToUint64(appOpts.Get(ethermintflags.EVMMaxTxGasWanted)),
TelemetryOptions: metricstypes.TelemetryOptionsFromAppOpts(appOpts),
},
baseapp.SetPruning(pruningOpts),
baseapp.SetMinGasPrices(strings.Replace(cast.ToString(appOpts.Get(server.FlagMinGasPrices)), ";", ",", -1)),

4
go.mod
View File

@ -12,12 +12,14 @@ require (
github.com/cosmos/ibc-go/v6 v6.1.1
github.com/ethereum/go-ethereum v1.10.26
github.com/evmos/ethermint v0.21.0
github.com/go-kit/kit v0.12.0
github.com/gogo/protobuf v1.3.3
github.com/golang/protobuf v1.5.3
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/linxGnu/grocksdb v1.8.0
github.com/pelletier/go-toml/v2 v2.0.6
github.com/prometheus/client_golang v1.14.0
github.com/spf13/cast v1.5.0
github.com/spf13/cobra v1.6.1
github.com/spf13/viper v1.15.0
@ -85,7 +87,6 @@ require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/gin-gonic/gin v1.8.1 // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
@ -148,7 +149,6 @@ require (
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.40.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect

12
x/metrics/abci.go Normal file
View File

@ -0,0 +1,12 @@
package metrics
import (
"github.com/kava-labs/kava/x/metrics/types"
sdk "github.com/cosmos/cosmos-sdk/types"
)
// BeginBlocker publishes metrics at the start of each block.
func BeginBlocker(ctx sdk.Context, metrics *types.Metrics) {
metrics.LatestBlockHeight.Set(float64(ctx.BlockHeight()))
}

45
x/metrics/abci_test.go Normal file
View File

@ -0,0 +1,45 @@
package metrics_test
import (
"testing"
kitmetrics "github.com/go-kit/kit/metrics"
"github.com/stretchr/testify/require"
sdk "github.com/cosmos/cosmos-sdk/types"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/kava-labs/kava/app"
"github.com/kava-labs/kava/x/metrics"
"github.com/kava-labs/kava/x/metrics/types"
)
type MockGauge struct {
value float64
}
func (mg *MockGauge) With(labelValues ...string) kitmetrics.Gauge { return mg }
func (mg *MockGauge) Set(value float64) { mg.value = value }
func (*MockGauge) Add(_ float64) {}
func ctxWithHeight(height int64) sdk.Context {
tApp := app.NewTestApp()
tApp.InitializeFromGenesisStates()
return tApp.NewContext(false, tmproto.Header{Height: height})
}
func TestBeginBlockEmitsLatestHeight(t *testing.T) {
gauge := MockGauge{}
myMetrics := &types.Metrics{
LatestBlockHeight: &gauge,
}
metrics.BeginBlocker(ctxWithHeight(1), myMetrics)
require.EqualValues(t, 1, gauge.value)
metrics.BeginBlocker(ctxWithHeight(100), myMetrics)
require.EqualValues(t, 100, gauge.value)
metrics.BeginBlocker(ctxWithHeight(17e6), myMetrics)
require.EqualValues(t, 17e6, gauge.value)
}

125
x/metrics/module.go Normal file
View File

@ -0,0 +1,125 @@
package metrics
import (
"encoding/json"
"github.com/grpc-ecosystem/grpc-gateway/runtime"
"github.com/spf13/cobra"
"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/codec"
codectypes "github.com/cosmos/cosmos-sdk/codec/types"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/types/module"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/kava-labs/kava/x/metrics/types"
)
var (
_ module.AppModule = AppModule{}
_ module.AppModuleBasic = AppModuleBasic{}
)
// AppModuleBasic app module basics object
type AppModuleBasic struct{}
// Name returns the module name
func (AppModuleBasic) Name() string {
return types.ModuleName
}
// RegisterLegacyAminoCodec register module codec
// Deprecated: unused but necessary to fulfill AppModuleBasic interface
func (AppModuleBasic) RegisterLegacyAminoCodec(cdc *codec.LegacyAmino) {}
// DefaultGenesis default genesis state
func (AppModuleBasic) DefaultGenesis(_ codec.JSONCodec) json.RawMessage {
return []byte("{}")
}
// ValidateGenesis module validate genesis
func (AppModuleBasic) ValidateGenesis(_ codec.JSONCodec, _ client.TxEncodingConfig, _ json.RawMessage) error {
return nil
}
// RegisterInterfaces implements InterfaceModule.RegisterInterfaces
func (a AppModuleBasic) RegisterInterfaces(registry codectypes.InterfaceRegistry) {}
// RegisterGRPCGatewayRoutes registers the gRPC Gateway routes for the module.
func (a AppModuleBasic) RegisterGRPCGatewayRoutes(clientCtx client.Context, mux *runtime.ServeMux) {}
// GetTxCmd returns the root tx command for the module.
func (AppModuleBasic) GetTxCmd() *cobra.Command {
return nil
}
// GetQueryCmd returns no root query command for the module.
func (AppModuleBasic) GetQueryCmd() *cobra.Command {
return nil
}
//____________________________________________________________________________
// AppModule app module type
type AppModule struct {
AppModuleBasic
metrics *types.Metrics
}
// NewAppModule creates a new AppModule object
func NewAppModule(telemetryOpts types.TelemetryOptions) AppModule {
return AppModule{
AppModuleBasic: AppModuleBasic{},
metrics: types.NewMetrics(telemetryOpts),
}
}
// Name module name
func (am AppModule) Name() string {
return am.AppModuleBasic.Name()
}
// RegisterInvariants register module invariants
func (am AppModule) RegisterInvariants(_ sdk.InvariantRegistry) {}
// Route module message route name
// Deprecated: unused but necessary to fulfill AppModule interface
func (am AppModule) Route() sdk.Route { return sdk.Route{} }
// QuerierRoute module querier route name
// Deprecated: unused but necessary to fulfill AppModule interface
func (AppModule) QuerierRoute() string { return types.ModuleName }
// LegacyQuerierHandler returns no sdk.Querier.
// Deprecated: unused but necessary to fulfill AppModule interface
func (am AppModule) LegacyQuerierHandler(_ *codec.LegacyAmino) sdk.Querier {
return nil
}
// ConsensusVersion implements AppModule/ConsensusVersion.
func (AppModule) ConsensusVersion() uint64 { return 1 }
// RegisterServices registers module services.
func (am AppModule) RegisterServices(cfg module.Configurator) {}
// InitGenesis module init-genesis
func (am AppModule) InitGenesis(ctx sdk.Context, _ codec.JSONCodec, _ json.RawMessage) []abci.ValidatorUpdate {
return []abci.ValidatorUpdate{}
}
// ExportGenesis module export genesis
func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONCodec) json.RawMessage {
return nil
}
// BeginBlock module begin-block
func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) {
BeginBlocker(ctx, am.metrics)
}
// EndBlock module end-block
func (am AppModule) EndBlock(_ sdk.Context, _ abci.RequestEndBlock) []abci.ValidatorUpdate {
return []abci.ValidatorUpdate{}
}

36
x/metrics/spec/README.md Normal file
View File

@ -0,0 +1,36 @@
<!--
order: 0
title: "Metrics Overview"
parent:
title: "metrics"
-->
# `metrics`
## Abstract
`x/metrics` is a stateless module that does not affect consensus. It captures chain metrics and emits them when the `instrumentation.prometheus` option is enabled in `config.toml`.
## Precision
The metrics emitted by `x/metrics` are `float64`s. They use `github.com/go-kit/kit/metrics` Prometheus gauges. Cosmos-sdk's `telemetry` package was not used because, at the time of writing, it only supports `float32`s and so does not maintain accurate representations of ints larger than ~16.8M. With `float64`s, integers may be accurately represented up to ~9e15.
## Metrics
The following metrics are defined:
* `cometbft_blocksync_latest_block_height` - this emulates the blocksync `latest_block_height` metric in CometBFT v0.38+. The `cometbft` namespace comes from the `instrumentation.namespace` config.toml value.
## Metric Labels
All metrics emitted have the labels defined in app.toml's `telemetry.global-labels` field. This is the same field used by cosmos-sdk's `telemetry` package.
example:
```toml
# app.toml
[telemetry]
global-labels = [
["chain_id", "kava_2222-10"],
["my_label", "my_value"],
]
```

6
x/metrics/types/keys.go Normal file
View File

@ -0,0 +1,6 @@
package types
const (
// Name of the module
ModuleName = "metrics"
)

View File

@ -0,0 +1,89 @@
package types
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
"github.com/spf13/cast"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
)
// TelemetryOptions defines the app configurations for the x/metrics module
type TelemetryOptions struct {
// CometBFT config value for instrumentation.prometheus (config.toml)
PrometheusEnabled bool
// Namespace for CometBFT metrics. Used to emulate CometBFT metrics.
CometBFTMetricsNamespace string
// A list of keys and values used as labels on all metrics
GlobalLabelsAndValues []string
}
// TelemetryOptionsFromAppOpts creates the TelemetryOptions from server AppOptions
func TelemetryOptionsFromAppOpts(appOpts servertypes.AppOptions) TelemetryOptions {
prometheusEnabled := cast.ToBool(appOpts.Get("instrumentation.prometheus"))
if !prometheusEnabled {
return TelemetryOptions{
GlobalLabelsAndValues: []string{},
}
}
// parse the app.toml global-labels into a slice of alternating label & value strings
// the value is expected to be a list of [label, value] tuples.
rawLabels := cast.ToSlice(appOpts.Get("telemetry.global-labels"))
globalLabelsAndValues := make([]string, 0, len(rawLabels)*2)
for _, gl := range rawLabels {
l := cast.ToStringSlice(gl)
globalLabelsAndValues = append(globalLabelsAndValues, l[0], l[1])
}
return TelemetryOptions{
PrometheusEnabled: true,
CometBFTMetricsNamespace: cast.ToString(appOpts.Get("instrumentation.namespace")),
GlobalLabelsAndValues: globalLabelsAndValues,
}
}
// Metrics contains metrics exposed by this module.
// They use go-kit metrics like CometBFT as opposed to using cosmos-sdk telemetry
// because the sdk's telemetry only supports float32s, whereas go-kit prometheus
// metrics correctly handle float64s (and thus a larger number of int64s)
type Metrics struct {
// The height of the latest block.
// This gauges exactly emulates the default blocksync metric in CometBFT v0.38+
// It should be removed when kava has been updated to CometBFT v0.38+.
// see https://github.com/cometbft/cometbft/blob/v0.38.0-rc3/blocksync/metrics.gen.go
LatestBlockHeight metrics.Gauge
}
// NewMetrics creates a new Metrics object based on whether or not prometheus instrumentation is enabled.
func NewMetrics(opts TelemetryOptions) *Metrics {
if opts.PrometheusEnabled {
return PrometheusMetrics(opts)
}
return NoopMetrics()
}
// PrometheusMetrics returns the gauges for when prometheus instrumentation is enabled.
func PrometheusMetrics(opts TelemetryOptions) *Metrics {
labels := []string{}
for i := 0; i < len(opts.GlobalLabelsAndValues); i += 2 {
labels = append(labels, opts.GlobalLabelsAndValues[i])
}
return &Metrics{
LatestBlockHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: opts.CometBFTMetricsNamespace,
Subsystem: "blocksync",
Name: "latest_block_height",
Help: "The height of the latest block.",
}, labels).With(opts.GlobalLabelsAndValues...),
}
}
// NoopMetrics are a do-nothing placeholder used when prometheus instrumentation is not enabled.
func NoopMetrics() *Metrics {
return &Metrics{
LatestBlockHeight: discard.NewGauge(),
}
}

View File

@ -0,0 +1,72 @@
package types_test
import (
"testing"
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/prometheus"
"github.com/kava-labs/kava/x/metrics/types"
"github.com/stretchr/testify/require"
)
func isPrometheusGauge(g metrics.Gauge) bool {
_, ok := g.(*prometheus.Gauge)
return ok
}
var (
disabledOpts = types.TelemetryOptions{
PrometheusEnabled: false,
}
enabledOpts = types.TelemetryOptions{
PrometheusEnabled: true,
CometBFTMetricsNamespace: "cometbft",
GlobalLabelsAndValues: []string{"label1", "value1", "label2", "value2"},
}
)
func TestNewMetrics_DisabledVsEnabled(t *testing.T) {
myMetrics := types.NewMetrics(disabledOpts)
require.False(t, isPrometheusGauge(myMetrics.LatestBlockHeight))
myMetrics = types.NewMetrics(enabledOpts)
require.True(t, isPrometheusGauge(myMetrics.LatestBlockHeight))
}
type MockAppOpts struct {
store map[string]interface{}
}
func (mao *MockAppOpts) Get(key string) interface{} {
return mao.store[key]
}
func TestTelemetryOptionsFromAppOpts(t *testing.T) {
appOpts := MockAppOpts{store: make(map[string]interface{})}
// test disabled functionality
appOpts.store["instrumentation.prometheus"] = false
opts := types.TelemetryOptionsFromAppOpts(&appOpts)
require.False(t, opts.PrometheusEnabled)
// test enabled functionality
appOpts.store["instrumentation.prometheus"] = true
appOpts.store["instrumentation.namespace"] = "magic"
appOpts.store["telemetry.global-labels"] = []interface{}{}
opts = types.TelemetryOptionsFromAppOpts(&appOpts)
require.True(t, opts.PrometheusEnabled)
require.Equal(t, "magic", opts.CometBFTMetricsNamespace)
require.Len(t, opts.GlobalLabelsAndValues, 0)
appOpts.store["telemetry.global-labels"] = []interface{}{
[]interface{}{"label1", "value1"},
[]interface{}{"label2", "value2"},
}
opts = types.TelemetryOptionsFromAppOpts(&appOpts)
require.True(t, opts.PrometheusEnabled)
require.Equal(t, "magic", opts.CometBFTMetricsNamespace)
require.Len(t, opts.GlobalLabelsAndValues, 4)
require.Equal(t, enabledOpts.GlobalLabelsAndValues, opts.GlobalLabelsAndValues)
}