mirror of
https://source.quilibrium.com/quilibrium/ceremonyclient.git
synced 2024-11-15 12:45:19 +00:00
6a7cbab864
- removed defer/recover/panic where warranted - found and eliminated race conditions when extremely high connection pressure was hit - resolved race condition in blossomsub around mesh maintenance - incorporated resource manager scaling to work off low/hi watermark values
715 lines
13 KiB
Go
715 lines
13 KiB
Go
package rcmgr
|
|
|
|
import (
|
|
"compress/gzip"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libp2p/go-libp2p/core/network"
|
|
)
|
|
|
|
type trace struct {
|
|
path string
|
|
|
|
ctx context.Context
|
|
cancel func()
|
|
wg sync.WaitGroup
|
|
|
|
mx sync.Mutex
|
|
done bool
|
|
pendingWrites []interface{}
|
|
reporters []TraceReporter
|
|
}
|
|
|
|
type TraceReporter interface {
|
|
// ConsumeEvent consumes a trace event. This is called synchronously,
|
|
// implementations should process the event quickly.
|
|
ConsumeEvent(TraceEvt)
|
|
}
|
|
|
|
func WithTrace(path string) Option {
|
|
return func(r *resourceManager) error {
|
|
if r.trace == nil {
|
|
r.trace = &trace{path: path}
|
|
} else {
|
|
r.trace.path = path
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func WithTraceReporter(reporter TraceReporter) Option {
|
|
return func(r *resourceManager) error {
|
|
if r.trace == nil {
|
|
r.trace = &trace{}
|
|
}
|
|
r.trace.reporters = append(r.trace.reporters, reporter)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
type TraceEvtTyp string
|
|
|
|
const (
|
|
TraceStartEvt TraceEvtTyp = "start"
|
|
TraceCreateScopeEvt TraceEvtTyp = "create_scope"
|
|
TraceDestroyScopeEvt TraceEvtTyp = "destroy_scope"
|
|
TraceReserveMemoryEvt TraceEvtTyp = "reserve_memory"
|
|
TraceBlockReserveMemoryEvt TraceEvtTyp = "block_reserve_memory"
|
|
TraceReleaseMemoryEvt TraceEvtTyp = "release_memory"
|
|
TraceAddStreamEvt TraceEvtTyp = "add_stream"
|
|
TraceBlockAddStreamEvt TraceEvtTyp = "block_add_stream"
|
|
TraceRemoveStreamEvt TraceEvtTyp = "remove_stream"
|
|
TraceAddConnEvt TraceEvtTyp = "add_conn"
|
|
TraceBlockAddConnEvt TraceEvtTyp = "block_add_conn"
|
|
TraceRemoveConnEvt TraceEvtTyp = "remove_conn"
|
|
)
|
|
|
|
type scopeClass struct {
|
|
name string
|
|
}
|
|
|
|
func (s scopeClass) MarshalJSON() ([]byte, error) {
|
|
name := s.name
|
|
var span string
|
|
if idx := strings.Index(name, "span:"); idx > -1 {
|
|
name = name[:idx-1]
|
|
span = name[idx+5:]
|
|
}
|
|
// System and Transient scope
|
|
if name == "system" || name == "transient" || name == "allowlistedSystem" || name == "allowlistedTransient" {
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: name,
|
|
Span: span,
|
|
})
|
|
}
|
|
// Connection scope
|
|
if strings.HasPrefix(name, "conn-") {
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Conn string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "conn",
|
|
Conn: name[5:],
|
|
Span: span,
|
|
})
|
|
}
|
|
// Stream scope
|
|
if strings.HasPrefix(name, "stream-") {
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Stream string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "stream",
|
|
Stream: name[7:],
|
|
Span: span,
|
|
})
|
|
}
|
|
// Peer scope
|
|
if strings.HasPrefix(name, "peer:") {
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Peer string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "peer",
|
|
Peer: name[5:],
|
|
Span: span,
|
|
})
|
|
}
|
|
|
|
if strings.HasPrefix(name, "service:") {
|
|
if idx := strings.Index(name, "peer:"); idx > 0 { // Peer-Service scope
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Service string
|
|
Peer string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "service-peer",
|
|
Service: name[8 : idx-1],
|
|
Peer: name[idx+5:],
|
|
Span: span,
|
|
})
|
|
} else { // Service scope
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Service string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "service",
|
|
Service: name[8:],
|
|
Span: span,
|
|
})
|
|
}
|
|
}
|
|
|
|
if strings.HasPrefix(name, "protocol:") {
|
|
if idx := strings.Index(name, "peer:"); idx > -1 { // Peer-Protocol scope
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Protocol string
|
|
Peer string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "protocol-peer",
|
|
Protocol: name[9 : idx-1],
|
|
Peer: name[idx+5:],
|
|
Span: span,
|
|
})
|
|
} else { // Protocol scope
|
|
return json.Marshal(struct {
|
|
Class string
|
|
Protocol string
|
|
Span string `json:",omitempty"`
|
|
}{
|
|
Class: "protocol",
|
|
Protocol: name[9:],
|
|
Span: span,
|
|
})
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("unrecognized scope: %s", name)
|
|
}
|
|
|
|
type TraceEvt struct {
|
|
Time string
|
|
Type TraceEvtTyp
|
|
|
|
Scope *scopeClass `json:",omitempty"`
|
|
Name string `json:",omitempty"`
|
|
|
|
Limit interface{} `json:",omitempty"`
|
|
|
|
Priority uint8 `json:",omitempty"`
|
|
|
|
Delta int64 `json:",omitempty"`
|
|
DeltaIn int `json:",omitempty"`
|
|
DeltaOut int `json:",omitempty"`
|
|
|
|
Memory int64 `json:",omitempty"`
|
|
|
|
StreamsIn int `json:",omitempty"`
|
|
StreamsOut int `json:",omitempty"`
|
|
|
|
ConnsIn int `json:",omitempty"`
|
|
ConnsOut int `json:",omitempty"`
|
|
|
|
FD int `json:",omitempty"`
|
|
}
|
|
|
|
func (t *trace) push(evt TraceEvt) {
|
|
t.mx.Lock()
|
|
|
|
if t.done {
|
|
t.mx.Unlock()
|
|
return
|
|
}
|
|
evt.Time = time.Now().Format(time.RFC3339Nano)
|
|
if evt.Name != "" {
|
|
evt.Scope = &scopeClass{name: evt.Name}
|
|
}
|
|
|
|
for _, reporter := range t.reporters {
|
|
reporter.ConsumeEvent(evt)
|
|
}
|
|
|
|
if t.path != "" {
|
|
t.pendingWrites = append(t.pendingWrites, evt)
|
|
}
|
|
t.mx.Unlock()
|
|
}
|
|
|
|
func (t *trace) backgroundWriter(out io.WriteCloser) {
|
|
gzOut := gzip.NewWriter(out)
|
|
jsonOut := json.NewEncoder(gzOut)
|
|
ticker := time.NewTicker(time.Second)
|
|
|
|
var pend []interface{}
|
|
|
|
getEvents := func() {
|
|
t.mx.Lock()
|
|
tmp := t.pendingWrites
|
|
t.pendingWrites = pend[:0]
|
|
pend = tmp
|
|
t.mx.Unlock()
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
getEvents()
|
|
|
|
if len(pend) == 0 {
|
|
continue
|
|
}
|
|
|
|
if err := t.writeEvents(pend, jsonOut); err != nil {
|
|
log.Warnf("error writing rcmgr trace: %s", err)
|
|
t.mx.Lock()
|
|
t.done = true
|
|
t.mx.Unlock()
|
|
|
|
ticker.Stop()
|
|
gzOut.Close()
|
|
out.Close()
|
|
t.wg.Done()
|
|
return
|
|
}
|
|
|
|
if err := gzOut.Flush(); err != nil {
|
|
log.Warnf("error flushing rcmgr trace: %s", err)
|
|
t.mx.Lock()
|
|
t.done = true
|
|
t.mx.Unlock()
|
|
|
|
ticker.Stop()
|
|
gzOut.Close()
|
|
out.Close()
|
|
t.wg.Done()
|
|
return
|
|
}
|
|
|
|
case <-t.ctx.Done():
|
|
getEvents()
|
|
|
|
if len(pend) == 0 {
|
|
ticker.Stop()
|
|
gzOut.Close()
|
|
out.Close()
|
|
t.wg.Done()
|
|
return
|
|
}
|
|
|
|
if err := t.writeEvents(pend, jsonOut); err != nil {
|
|
log.Warnf("error writing rcmgr trace: %s", err)
|
|
ticker.Stop()
|
|
gzOut.Close()
|
|
out.Close()
|
|
t.wg.Done()
|
|
return
|
|
}
|
|
|
|
if err := gzOut.Flush(); err != nil {
|
|
log.Warnf("error flushing rcmgr trace: %s", err)
|
|
}
|
|
|
|
ticker.Stop()
|
|
gzOut.Close()
|
|
out.Close()
|
|
t.wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (t *trace) writeEvents(pend []interface{}, jout *json.Encoder) error {
|
|
for _, e := range pend {
|
|
if err := jout.Encode(e); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (t *trace) Start(limits Limiter) error {
|
|
if t == nil {
|
|
return nil
|
|
}
|
|
|
|
t.ctx, t.cancel = context.WithCancel(context.Background())
|
|
|
|
if t.path != "" {
|
|
out, err := os.OpenFile(t.path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
t.wg.Add(1)
|
|
go t.backgroundWriter(out)
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceStartEvt,
|
|
Limit: limits,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
func (t *trace) Close() error {
|
|
if t == nil {
|
|
return nil
|
|
}
|
|
|
|
t.mx.Lock()
|
|
|
|
if t.done {
|
|
t.mx.Unlock()
|
|
return nil
|
|
}
|
|
|
|
t.cancel()
|
|
t.done = true
|
|
t.mx.Unlock()
|
|
|
|
t.wg.Wait()
|
|
return nil
|
|
}
|
|
|
|
func (t *trace) CreateScope(scope string, limit Limit) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceCreateScopeEvt,
|
|
Name: scope,
|
|
Limit: limit,
|
|
})
|
|
}
|
|
|
|
func (t *trace) DestroyScope(scope string) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceDestroyScopeEvt,
|
|
Name: scope,
|
|
})
|
|
}
|
|
|
|
func (t *trace) ReserveMemory(scope string, prio uint8, size, mem int64) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if size == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceReserveMemoryEvt,
|
|
Name: scope,
|
|
Priority: prio,
|
|
Delta: size,
|
|
Memory: mem,
|
|
})
|
|
}
|
|
|
|
func (t *trace) BlockReserveMemory(scope string, prio uint8, size, mem int64) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if size == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceBlockReserveMemoryEvt,
|
|
Name: scope,
|
|
Priority: prio,
|
|
Delta: size,
|
|
Memory: mem,
|
|
})
|
|
}
|
|
|
|
func (t *trace) ReleaseMemory(scope string, size, mem int64) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if size == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceReleaseMemoryEvt,
|
|
Name: scope,
|
|
Delta: -size,
|
|
Memory: mem,
|
|
})
|
|
}
|
|
|
|
func (t *trace) AddStream(scope string, dir network.Direction, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut int
|
|
if dir == network.DirInbound {
|
|
deltaIn = 1
|
|
} else {
|
|
deltaOut = 1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceAddStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) BlockAddStream(scope string, dir network.Direction, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut int
|
|
if dir == network.DirInbound {
|
|
deltaIn = 1
|
|
} else {
|
|
deltaOut = 1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceBlockAddStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) RemoveStream(scope string, dir network.Direction, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut int
|
|
if dir == network.DirInbound {
|
|
deltaIn = -1
|
|
} else {
|
|
deltaOut = -1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceRemoveStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) AddStreams(scope string, deltaIn, deltaOut, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceAddStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) BlockAddStreams(scope string, deltaIn, deltaOut, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceBlockAddStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) RemoveStreams(scope string, deltaIn, deltaOut, nstreamsIn, nstreamsOut int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceRemoveStreamEvt,
|
|
Name: scope,
|
|
DeltaIn: -deltaIn,
|
|
DeltaOut: -deltaOut,
|
|
StreamsIn: nstreamsIn,
|
|
StreamsOut: nstreamsOut,
|
|
})
|
|
}
|
|
|
|
func (t *trace) AddConn(scope string, dir network.Direction, usefd bool, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut, deltafd int
|
|
if dir == network.DirInbound {
|
|
deltaIn = 1
|
|
} else {
|
|
deltaOut = 1
|
|
}
|
|
if usefd {
|
|
deltafd = 1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceAddConnEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
Delta: int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|
|
|
|
func (t *trace) BlockAddConn(scope string, dir network.Direction, usefd bool, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut, deltafd int
|
|
if dir == network.DirInbound {
|
|
deltaIn = 1
|
|
} else {
|
|
deltaOut = 1
|
|
}
|
|
if usefd {
|
|
deltafd = 1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceBlockAddConnEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
Delta: int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|
|
|
|
func (t *trace) RemoveConn(scope string, dir network.Direction, usefd bool, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
var deltaIn, deltaOut, deltafd int
|
|
if dir == network.DirInbound {
|
|
deltaIn = -1
|
|
} else {
|
|
deltaOut = -1
|
|
}
|
|
if usefd {
|
|
deltafd = -1
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceRemoveConnEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
Delta: int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|
|
|
|
func (t *trace) AddConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 && deltafd == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceAddConnEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
Delta: int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|
|
|
|
func (t *trace) BlockAddConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 && deltafd == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceBlockAddConnEvt,
|
|
Name: scope,
|
|
DeltaIn: deltaIn,
|
|
DeltaOut: deltaOut,
|
|
Delta: int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|
|
|
|
func (t *trace) RemoveConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn, nconnsOut, nfd int) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
|
|
if deltaIn == 0 && deltaOut == 0 && deltafd == 0 {
|
|
return
|
|
}
|
|
|
|
t.push(TraceEvt{
|
|
Type: TraceRemoveConnEvt,
|
|
Name: scope,
|
|
DeltaIn: -deltaIn,
|
|
DeltaOut: -deltaOut,
|
|
Delta: -int64(deltafd),
|
|
ConnsIn: nconnsIn,
|
|
ConnsOut: nconnsOut,
|
|
FD: nfd,
|
|
})
|
|
}
|