fix: nodes with long-tail syncs or not in validator trie hit OOM due to accumulation of state transition messages that do not clear

This commit is contained in:
Cassandra Heart 2024-03-16 19:33:48 -05:00
parent 3f867d80f6
commit 7ccd9f9ab0
No known key found for this signature in database
GPG Key ID: 6352152859385958
3 changed files with 25 additions and 18 deletions

View File

@ -460,17 +460,17 @@ func (e *CeremonyDataClockConsensusEngine) runLoop() {
e.frameChan <- latestFrame e.frameChan <- latestFrame
}() }()
var nextFrame *protobufs.ClockFrame
if nextFrame, err = e.prove(latestFrame); err != nil {
e.logger.Error("could not prove", zap.Error(err))
e.state = consensus.EngineStateCollecting
continue
}
if bytes.Equal( if bytes.Equal(
e.frameProverTrie.FindNearest(e.provingKeyAddress).External.Key, e.frameProverTrie.FindNearest(e.provingKeyAddress).External.Key,
e.provingKeyAddress, e.provingKeyAddress,
) { ) {
var nextFrame *protobufs.ClockFrame
if nextFrame, err = e.prove(latestFrame); err != nil {
e.logger.Error("could not prove", zap.Error(err))
e.state = consensus.EngineStateCollecting
continue
}
e.dataTimeReel.Insert(nextFrame) e.dataTimeReel.Insert(nextFrame)
if err = e.publishProof(nextFrame); err != nil { if err = e.publishProof(nextFrame); err != nil {
@ -502,14 +502,17 @@ func (e *CeremonyDataClockConsensusEngine) runLoop() {
e.frameChan <- latestFrame e.frameChan <- latestFrame
}() }()
if e.frameProverTrie.Contains(e.provingKeyAddress) { var nextFrame *protobufs.ClockFrame
var nextFrame *protobufs.ClockFrame if nextFrame, err = e.prove(latestFrame); err != nil {
if nextFrame, err = e.prove(latestFrame); err != nil { e.logger.Error("could not prove", zap.Error(err))
e.logger.Error("could not prove", zap.Error(err)) e.state = consensus.EngineStateCollecting
e.state = consensus.EngineStateCollecting continue
continue }
}
if bytes.Equal(
e.frameProverTrie.FindNearest(e.provingKeyAddress).External.Key,
e.provingKeyAddress,
) {
e.dataTimeReel.Insert(nextFrame) e.dataTimeReel.Insert(nextFrame)
if err = e.publishProof(nextFrame); err != nil { if err = e.publishProof(nextFrame); err != nil {

View File

@ -454,9 +454,6 @@ func (e *CeremonyDataClockConsensusEngine) collect(
e.logger.Info("collecting vdf proofs") e.logger.Info("collecting vdf proofs")
latest := currentFramePublished latest := currentFramePublished
if e.syncingStatus == SyncStatusFailed {
e.syncingStatus = SyncStatusNotSyncing
}
// With the increase of network size, constrain down to top thirty // With the increase of network size, constrain down to top thirty
for i := 0; i < 30; i++ { for i := 0; i < 30; i++ {
@ -468,6 +465,7 @@ func (e *CeremonyDataClockConsensusEngine) collect(
e.logger.Info("currently up to date, skipping sync") e.logger.Info("currently up to date, skipping sync")
break break
} else if maxFrame-2 > latest.FrameNumber { } else if maxFrame-2 > latest.FrameNumber {
e.syncingStatus = SyncStatusSynchronizing
latest, err = e.sync(latest, maxFrame, peerId) latest, err = e.sync(latest, maxFrame, peerId)
if err == nil { if err == nil {
break break
@ -475,6 +473,8 @@ func (e *CeremonyDataClockConsensusEngine) collect(
} }
} }
e.syncingStatus = SyncStatusNotSyncing
if latest.FrameNumber < currentFramePublished.FrameNumber { if latest.FrameNumber < currentFramePublished.FrameNumber {
latest = currentFramePublished latest = currentFramePublished
} }

View File

@ -30,7 +30,11 @@ func (e *CeremonyDataClockConsensusEngine) runMessageHandler() {
peer, ok := e.peerMap[string(message.From)] peer, ok := e.peerMap[string(message.From)]
e.peerMapMx.RUnlock() e.peerMapMx.RUnlock()
if ok && bytes.Compare(peer.version, config.GetMinimumVersion()) >= 0 { if ok && bytes.Compare(peer.version, config.GetMinimumVersion()) >= 0 &&
bytes.Equal(
e.frameProverTrie.FindNearest(e.provingKeyAddress).External.Key,
e.provingKeyAddress,
) && e.syncingStatus == SyncStatusNotSyncing {
for name := range e.executionEngines { for name := range e.executionEngines {
name := name name := name
go func() error { go func() error {