This commit is contained in:
Cassandra Heart 2024-02-17 22:52:19 -06:00 committed by GitHub
parent 0587e7d63d
commit bfbd35aae0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 199 additions and 153 deletions

View File

@ -169,7 +169,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
) )
if err != nil { if err != nil {
m.errorMsg = hex.EncodeToString( m.errorMsg = hex.EncodeToString(
selector.Bytes(), selector.FillBytes(make([]byte, 32)),
) + ":" + err.Error() ) + ":" + err.Error()
break break
} }
@ -286,7 +286,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
) )
if err != nil { if err != nil {
m.errorMsg = hex.EncodeToString( m.errorMsg = hex.EncodeToString(
selector.Bytes(), selector.FillBytes(make([]byte, 32)),
) + ":" + err.Error() ) + ":" + err.Error()
break break
} }
@ -522,7 +522,7 @@ func (m model) View() string {
explorerContent = fmt.Sprintf( explorerContent = fmt.Sprintf(
"Frame %d (Selector: %x, %s):\n\tParent: %x\n\tVDF Proof: %x\n", "Frame %d (Selector: %x, %s):\n\tParent: %x\n\tVDF Proof: %x\n",
m.frame.FrameNumber, m.frame.FrameNumber,
selector.Bytes(), selector.FillBytes(make([]byte, 32)),
committed, committed,
m.frame.ParentSelector, m.frame.ParentSelector,
m.frame.Input[:516], m.frame.Input[:516],
@ -886,11 +886,11 @@ func logoVersion(width int) string {
out += " ####################################### ########\n" out += " ####################################### ########\n"
out += " ############################# ##\n" out += " ############################# ##\n"
out += " \n" out += " \n"
out += " Quilibrium Node - v1.2.7 Dawn\n" out += " Quilibrium Node - v1.2.8 Dawn\n"
out += " \n" out += " \n"
out += " DB Console\n" out += " DB Console\n"
} else { } else {
out = "Quilibrium Node - v1.2.7 Dawn - DB Console\n" out = "Quilibrium Node - v1.2.8 Dawn - DB Console\n"
} }
return out return out
} }

View File

@ -59,5 +59,5 @@ func GetMinimumVersion() []byte {
} }
func GetVersion() []byte { func GetVersion() []byte {
return []byte{0x01, 0x02, 0x07} return []byte{0x01, 0x02, 0x08}
} }

View File

@ -2,6 +2,7 @@ package time
import ( import (
"bytes" "bytes"
"encoding/hex"
"math/big" "math/big"
"sync" "sync"
@ -30,8 +31,8 @@ var unknownDistance = new(big.Int).SetBytes([]byte{
}) })
type pendingFrame struct { type pendingFrame struct {
selector *big.Int
parentSelector *big.Int parentSelector *big.Int
distance *big.Int
} }
type DataTimeReel struct { type DataTimeReel struct {
@ -154,6 +155,12 @@ func (d *DataTimeReel) Insert(frame *protobufs.ClockFrame) error {
return nil return nil
} }
d.logger.Debug(
"insert frame",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("output_tag", hex.EncodeToString(frame.Output[:64])),
)
if d.lruFrames.Contains(string(frame.Output[:64])) { if d.lruFrames.Contains(string(frame.Output[:64])) {
return nil return nil
} }
@ -243,8 +250,17 @@ func (d *DataTimeReel) runLoop() {
for { for {
select { select {
case frame := <-d.frames: case frame := <-d.frames:
d.logger.Debug(
"processing frame",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("output_tag", hex.EncodeToString(frame.Output[:64])),
zap.Uint64("head_number", d.head.FrameNumber),
zap.String("head_output_tag", hex.EncodeToString(d.head.Output[:64])),
)
// Most common scenario: in order new frame is higher number // Most common scenario: in order new frame is higher number
if d.head.FrameNumber < frame.FrameNumber { if d.head.FrameNumber < frame.FrameNumber {
d.logger.Debug("frame is higher")
parent := new(big.Int).SetBytes(frame.ParentSelector) parent := new(big.Int).SetBytes(frame.ParentSelector)
selector, err := frame.GetSelector() selector, err := frame.GetSelector()
if err != nil { if err != nil {
@ -253,6 +269,8 @@ func (d *DataTimeReel) runLoop() {
distance, err := d.GetDistance(frame) distance, err := d.GetDistance(frame)
if err != nil { if err != nil {
d.logger.Debug("no master, add pending")
// If the frame arrived ahead of a master, e.g. the master data is not // If the frame arrived ahead of a master, e.g. the master data is not
// synced, we'll go ahead and mark it as pending and process it when // synced, we'll go ahead and mark it as pending and process it when
// we can, but if we had a general fault, panic: // we can, but if we had a general fault, panic:
@ -260,7 +278,7 @@ func (d *DataTimeReel) runLoop() {
panic(err) panic(err)
} }
d.addPending(selector, parent, unknownDistance, frame) d.addPending(selector, parent, distance, frame)
continue continue
} }
@ -273,16 +291,25 @@ func (d *DataTimeReel) runLoop() {
// pending: // pending:
if frame.FrameNumber-d.head.FrameNumber != 1 || if frame.FrameNumber-d.head.FrameNumber != 1 ||
parent.Cmp(headSelector) != 0 { parent.Cmp(headSelector) != 0 {
d.logger.Debug(
"frame has has gap or is non-descendent, add pending",
zap.Bool("has_gap", frame.FrameNumber-d.head.FrameNumber != 1),
zap.String("parent_selector", parent.Text(16)),
zap.String("head_selector", headSelector.Text(16)),
)
d.addPending(selector, parent, distance, frame) d.addPending(selector, parent, distance, frame)
continue continue
} }
// Otherwise set it as the next and process all pending // Otherwise set it as the next and process all pending
d.setHead(frame, distance) d.setHead(frame, distance)
d.processPending(frame) d.processPending(d.head)
} else if d.head.FrameNumber == frame.FrameNumber { } else if d.head.FrameNumber == frame.FrameNumber {
// frames are equivalent, no need to act // frames are equivalent, no need to act
if bytes.Equal(d.head.Output, frame.Output) { if bytes.Equal(d.head.Output, frame.Output) {
d.logger.Debug("equivalent frame")
d.processPending(d.head)
continue continue
} }
@ -290,11 +317,19 @@ func (d *DataTimeReel) runLoop() {
if err != nil { if err != nil {
panic(err) panic(err)
} }
d.logger.Debug(
"frame is same height",
zap.String("head_distance", d.headDistance.Text(16)),
zap.String("distance", distance.Text(16)),
)
// Optimization: if competing frames share a parent we can short-circuit // Optimization: if competing frames share a parent we can short-circuit
// fork choice // fork choice
if bytes.Equal(d.head.ParentSelector, frame.ParentSelector) && if bytes.Equal(d.head.ParentSelector, frame.ParentSelector) &&
distance.Cmp(d.headDistance) < 0 { distance.Cmp(d.headDistance) < 0 {
d.logger.Debug(
"frame shares parent, has shorter distance, short circuit",
)
d.totalDistance.Sub(d.totalDistance, d.headDistance) d.totalDistance.Sub(d.totalDistance, d.headDistance)
d.setHead(frame, distance) d.setHead(frame, distance)
d.processPending(d.head) d.processPending(d.head)
@ -305,6 +340,8 @@ func (d *DataTimeReel) runLoop() {
d.forkChoice(frame, distance) d.forkChoice(frame, distance)
d.processPending(d.head) d.processPending(d.head)
} else { } else {
d.logger.Debug("frame is lower height")
// tag: dusk we should have some kind of check here to avoid brutal // tag: dusk we should have some kind of check here to avoid brutal
// thrashing // thrashing
existing, _, err := d.clockStore.GetDataClockFrame( existing, _, err := d.clockStore.GetDataClockFrame(
@ -320,6 +357,8 @@ func (d *DataTimeReel) runLoop() {
// It's a fork, but it's behind. We need to stash it until it catches // It's a fork, but it's behind. We need to stash it until it catches
// up (or dies off) // up (or dies off)
if !bytes.Equal(existing.Output, frame.Output) { if !bytes.Equal(existing.Output, frame.Output) {
d.logger.Debug("is fork, add pending")
distance, err := d.GetDistance(frame) distance, err := d.GetDistance(frame)
if err != nil { if err != nil {
panic(err) panic(err)
@ -346,20 +385,27 @@ func (d *DataTimeReel) addPending(
distance *big.Int, distance *big.Int,
frame *protobufs.ClockFrame, frame *protobufs.ClockFrame,
) { ) {
d.logger.Debug(
"add pending",
zap.Uint64("head_frame_number", d.head.FrameNumber),
zap.Uint64("add_frame_number", frame.FrameNumber),
zap.String("selector", selector.Text(16)),
zap.String("parent", parent.Text(16)),
zap.String("distance", distance.Text(16)),
)
if d.head.FrameNumber <= frame.FrameNumber {
if _, ok := d.pending[frame.FrameNumber]; !ok { if _, ok := d.pending[frame.FrameNumber]; !ok {
d.pending[frame.FrameNumber] = []*pendingFrame{} d.pending[frame.FrameNumber] = []*pendingFrame{}
} }
// avoid heavy thrashing // avoid heavy thrashing
for _, frame := range d.pending[frame.FrameNumber] { for _, frame := range d.pending[frame.FrameNumber] {
if frame.parentSelector.Cmp(parent) == 0 { if frame.selector.Cmp(selector) == 0 {
d.logger.Debug("exists in pending already")
return return
} }
} }
if distance.Cmp(unknownDistance) == 0 {
distance = new(big.Int).Set(unknownDistance)
distance.Sub(distance, big.NewInt(int64(len(d.pending[frame.FrameNumber]))))
} }
// avoid db thrashing // avoid db thrashing
@ -368,6 +414,14 @@ func (d *DataTimeReel) addPending(
frame.FrameNumber, frame.FrameNumber,
selector.FillBytes(make([]byte, 32)), selector.FillBytes(make([]byte, 32)),
); err != nil && existing == nil { ); err != nil && existing == nil {
d.logger.Debug(
"not stored yet, save data candidate",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("selector", selector.Text(16)),
zap.String("parent", parent.Text(16)),
zap.String("distance", distance.Text(16)),
)
txn, err := d.clockStore.NewTransaction() txn, err := d.clockStore.NewTransaction()
if err != nil { if err != nil {
panic(err) panic(err)
@ -388,119 +442,90 @@ func (d *DataTimeReel) addPending(
} }
} }
if d.head.FrameNumber <= frame.FrameNumber {
d.logger.Debug(
"accumulate in pending",
zap.Int("pending_neighbors", len(d.pending[frame.FrameNumber])),
)
d.pending[frame.FrameNumber] = append( d.pending[frame.FrameNumber] = append(
d.pending[frame.FrameNumber], d.pending[frame.FrameNumber],
&pendingFrame{ &pendingFrame{
selector: selector,
parentSelector: parent, parentSelector: parent,
distance: distance,
}, },
) )
}
} }
func (d *DataTimeReel) processPending(frame *protobufs.ClockFrame) { func (d *DataTimeReel) processPending(
neighbors := false frame *protobufs.ClockFrame,
// Flush the current pending frames ) {
neighborPending, ok := d.pending[frame.FrameNumber] d.logger.Debug("process pending")
for ok && neighborPending != nil { for f, nextPending := range d.pending {
next := neighborPending[0] d.logger.Debug(
d.pending[frame.FrameNumber] = "checking frame set",
d.pending[frame.FrameNumber][1:] zap.Uint64("pending_frame_number", f),
if len(d.pending[frame.FrameNumber]) == 0 { zap.Uint64("frame_number", frame.FrameNumber),
delete(d.pending, frame.FrameNumber)
}
nextFrame, err := d.clockStore.GetCandidateDataClockFrame(
d.filter,
frame.FrameNumber,
next.parentSelector.FillBytes(make([]byte, 32)),
next.distance.FillBytes(make([]byte, 32)),
) )
if err != nil && !errors.Is(err, store.ErrNotFound) { if f < frame.FrameNumber {
panic(err) d.logger.Debug(
"purging frame set",
zap.Uint64("pending_frame_number", f),
zap.Uint64("frame_number", frame.FrameNumber),
)
delete(d.pending, f)
continue
} }
if nextFrame != nil {
neighbors = true
go func() {
d.frames <- nextFrame
}()
}
neighborPending, ok = d.pending[frame.FrameNumber]
}
above := false
if !neighbors {
// Pull the next // Pull the next
nextPending, ok := d.pending[frame.FrameNumber+1] if nextPending != nil {
if ok { d.logger.Debug("try process next")
next := nextPending[0] next := nextPending[0]
d.pending[frame.FrameNumber+1] = d.pending[f] =
d.pending[frame.FrameNumber+1][1:] d.pending[f][1:]
if len(d.pending[frame.FrameNumber+1]) == 0 {
delete(d.pending, frame.FrameNumber+1)
}
nextFrame, err := d.clockStore.GetCandidateDataClockFrame( nextFrame, err := d.clockStore.GetParentDataClockFrame(
d.filter, d.filter,
frame.FrameNumber+1, f,
next.parentSelector.FillBytes(make([]byte, 32)), next.selector.FillBytes(make([]byte, 32)),
next.distance.FillBytes(make([]byte, 32)),
) )
if err != nil && !errors.Is(err, store.ErrNotFound) { if err != nil && !errors.Is(err, store.ErrNotFound) {
panic(err) panic(err)
} }
if nextFrame != nil { if nextFrame != nil {
above = true d.pending[f] = append(d.pending[f], next)
d.logger.Debug("next found, send frame back in")
go func() { go func() {
d.frames <- nextFrame d.frames <- nextFrame
}() }()
} }
} if len(d.pending[f]) == 0 {
} d.logger.Debug("last next processing, clear list")
delete(d.pending, f)
if !above {
// Pull below
min := frame.FrameNumber
for k := range d.pending {
if k < min {
min = k
}
}
if min == frame.FrameNumber {
return
}
nextPending, ok := d.pending[min]
if ok {
next := nextPending[0]
d.pending[min] =
d.pending[min][1:]
if len(d.pending[min]) == 0 {
delete(d.pending, min)
}
nextFrame, err := d.clockStore.GetCandidateDataClockFrame(
d.filter,
min,
next.parentSelector.FillBytes(make([]byte, 32)),
next.distance.FillBytes(make([]byte, 32)),
)
if err != nil && !errors.Is(err, store.ErrNotFound) {
panic(err)
}
if nextFrame != nil {
go func() {
d.frames <- nextFrame
}()
} }
} }
} }
} }
func (d *DataTimeReel) setHead(frame *protobufs.ClockFrame, distance *big.Int) { func (d *DataTimeReel) setHead(frame *protobufs.ClockFrame, distance *big.Int) {
d.logger.Debug(
"set frame to head",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("output_tag", hex.EncodeToString(frame.Output[:64])),
zap.Uint64("head_number", d.head.FrameNumber),
zap.String("head_output_tag", hex.EncodeToString(d.head.Output[:64])),
)
txn, err := d.clockStore.NewTransaction() txn, err := d.clockStore.NewTransaction()
if err != nil { if err != nil {
panic(err) panic(err)
} }
d.logger.Debug(
"save data",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("distance", distance.Text(16)),
)
if err := d.clockStore.PutDataClockFrame( if err := d.clockStore.PutDataClockFrame(
frame, frame,
d.proverTrie, d.proverTrie,
@ -555,12 +580,12 @@ func (d *DataTimeReel) GetDistance(frame *protobufs.ClockFrame) (
allBitmaskFilter, allBitmaskFilter,
frame.FrameNumber) frame.FrameNumber)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "get distance") return unknownDistance, errors.Wrap(err, "get distance")
} }
masterSelector, err := master.GetSelector() masterSelector, err := master.GetSelector()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "get distance") return unknownDistance, errors.Wrap(err, "get distance")
} }
discriminatorNode := discriminatorNode :=
@ -568,7 +593,7 @@ func (d *DataTimeReel) GetDistance(frame *protobufs.ClockFrame) (
discriminator := discriminatorNode.External.Key discriminator := discriminatorNode.External.Key
addr, err := frame.GetAddress() addr, err := frame.GetAddress()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "get distance") return unknownDistance, errors.Wrap(err, "get distance")
} }
distance := new(big.Int).Sub( distance := new(big.Int).Sub(
new(big.Int).SetBytes(discriminator), new(big.Int).SetBytes(discriminator),
@ -583,6 +608,13 @@ func (d *DataTimeReel) forkChoice(
frame *protobufs.ClockFrame, frame *protobufs.ClockFrame,
distance *big.Int, distance *big.Int,
) { ) {
d.logger.Debug(
"fork choice",
zap.Uint64("frame_number", frame.FrameNumber),
zap.String("output_tag", hex.EncodeToString(frame.Output[:64])),
zap.Uint64("head_number", d.head.FrameNumber),
zap.String("head_output_tag", hex.EncodeToString(d.head.Output[:64])),
)
parentSelector, selector, err := frame.GetParentAndSelector() parentSelector, selector, err := frame.GetParentAndSelector()
if err != nil { if err != nil {
panic(err) panic(err)
@ -600,6 +632,8 @@ func (d *DataTimeReel) forkChoice(
// Walk backwards through the parents, until we find a matching parent // Walk backwards through the parents, until we find a matching parent
// selector: // selector:
for !bytes.Equal(left, right) { for !bytes.Equal(left, right) {
d.logger.Debug("scan backwards")
rightReplaySelectors = append( rightReplaySelectors = append(
append( append(
[][]byte{}, [][]byte{},
@ -646,11 +680,20 @@ func (d *DataTimeReel) forkChoice(
leftTotal.Add(leftTotal, leftIndexDistance) leftTotal.Add(leftTotal, leftIndexDistance)
rightTotal.Add(rightTotal, rightIndexDistance) rightTotal.Add(rightTotal, rightIndexDistance)
} }
d.logger.Debug("found mutual root")
frameNumber := rightIndex.FrameNumber frameNumber := rightIndex.FrameNumber
// Choose new fork based on lightest distance sub-tree // Choose new fork based on lightest distance sub-tree
if rightTotal.Cmp(leftTotal) < 0 { if rightTotal.Cmp(leftTotal) > 0 {
d.logger.Debug("proposed fork has greater distance",
zap.String("right_total", rightTotal.Text(16)),
zap.String("left_total", leftTotal.Text(16)),
)
d.addPending(selector, parentSelector, distance, frame)
return
}
for { for {
if len(rightReplaySelectors) == 0 { if len(rightReplaySelectors) == 0 {
break break
@ -711,10 +754,13 @@ func (d *DataTimeReel) forkChoice(
d.totalDistance.Sub(d.totalDistance, leftTotal) d.totalDistance.Sub(d.totalDistance, leftTotal)
d.totalDistance.Add(d.totalDistance, rightTotal) d.totalDistance.Add(d.totalDistance, rightTotal)
d.headDistance = distance d.headDistance = distance
d.logger.Debug(
"set total distance",
zap.String("total_distance", d.totalDistance.Text(16)),
)
go func() { go func() {
d.newFrameCh <- frame d.newFrameCh <- frame
}() }()
}
} }
var _ TimeReel = (*DataTimeReel)(nil) var _ TimeReel = (*DataTimeReel)(nil)

View File

@ -271,7 +271,7 @@ func (w *WesolowskiFrameProver) ProveDataClockFrame(
FrameNumber: previousFrame.FrameNumber + 1, FrameNumber: previousFrame.FrameNumber + 1,
Timestamp: timestamp, Timestamp: timestamp,
Difficulty: difficulty, Difficulty: difficulty,
ParentSelector: parent.Bytes(), ParentSelector: parent.FillBytes(make([]byte, 32)),
Input: append( Input: append(
append([]byte{}, previousFrame.Output...), append([]byte{}, previousFrame.Output...),
commitmentInput..., commitmentInput...,

View File

@ -284,5 +284,5 @@ func printLogo() {
func printVersion() { func printVersion() {
fmt.Println(" ") fmt.Println(" ")
fmt.Println(" Quilibrium Node - v1.2.7 Dawn") fmt.Println(" Quilibrium Node - v1.2.8 Dawn")
} }