This commit is contained in:
Cassandra Heart 2024-02-16 03:42:37 -06:00 committed by GitHub
parent ac5c393745
commit d499a60937
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 147 additions and 35 deletions

View File

@ -886,11 +886,11 @@ func logoVersion(width int) string {
out += " ####################################### ########\n" out += " ####################################### ########\n"
out += " ############################# ##\n" out += " ############################# ##\n"
out += " \n" out += " \n"
out += " Quilibrium Node - v1.2.5 Dawn\n" out += " Quilibrium Node - v1.2.6 Dawn\n"
out += " \n" out += " \n"
out += " DB Console\n" out += " DB Console\n"
} else { } else {
out = "Quilibrium Node - v1.2.5 Dawn - DB Console\n" out = "Quilibrium Node - v1.2.6 Dawn - DB Console\n"
} }
return out return out
} }

View File

@ -201,15 +201,12 @@ func (e *CeremonyDataClockConsensusEngine) sync(
from = 1 from = 1
} }
if maxFrame > from && maxFrame > 3 {
from = maxFrame - 2
}
s, err := client.GetCompressedSyncFrames( s, err := client.GetCompressedSyncFrames(
context.Background(), context.Background(),
&protobufs.ClockFramesRequest{ &protobufs.ClockFramesRequest{
Filter: e.filter, Filter: e.filter,
FromFrameNumber: from, FromFrameNumber: from,
ToFrameNumber: from + 8,
}, },
grpc.MaxCallRecvMsgSize(600*1024*1024), grpc.MaxCallRecvMsgSize(600*1024*1024),
) )

View File

@ -59,5 +59,5 @@ func GetMinimumVersion() []byte {
} }
func GetVersion() []byte { func GetVersion() []byte {
return []byte{0x01, 0x02, 0x05} return []byte{0x01, 0x02, 0x06}
} }

View File

@ -238,8 +238,15 @@ func (d *DataTimeReel) runLoop() {
continue continue
} }
// If the frame has a gap from the head, mark it as pending: headSelector, err := d.head.GetSelector()
if frame.FrameNumber-d.head.FrameNumber != 1 { if err != nil {
panic(err)
}
// If the frame has a gap from the head or is not descendent, mark it as
// pending:
if frame.FrameNumber-d.head.FrameNumber != 1 ||
parent.Cmp(headSelector) != 0 {
d.addPending(selector, parent, distance, frame) d.addPending(selector, parent, distance, frame)
continue continue
} }
@ -264,10 +271,13 @@ func (d *DataTimeReel) runLoop() {
distance.Cmp(d.headDistance) < 0 { distance.Cmp(d.headDistance) < 0 {
d.totalDistance.Sub(d.totalDistance, d.headDistance) d.totalDistance.Sub(d.totalDistance, d.headDistance)
d.setHead(frame, distance) d.setHead(frame, distance)
d.processPending(d.head)
continue
} }
// Choose fork // Choose fork
d.forkChoice(frame, distance) d.forkChoice(frame, distance)
d.processPending(d.head)
} else { } else {
// tag: dusk we should have some kind of check here to avoid brutal // tag: dusk we should have some kind of check here to avoid brutal
// thrashing // thrashing
@ -359,7 +369,6 @@ func (d *DataTimeReel) processPending(frame *protobufs.ClockFrame) {
d.pending[frame.FrameNumber][1:] d.pending[frame.FrameNumber][1:]
if len(d.pending[frame.FrameNumber]) == 0 { if len(d.pending[frame.FrameNumber]) == 0 {
delete(d.pending, frame.FrameNumber) delete(d.pending, frame.FrameNumber)
break
} }
nextFrame, err := d.clockStore.GetCandidateDataClockFrame( nextFrame, err := d.clockStore.GetCandidateDataClockFrame(
@ -380,6 +389,7 @@ func (d *DataTimeReel) processPending(frame *protobufs.ClockFrame) {
neighborPending, ok = d.pending[frame.FrameNumber] neighborPending, ok = d.pending[frame.FrameNumber]
} }
above := false
if !neighbors { if !neighbors {
// Pull the next // Pull the next
nextPending, ok := d.pending[frame.FrameNumber+1] nextPending, ok := d.pending[frame.FrameNumber+1]
@ -400,6 +410,44 @@ func (d *DataTimeReel) processPending(frame *protobufs.ClockFrame) {
if err != nil && !errors.Is(err, store.ErrNotFound) { if err != nil && !errors.Is(err, store.ErrNotFound) {
panic(err) panic(err)
} }
if nextFrame != nil {
above = true
go func() {
d.frames <- nextFrame
}()
}
}
}
if !above {
// Pull below
min := frame.FrameNumber
for k := range d.pending {
if k < min {
min = k
}
}
if min == frame.FrameNumber {
return
}
nextPending, ok := d.pending[min]
if ok {
next := nextPending[0]
d.pending[min] =
d.pending[min][1:]
if len(d.pending[min]) == 0 {
delete(d.pending, min)
}
nextFrame, err := d.clockStore.GetCandidateDataClockFrame(
d.filter,
min,
next.parentSelector.FillBytes(make([]byte, 32)),
next.distance.FillBytes(make([]byte, 32)),
)
if err != nil && !errors.Is(err, store.ErrNotFound) {
panic(err)
}
if nextFrame != nil { if nextFrame != nil {
go func() { go func() {
d.frames <- nextFrame d.frames <- nextFrame
@ -509,7 +557,7 @@ func (d *DataTimeReel) forkChoice(
left := d.head.ParentSelector left := d.head.ParentSelector
right := frame.ParentSelector right := frame.ParentSelector
rightReplaySelectors := [][]byte{selector.FillBytes(make([]byte, 32))} rightReplaySelectors := [][]byte{}
// Walk backwards through the parents, until we find a matching parent // Walk backwards through the parents, until we find a matching parent
// selector: // selector:
@ -561,6 +609,8 @@ func (d *DataTimeReel) forkChoice(
rightTotal.Add(rightTotal, rightIndexDistance) rightTotal.Add(rightTotal, rightIndexDistance)
} }
frameNumber := rightIndex.FrameNumber
// Choose new fork based on lightest distance sub-tree // Choose new fork based on lightest distance sub-tree
if rightTotal.Cmp(leftTotal) < 0 { if rightTotal.Cmp(leftTotal) < 0 {
for { for {
@ -568,12 +618,12 @@ func (d *DataTimeReel) forkChoice(
break break
} }
next := rightReplaySelectors[0] next := rightReplaySelectors[0]
rightReplaySelectors[frame.FrameNumber] = rightReplaySelectors =
rightReplaySelectors[frame.FrameNumber][1:] rightReplaySelectors[1:]
rightIndex, err = d.clockStore.GetParentDataClockFrame( rightIndex, err = d.clockStore.GetParentDataClockFrame(
d.filter, d.filter,
rightIndex.FrameNumber+1, frameNumber,
next, next,
) )
if err != nil { if err != nil {
@ -597,6 +647,26 @@ func (d *DataTimeReel) forkChoice(
if err = txn.Commit(); err != nil { if err = txn.Commit(); err != nil {
panic(err) panic(err)
} }
frameNumber++
}
txn, err := d.clockStore.NewTransaction()
if err != nil {
panic(err)
}
if err := d.clockStore.PutDataClockFrame(
frame,
d.proverTrie,
txn,
false,
); err != nil {
panic(err)
}
if err = txn.Commit(); err != nil {
panic(err)
} }
d.head = frame d.head = frame

View File

@ -1,11 +1,11 @@
package time_test package time_test
import ( import (
"encoding/hex"
"fmt" "fmt"
"strings" "strings"
"sync" "sync"
"testing" "testing"
gotime "time"
"github.com/cloudflare/circl/sign/ed448" "github.com/cloudflare/circl/sign/ed448"
"github.com/iden3/go-iden3-crypto/poseidon" "github.com/iden3/go-iden3-crypto/poseidon"
@ -118,7 +118,7 @@ func TestDataTimeReel(t *testing.T) {
addrMap, addrMap,
proverTrie := generateTestProvers() proverTrie := generateTestProvers()
// We're going to set this up by churning 100 master frames so we don't // We're going to set this up by churning 40 master frames so we don't
// have to zig zag on master and data frames to confirm data time reel // have to zig zag on master and data frames to confirm data time reel
// behaviors // behaviors
m := time.NewMasterTimeReel( m := time.NewMasterTimeReel(
@ -143,14 +143,14 @@ func TestDataTimeReel(t *testing.T) {
wg.Add(1) wg.Add(1)
frameCh := m.NewFrameCh() frameCh := m.NewFrameCh()
go func() { go func() {
for i := 0; i < 100; i++ { for i := 0; i < 40; i++ {
frames = append(frames, <-frameCh) frames = append(frames, <-frameCh)
} }
wg.Done() wg.Done()
}() }()
// in order // in order
for i := int64(0); i < 100; i++ { for i := int64(0); i < 40; i++ {
frame, err = prover.ProveMasterClockFrame(frame, i+1, 10) frame, err = prover.ProveMasterClockFrame(frame, i+1, 10)
assert.NoError(t, err) assert.NoError(t, err)
@ -160,12 +160,23 @@ func TestDataTimeReel(t *testing.T) {
wg.Wait() wg.Wait()
for i := 0; i < 100; i++ { for i := 0; i < 40; i++ {
assert.NotNil(t, frames[i]) assert.NotNil(t, frames[i])
assert.Equal(t, frames[i].FrameNumber, uint64(i+1)) assert.Equal(t, frames[i].FrameNumber, uint64(i+1))
} }
filterBytes, _ := hex.DecodeString(filter) filterBytes := []byte{
0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff,
}
// Ok, now onto the data time reel. We're going to test the following // Ok, now onto the data time reel. We're going to test the following
// scenarios: // scenarios:
@ -203,7 +214,7 @@ func TestDataTimeReel(t *testing.T) {
datawg.Add(1) datawg.Add(1)
dataFrameCh := d.NewFrameCh() dataFrameCh := d.NewFrameCh()
go func() { go func() {
for i := 0; i < 40; i++ { for i := 0; i < 41; i++ {
dataFrames = append(dataFrames, <-dataFrameCh) dataFrames = append(dataFrames, <-dataFrameCh)
} }
datawg.Done() datawg.Done()
@ -211,7 +222,7 @@ func TestDataTimeReel(t *testing.T) {
// 1. z-dist optimal proof submission is strictly master-frame evoked leader // 1. z-dist optimal proof submission is strictly master-frame evoked leader
for i := int64(0); i < 10; i++ { for i := int64(0); i < 10; i++ {
masterSelector, err := frames[i+1].GetSelector() masterSelector, err := frames[i].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelection := proverTrie.FindNearest( proverSelection := proverTrie.FindNearest(
@ -236,7 +247,7 @@ func TestDataTimeReel(t *testing.T) {
insertFrames := []*protobufs.ClockFrame{} insertFrames := []*protobufs.ClockFrame{}
for i := int64(10); i < 20; i++ { for i := int64(10); i < 20; i++ {
masterSelector, err := frames[i+1].GetSelector() masterSelector, err := frames[i].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelection := proverTrie.FindNearest( proverSelection := proverTrie.FindNearest(
@ -265,7 +276,7 @@ func TestDataTimeReel(t *testing.T) {
insertFrames = []*protobufs.ClockFrame{} insertFrames = []*protobufs.ClockFrame{}
for i := int64(20); i < 25; i++ { for i := int64(20); i < 25; i++ {
masterSelector, err := frames[i+1].GetSelector() masterSelector, err := frames[i].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelection := proverTrie.FindNearest( proverSelection := proverTrie.FindNearest(
@ -285,7 +296,7 @@ func TestDataTimeReel(t *testing.T) {
d.Insert(frame) d.Insert(frame)
} }
masterSelector, err := frames[26].GetSelector() masterSelector, err := frames[25].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelections := proverTrie.FindNearestAndApproximateNeighbors( proverSelections := proverTrie.FindNearestAndApproximateNeighbors(
@ -307,7 +318,7 @@ func TestDataTimeReel(t *testing.T) {
insertFrames = append(insertFrames, frame) insertFrames = append(insertFrames, frame)
for i := int64(26); i < 30; i++ { for i := int64(26); i < 30; i++ {
masterSelector, err := frames[i+1].GetSelector() masterSelector, err := frames[i].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelection := proverTrie.FindNearest( proverSelection := proverTrie.FindNearest(
@ -339,7 +350,7 @@ func TestDataTimeReel(t *testing.T) {
optimalKeySet := [][]byte{} optimalKeySet := [][]byte{}
suppressedFrame := frame suppressedFrame := frame
for i := int64(30); i < 40; i++ { for i := int64(30); i < 40; i++ {
masterSelector, err := frames[i+1].GetSelector() masterSelector, err := frames[i].GetSelector()
assert.NoError(t, err) assert.NoError(t, err)
proverSelections := proverTrie.FindNearestAndApproximateNeighbors( proverSelections := proverTrie.FindNearestAndApproximateNeighbors(
@ -357,7 +368,7 @@ func TestDataTimeReel(t *testing.T) {
// What we're trying to simulate: the majority is intentionally ignoring // What we're trying to simulate: the majority is intentionally ignoring
// the most optimal signer // the most optimal signer
suppressedFrame, err := prover.ProveDataClockFrame( suppressedFrame, err = prover.ProveDataClockFrame(
suppressedFrame, suppressedFrame,
[][]byte{}, [][]byte{},
[]*protobufs.InclusionAggregateProof{}, []*protobufs.InclusionAggregateProof{},
@ -379,26 +390,29 @@ func TestDataTimeReel(t *testing.T) {
for i := 9; i >= 0; i-- { for i := 9; i >= 0; i-- {
err := d.Insert(conflictFrames[i]) err := d.Insert(conflictFrames[i])
// force linear ordering
gotime.Sleep(1 * gotime.Second)
assert.NoError(t, err) assert.NoError(t, err)
} }
// Someone is honest, but clumsy: // Someone is honest, but running backwards:
for i := 9; i >= 0; i-- { for i := 9; i >= 0; i-- {
err := d.Insert(insertFrames[i]) err := d.Insert(insertFrames[i])
gotime.Sleep(1 * gotime.Second)
assert.NoError(t, err) assert.NoError(t, err)
} }
datawg.Wait() datawg.Wait()
for i := 0; i < 40; i++ { for i := 0; i < 41; i++ {
assert.NotNil(t, dataFrames[i]) assert.NotNil(t, dataFrames[i])
if i >= 40 { if i >= 40 {
assert.Equal(t, uint64(i), dataFrames[i].FrameNumber) assert.Equal(t, uint64(i), dataFrames[i].FrameNumber)
assert.Equal( assert.Equal(
t, t,
optimalKeySet[i-31],
dataFrames[i].GetPublicKeySignatureEd448().PublicKey.KeyValue, dataFrames[i].GetPublicKeySignatureEd448().PublicKey.KeyValue,
optimalKeySet[i-30],
) )
} else { } else {
assert.Equal(t, uint64(i+1), dataFrames[i].FrameNumber) assert.Equal(t, uint64(i+1), dataFrames[i].FrameNumber)

View File

@ -284,5 +284,5 @@ func printLogo() {
func printVersion() { func printVersion() {
fmt.Println(" ") fmt.Println(" ")
fmt.Println(" Quilibrium Node - v1.2.5 Dawn") fmt.Println(" Quilibrium Node - v1.2.6 Dawn")
} }

View File

@ -869,11 +869,18 @@ func (p *PebbleClockStore) GetParentDataClockFrame(
frameNumber uint64, frameNumber uint64,
parentSelector []byte, parentSelector []byte,
) (*protobufs.ClockFrame, error) { ) (*protobufs.ClockFrame, error) {
check := false
data, closer, err := p.db.Get( data, closer, err := p.db.Get(
clockDataParentIndexKey(filter, frameNumber, parentSelector), clockDataParentIndexKey(filter, frameNumber, parentSelector),
) )
if err != nil { if err != nil && !errors.Is(err, pebble.ErrNotFound) {
return nil, errors.Wrap(err, "get parent data clock frame") return nil, errors.Wrap(err, "get parent data clock frame")
} else if err != nil && errors.Is(err, pebble.ErrNotFound) {
data, closer, err = p.db.Get(clockDataFrameKey(filter, frameNumber))
if err != nil {
return nil, errors.Wrap(err, "get parent data clock frame")
}
check = true
} }
parent := &protobufs.ClockFrame{} parent := &protobufs.ClockFrame{}
@ -881,6 +888,16 @@ func (p *PebbleClockStore) GetParentDataClockFrame(
return nil, errors.Wrap(err, "get parent data clock frame") return nil, errors.Wrap(err, "get parent data clock frame")
} }
if check {
selector, err := parent.GetSelector()
if err != nil {
return nil, errors.Wrap(err, "get parent data clock frame")
}
if !bytes.Equal(selector.FillBytes(make([]byte, 32)), parentSelector) {
return nil, errors.Wrap(ErrNotFound, "get parent data clock frame")
}
}
if err := p.fillAggregateProofs(parent); err != nil { if err := p.fillAggregateProofs(parent); err != nil {
return nil, errors.Wrap( return nil, errors.Wrap(
errors.Wrap(err, ErrInvalidData.Error()), errors.Wrap(err, ErrInvalidData.Error()),
@ -1019,6 +1036,22 @@ func (p *PebbleClockStore) PutDataClockFrame(
return errors.Wrap(err, "put data clock frame") return errors.Wrap(err, "put data clock frame")
} }
selector, err := frame.GetSelector()
if err != nil {
return errors.Wrap(err, "put data clock frame")
}
if err = txn.Set(
clockDataParentIndexKey(
frame.Filter,
frame.FrameNumber,
selector.FillBytes(make([]byte, 32)),
),
data,
); err != nil {
return errors.Wrap(err, "put data clock frame")
}
proverData, err := proverTrie.Serialize() proverData, err := proverTrie.Serialize()
if err != nil { if err != nil {
return errors.Wrap(err, "put data clock frame") return errors.Wrap(err, "put data clock frame")

View File

@ -166,7 +166,6 @@ func (t *InMemKVDBTransaction) Set(key []byte, value []byte) error {
if !t.db.open { if !t.db.open {
return errors.New("inmem db closed") return errors.New("inmem db closed")
} }
t.changes = append(t.changes, InMemKVDBOperation{ t.changes = append(t.changes, InMemKVDBOperation{
op: SetOperation, op: SetOperation,
key: key, key: key,
@ -205,7 +204,6 @@ func (t *InMemKVDBTransaction) Delete(key []byte) error {
if !t.db.open { if !t.db.open {
return errors.New("inmem db closed") return errors.New("inmem db closed")
} }
t.changes = append(t.changes, InMemKVDBOperation{ t.changes = append(t.changes, InMemKVDBOperation{
op: DeleteOperation, op: DeleteOperation,
key: key, key: key,