fix: upgrade pebble to stop OOM killer being out of control on some platforms

This commit is contained in:
Cassandra Heart 2023-12-02 19:25:01 -06:00
parent 645afad9c6
commit a0c1feb0bc
No known key found for this signature in database
GPG Key ID: 6352152859385958
5 changed files with 76 additions and 25 deletions

View File

@ -13,7 +13,7 @@ replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go-
require (
filippo.io/edwards25519 v1.0.0-rc.1
github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff
github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055
github.com/libp2p/go-libp2p v0.31.0
github.com/libp2p/go-libp2p-gostream v0.6.0
github.com/libp2p/go-libp2p-kad-dht v0.23.0
@ -28,19 +28,19 @@ require (
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401 // indirect
github.com/bwesterb/go-ristretto v1.2.3 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.9.1 // indirect
github.com/cockroachdb/errors v1.8.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/charmbracelet/bubbletea v0.24.2
github.com/charmbracelet/lipgloss v0.9.1
github.com/cockroachdb/errors v1.11.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/gnark-crypto v0.5.3 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/getsentry/sentry-go v0.18.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
github.com/gtank/merlin v0.1.1 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
@ -52,7 +52,7 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/multiformats/go-multiaddr v0.11.0 // indirect
github.com/multiformats/go-multiaddr v0.11.0
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quic-go/qtls-go1-19 v0.3.3 // indirect
github.com/quic-go/qtls-go1-20 v0.2.3 // indirect

View File

@ -66,14 +66,24 @@ github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaY
github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff h1:/F1VgP7wxZCRj8PzresPo2NbAdgPwmU7pi+CgZ8sBZw=
github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw=
github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055 h1:EigfnVX/iY/WTi3F+f4ezhAxJO+BePglQkEAKycNhqo=
github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
github.com/consensys/gnark-crypto v0.5.3 h1:4xLFGZR3NWEH2zy+YzvzHicpToQR8FXFbfLNvpGB+rE=
@ -132,6 +142,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=

View File

@ -547,10 +547,13 @@ func (p *PebbleClockStore) RangeMasterClockFrames(
startFrameNumber = temp
}
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: clockMasterFrameKey(filter, startFrameNumber),
UpperBound: clockMasterFrameKey(filter, endFrameNumber),
})
if err != nil {
return nil, errors.Wrap(err, "range master clock frames")
}
return &PebbleMasterClockIterator{i: iter}, nil
}
@ -1001,7 +1004,7 @@ func (p *PebbleClockStore) GetCandidateDataClockFrames(
filter []byte,
frameNumber uint64,
) ([]*protobufs.ClockFrame, error) {
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: clockDataCandidateFrameKey(
filter,
frameNumber,
@ -1035,6 +1038,9 @@ func (p *PebbleClockStore) GetCandidateDataClockFrames(
},
),
})
if err != nil {
return nil, errors.Wrap(err, "get candidate data clock frames")
}
frames := []*protobufs.ClockFrame{}
i := &PebbleCandidateClockIterator{i: iter, db: p}
@ -1078,7 +1084,7 @@ func (p *PebbleClockStore) RangeCandidateDataClockFrames(
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
}
}
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: clockDataCandidateFrameKey(
filter,
frameNumber,
@ -1102,6 +1108,9 @@ func (p *PebbleClockStore) RangeCandidateDataClockFrames(
},
),
})
if err != nil {
return nil, errors.Wrap(err, "range candidate data clock frames")
}
return &PebbleCandidateClockIterator{i: iter, db: p}, nil
}
@ -1118,10 +1127,13 @@ func (p *PebbleClockStore) RangeDataClockFrames(
startFrameNumber = temp
}
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: clockDataFrameKey(filter, startFrameNumber),
UpperBound: clockDataFrameKey(filter, endFrameNumber),
})
if err != nil {
return nil, errors.Wrap(err, "get data clock frames")
}
return &PebbleClockIterator{i: iter, db: p}, nil
}
@ -1149,10 +1161,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
},
)
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})
if err != nil {
return errors.Wrap(err, "deduplicate")
}
i := 0
for iter.First(); iter.Valid(); iter.Next() {
@ -1190,10 +1205,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
from = clockDataFrameKey(filter, 1)
to = clockDataFrameKey(filter, 20000)
iter = p.db.NewIter(&pebble.IterOptions{
iter, err = p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})
if err != nil {
return errors.Wrap(err, "deduplicate")
}
i = 0
for iter.First(); iter.Valid(); iter.Next() {
@ -1261,10 +1279,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
},
)
iter = p.db.NewIter(&pebble.IterOptions{
iter, err = p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})
if err != nil {
return errors.Wrap(err, "deduplicate")
}
i = 0
for iter.First(); iter.Valid(); iter.Next() {
@ -1313,10 +1334,13 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
from := clockDataFrameKey(filter, fromFrameNumber)
to := clockDataFrameKey(filter, toFrameNumber+1)
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})
if err != nil {
return nil, errors.Wrap(err, "get compressed data clock frames")
}
syncMessage := &protobufs.CeremonyCompressedSync{}
proofs := map[string]*protobufs.InclusionProofsMap{}
@ -1394,10 +1418,13 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
},
)
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})
if err != nil {
return nil, errors.Wrap(err, "get compressed data clock frames")
}
candidates := []*protobufs.ClockFrame{}
for iter.First(); iter.Valid(); iter.Next() {
@ -1508,7 +1535,7 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
return nil, errors.Wrap(err, "get compressed data clock frames")
}
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: dataProofInclusionKey(filter, []byte(k), 0),
UpperBound: dataProofInclusionKey(filter, []byte(k), limit+1),
})
@ -1700,7 +1727,7 @@ func (p *PebbleClockStore) GetHighestCandidateDataClockFrame(
},
)
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: from,
UpperBound: to,
})

View File

@ -114,10 +114,13 @@ func internalGetAggregateProof(
Proof: copied,
}
iter := db.NewIter(&pebble.IterOptions{
iter, err := db.NewIter(&pebble.IterOptions{
LowerBound: dataProofInclusionKey(filter, commitment, 0),
UpperBound: dataProofInclusionKey(filter, commitment, limit+1),
})
if err != nil {
return nil, errors.Wrap(err, "get aggregate proof")
}
i := uint32(0)

View File

@ -462,7 +462,7 @@ func (p *PebbleKeyStore) PutKeyBundle(
}
func (p *PebbleKeyStore) RangeProvingKeys() (*PebbleProvingKeyIterator, error) {
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: provingKeyKey([]byte{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@ -484,6 +484,9 @@ func (p *PebbleKeyStore) RangeProvingKeys() (*PebbleProvingKeyIterator, error) {
0xff,
}),
})
if err != nil {
return nil, errors.Wrap(err, "range proving keys")
}
return &PebbleProvingKeyIterator{i: iter}, nil
}
@ -492,7 +495,7 @@ func (p *PebbleKeyStore) RangeStagedProvingKeys() (
*PebbleStagedProvingKeyIterator,
error,
) {
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: stagedProvingKeyKey([]byte{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@ -514,6 +517,9 @@ func (p *PebbleKeyStore) RangeStagedProvingKeys() (
0xff,
}),
})
if err != nil {
return nil, errors.Wrap(err, "range staged proving keys")
}
return &PebbleStagedProvingKeyIterator{i: iter}, nil
}
@ -522,10 +528,13 @@ func (p *PebbleKeyStore) RangeKeyBundleKeys(provingKey []byte) (
*PebbleKeyBundleIterator,
error,
) {
iter := p.db.NewIter(&pebble.IterOptions{
iter, err := p.db.NewIter(&pebble.IterOptions{
LowerBound: keyBundleKey(provingKey, 0),
UpperBound: keyBundleKey(provingKey, 0xffffffffffffffff),
})
if err != nil {
return nil, errors.Wrap(err, "range key bundle keys")
}
return &PebbleKeyBundleIterator{i: iter}, nil
}