fix: upgrade pebble to stop OOM killer being out of control on some platforms

node/go.mod

@@ -13,7 +13,7 @@ replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go-
 
 require (
 	filippo.io/edwards25519 v1.0.0-rc.1
-	github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff
+	github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055
 	github.com/libp2p/go-libp2p v0.31.0
 	github.com/libp2p/go-libp2p-gostream v0.6.0
 	github.com/libp2p/go-libp2p-kad-dht v0.23.0
@@ -28,19 +28,19 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401 // indirect
 	github.com/bwesterb/go-ristretto v1.2.3 // indirect
-	github.com/charmbracelet/bubbletea v0.24.2 // indirect
+	github.com/charmbracelet/bubbletea v0.24.2
-	github.com/charmbracelet/lipgloss v0.9.1 // indirect
+	github.com/charmbracelet/lipgloss v0.9.1
-	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/errors v1.11.1 // indirect
-	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/consensys/gnark-crypto v0.5.3 // indirect
 	github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/glog v1.1.0 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
 	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -52,7 +52,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
-	github.com/multiformats/go-multiaddr v0.11.0 // indirect
+	github.com/multiformats/go-multiaddr v0.11.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/quic-go/qtls-go1-19 v0.3.3 // indirect
 	github.com/quic-go/qtls-go1-20 v0.2.3 // indirect

node/go.sum

@@ -66,14 +66,24 @@ github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaY
 github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
 github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
 github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
+github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
+github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff h1:/F1VgP7wxZCRj8PzresPo2NbAdgPwmU7pi+CgZ8sBZw=
 github.com/cockroachdb/pebble v0.0.0-20230527012508-ac69476c46ff/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw=
+github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055 h1:EigfnVX/iY/WTi3F+f4ezhAxJO+BePglQkEAKycNhqo=
+github.com/cockroachdb/pebble v0.0.0-20231025190044-422dce910055/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
 github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
+github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.5.3 h1:4xLFGZR3NWEH2zy+YzvzHicpToQR8FXFbfLNvpGB+rE=
@@ -132,6 +142,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
+github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=

node/store/clock.go

@@ -547,10 +547,13 @@ func (p *PebbleClockStore) RangeMasterClockFrames(
 		startFrameNumber = temp
 	}
 
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: clockMasterFrameKey(filter, startFrameNumber),
 		UpperBound: clockMasterFrameKey(filter, endFrameNumber),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "range master clock frames")
+	}
 
 	return &PebbleMasterClockIterator{i: iter}, nil
 }
@@ -1001,7 +1004,7 @@ func (p *PebbleClockStore) GetCandidateDataClockFrames(
 	filter []byte,
 	frameNumber uint64,
 ) ([]*protobufs.ClockFrame, error) {
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: clockDataCandidateFrameKey(
 			filter,
 			frameNumber,
@@ -1035,6 +1038,9 @@ func (p *PebbleClockStore) GetCandidateDataClockFrames(
 			},
 		),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "get candidate data clock frames")
+	}
 
 	frames := []*protobufs.ClockFrame{}
 	i := &PebbleCandidateClockIterator{i: iter, db: p}
@@ -1078,7 +1084,7 @@ func (p *PebbleClockStore) RangeCandidateDataClockFrames(
 			0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 		}
 	}
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: clockDataCandidateFrameKey(
 			filter,
 			frameNumber,
@@ -1102,6 +1108,9 @@ func (p *PebbleClockStore) RangeCandidateDataClockFrames(
 			},
 		),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "range candidate data clock frames")
+	}
 
 	return &PebbleCandidateClockIterator{i: iter, db: p}, nil
 }
@@ -1118,10 +1127,13 @@ func (p *PebbleClockStore) RangeDataClockFrames(
 		startFrameNumber = temp
 	}
 
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: clockDataFrameKey(filter, startFrameNumber),
 		UpperBound: clockDataFrameKey(filter, endFrameNumber),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "get data clock frames")
+	}
 
 	return &PebbleClockIterator{i: iter, db: p}, nil
 }
@@ -1149,10 +1161,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
 		},
 	)
 
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: from,
 		UpperBound: to,
 	})
+	if err != nil {
+		return errors.Wrap(err, "deduplicate")
+	}
 
 	i := 0
 	for iter.First(); iter.Valid(); iter.Next() {
@@ -1190,10 +1205,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
 	from = clockDataFrameKey(filter, 1)
 	to = clockDataFrameKey(filter, 20000)
 
-	iter = p.db.NewIter(&pebble.IterOptions{
+	iter, err = p.db.NewIter(&pebble.IterOptions{
 		LowerBound: from,
 		UpperBound: to,
 	})
+	if err != nil {
+		return errors.Wrap(err, "deduplicate")
+	}
 
 	i = 0
 	for iter.First(); iter.Valid(); iter.Next() {
@@ -1261,10 +1279,13 @@ func (p *PebbleClockStore) Deduplicate(filter []byte) error {
 		},
 	)
 
-	iter = p.db.NewIter(&pebble.IterOptions{
+	iter, err = p.db.NewIter(&pebble.IterOptions{
 		LowerBound: from,
 		UpperBound: to,
 	})
+	if err != nil {
+		return errors.Wrap(err, "deduplicate")
+	}
 
 	i = 0
 	for iter.First(); iter.Valid(); iter.Next() {
@@ -1313,10 +1334,13 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
 	from := clockDataFrameKey(filter, fromFrameNumber)
 	to := clockDataFrameKey(filter, toFrameNumber+1)
 
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: from,
 		UpperBound: to,
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "get compressed data clock frames")
+	}
 
 	syncMessage := &protobufs.CeremonyCompressedSync{}
 	proofs := map[string]*protobufs.InclusionProofsMap{}
@@ -1394,10 +1418,13 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
 			},
 		)
 
-		iter := p.db.NewIter(&pebble.IterOptions{
+		iter, err := p.db.NewIter(&pebble.IterOptions{
 			LowerBound: from,
 			UpperBound: to,
 		})
+		if err != nil {
+			return nil, errors.Wrap(err, "get compressed data clock frames")
+		}
 
 		candidates := []*protobufs.ClockFrame{}
 		for iter.First(); iter.Valid(); iter.Next() {
@@ -1508,7 +1535,7 @@ func (p *PebbleClockStore) GetCompressedDataClockFrames(
 			return nil, errors.Wrap(err, "get compressed data clock frames")
 		}
 
-		iter := p.db.NewIter(&pebble.IterOptions{
+		iter, err := p.db.NewIter(&pebble.IterOptions{
 			LowerBound: dataProofInclusionKey(filter, []byte(k), 0),
 			UpperBound: dataProofInclusionKey(filter, []byte(k), limit+1),
 		})
@@ -1700,7 +1727,7 @@ func (p *PebbleClockStore) GetHighestCandidateDataClockFrame(
 		},
 	)
 
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: from,
 		UpperBound: to,
 	})

node/store/data_proof.go

@@ -114,10 +114,13 @@ func internalGetAggregateProof(
 		Proof:                copied,
 	}
 
-	iter := db.NewIter(&pebble.IterOptions{
+	iter, err := db.NewIter(&pebble.IterOptions{
 		LowerBound: dataProofInclusionKey(filter, commitment, 0),
 		UpperBound: dataProofInclusionKey(filter, commitment, limit+1),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "get aggregate proof")
+	}
 
 	i := uint32(0)
 

node/store/key.go

@@ -462,7 +462,7 @@ func (p *PebbleKeyStore) PutKeyBundle(
 }
 
 func (p *PebbleKeyStore) RangeProvingKeys() (*PebbleProvingKeyIterator, error) {
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: provingKeyKey([]byte{
 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -484,6 +484,9 @@ func (p *PebbleKeyStore) RangeProvingKeys() (*PebbleProvingKeyIterator, error) {
 			0xff,
 		}),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "range proving keys")
+	}
 
 	return &PebbleProvingKeyIterator{i: iter}, nil
 }
@@ -492,7 +495,7 @@ func (p *PebbleKeyStore) RangeStagedProvingKeys() (
 	*PebbleStagedProvingKeyIterator,
 	error,
 ) {
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: stagedProvingKeyKey([]byte{
 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -514,6 +517,9 @@ func (p *PebbleKeyStore) RangeStagedProvingKeys() (
 			0xff,
 		}),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "range staged proving keys")
+	}
 
 	return &PebbleStagedProvingKeyIterator{i: iter}, nil
 }
@@ -522,10 +528,13 @@ func (p *PebbleKeyStore) RangeKeyBundleKeys(provingKey []byte) (
 	*PebbleKeyBundleIterator,
 	error,
 ) {
-	iter := p.db.NewIter(&pebble.IterOptions{
+	iter, err := p.db.NewIter(&pebble.IterOptions{
 		LowerBound: keyBundleKey(provingKey, 0),
 		UpperBound: keyBundleKey(provingKey, 0xffffffffffffffff),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "range key bundle keys")
+	}
 
 	return &PebbleKeyBundleIterator{i: iter}, nil
 }