Output debug log with serde_json and fix debug root db key. (#185)
Some checks are pending
abi-consistent-check / build-and-compare (push) Waiting to run
code-coverage / unittest-cov (push) Waiting to run
rust / check (push) Waiting to run
rust / test (push) Waiting to run
rust / lints (push) Waiting to run
functional-test / test (push) Waiting to run

* Output debug log with serde_json and fix debug root db key.
This commit is contained in:
peilun-conflux 2024-09-08 20:21:34 +08:00 committed by GitHub
parent 052d2d781b
commit 678d233f69
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 44 additions and 14 deletions

6
Cargo.lock generated
View File

@ -6892,11 +6892,12 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.118"
version = "1.0.128"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
@ -7236,6 +7237,7 @@ dependencies = [
"rand 0.8.5",
"rayon",
"serde",
"serde_json",
"shared_types",
"static_assertions",
"tempdir",

View File

@ -28,6 +28,7 @@ tiny-keccak = "*"
itertools = "0.13.0"
serde = { version = "1.0.197", features = ["derive"] }
parking_lot = "0.12.3"
serde_json = "1.0.127"
[dev-dependencies]
tempdir = "0.3.7"

View File

@ -587,7 +587,10 @@ impl FlowDBStore {
fn get_batch_root(&self, batch_index: u64) -> Result<Option<DataRoot>> {
Ok(self
.kvdb
.get(COL_ENTRY_BATCH_ROOT, &batch_index.to_be_bytes())?
.get(
COL_ENTRY_BATCH_ROOT,
&encode_batch_root_key(batch_index as usize, 1),
)?
.map(|v| DataRoot::from_slice(&v)))
}
}

View File

@ -3,6 +3,7 @@ use std::ops::{Deref, DerefMut};
use ssz::{Decode, DecodeError, Encode};
use bitmaps::{Bitmap, Bits, BitsImpl};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Default, Debug)]
pub struct WrappedBitmap<const N: usize>(pub Bitmap<N>)
@ -72,6 +73,24 @@ where
}
}
impl<'a> Deserialize<'a> for WrappedBitmap<64> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'a>,
{
Ok(Self(Bitmap::from_value(u64::deserialize(deserializer)?)))
}
}
impl Serialize for WrappedBitmap<64> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u64(self.0.into_value())
}
}
pub trait TruncateBitmap {
fn truncate(&mut self, index: u16);
}

View File

@ -1,4 +1,5 @@
use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use shared_types::{bytes_to_chunks, DataRoot};
use ssz_derive::{Decode, Encode};
use std::fmt::{Debug, Formatter};
@ -6,14 +7,14 @@ use std::mem;
use tracing::error;
use zgs_spec::{BYTES_PER_LOAD, BYTES_PER_SECTOR, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
#[derive(Debug)]
#[derive(Debug, Deserialize, Serialize)]
pub enum EntryBatchData {
Complete(Vec<u8>),
/// All `PartialBatch`s are ordered based on `start_index`.
Incomplete(IncompleteData),
}
#[derive(Default, Debug, Encode, Decode)]
#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)]
pub struct IncompleteData {
pub subtrees: Vec<Subtree>,
pub known_data: Vec<PartialBatch>,
@ -57,14 +58,14 @@ impl IncompleteData {
}
}
#[derive(Default, Debug, Encode, Decode)]
#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)]
pub struct Subtree {
pub start_sector: usize,
pub subtree_height: usize,
pub root: DataRoot,
}
#[derive(PartialEq, Eq)]
#[derive(PartialEq, Eq, Deserialize, Serialize)]
pub struct PartialBatch {
/// Offset in this batch.
pub(super) start_sector: usize,

View File

@ -3,6 +3,7 @@ mod chunk_data;
mod seal;
mod serde;
use ::serde::{Deserialize, Serialize};
use std::cmp::min;
use anyhow::Result;
@ -23,7 +24,7 @@ use super::SealAnswer;
use chunk_data::EntryBatchData;
use seal::SealInfo;
#[derive(Debug, Encode, Decode)]
#[derive(Debug, Encode, Decode, Deserialize, Serialize)]
pub struct EntryBatch {
seal: SealInfo,
// the inner data
@ -275,6 +276,7 @@ mod tests {
use zgs_spec::{
BYTES_PER_SEAL, BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL,
};
const LOAD_INDEX: u64 = 1;
fn seal(
batch: &mut EntryBatch,

View File

@ -1,4 +1,5 @@
use ethereum_types::H256;
use serde::{Deserialize, Serialize};
use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
use static_assertions::const_assert;
use tracing::info;
@ -8,7 +9,7 @@ use zgs_spec::{SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
use super::bitmap::WrappedBitmap;
#[derive(Debug, DeriveEncode, DeriveDecode)]
#[derive(Debug, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct SealContextInfo {
/// The context digest for this seal group
context_digest: H256,
@ -19,9 +20,10 @@ pub struct SealContextInfo {
type ChunkSealBitmap = WrappedBitmap<SEALS_PER_LOAD>;
const_assert!(SEALS_PER_LOAD <= u128::BITS as usize);
#[derive(Debug, Default, DeriveEncode, DeriveDecode)]
#[derive(Debug, Default, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct SealInfo {
// a bitmap specify which sealing chunks have been sealed
#[serde(skip)]
bitmap: ChunkSealBitmap,
// the batch_offset (seal chunks) of the EntryBatch this seal info belongs to
load_index: u64,

View File

@ -778,16 +778,16 @@ impl LogManager {
};
let r = entry_proof(&top_proof, &sub_proof);
if r.is_err() {
let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?;
let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?.unwrap();
let db_root = self.flow_store.get_batch_root(seg_index as u64)?;
error!(
?r,
?raw_batch,
?db_root,
?seg_index,
"gen proof error: top_leaves={}, last={}",
"gen proof error: top_leaves={}, last={}, raw_batch={}",
merkle.pora_chunks_merkle.leaves(),
merkle.last_chunk_merkle.leaves()
merkle.last_chunk_merkle.leaves(),
serde_json::to_string(&raw_batch).unwrap(),
);
}
r