Output debug log with serde_json and fix debug root db key.

This commit is contained in:
Peilun Li 2024-09-06 17:40:29 +08:00
parent 29fcc415a6
commit ff9810e1ca
8 changed files with 65 additions and 23 deletions

6
Cargo.lock generated
View File

@ -6892,11 +6892,12 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.118"
version = "1.0.128"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
@ -7236,6 +7237,7 @@ dependencies = [
"rand 0.8.5",
"rayon",
"serde",
"serde_json",
"shared_types",
"static_assertions",
"tempdir",

View File

@ -28,6 +28,7 @@ tiny-keccak = "*"
itertools = "0.13.0"
serde = { version = "1.0.197", features = ["derive"] }
parking_lot = "0.12.3"
serde_json = "1.0.127"
[dev-dependencies]
tempdir = "0.3.7"

View File

@ -587,7 +587,10 @@ impl FlowDBStore {
fn get_batch_root(&self, batch_index: u64) -> Result<Option<DataRoot>> {
Ok(self
.kvdb
.get(COL_ENTRY_BATCH_ROOT, &batch_index.to_be_bytes())?
.get(
COL_ENTRY_BATCH_ROOT,
&encode_batch_root_key(batch_index as usize, 1),
)?
.map(|v| DataRoot::from_slice(&v)))
}
}

View File

@ -3,6 +3,7 @@ use std::ops::{Deref, DerefMut};
use ssz::{Decode, DecodeError, Encode};
use bitmaps::{Bitmap, Bits, BitsImpl};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Default, Debug)]
pub struct WrappedBitmap<const N: usize>(pub Bitmap<N>)
@ -72,6 +73,24 @@ where
}
}
impl<'a> Deserialize<'a> for WrappedBitmap<64> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'a>,
{
Ok(Self(Bitmap::from_value(u64::deserialize(deserializer)?)))
}
}
impl Serialize for WrappedBitmap<64> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u64(self.0.into_value())
}
}
pub trait TruncateBitmap {
fn truncate(&mut self, index: u16);
}

View File

@ -1,4 +1,5 @@
use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use shared_types::{bytes_to_chunks, DataRoot};
use ssz_derive::{Decode, Encode};
use std::fmt::{Debug, Formatter};
@ -6,14 +7,14 @@ use std::mem;
use tracing::error;
use zgs_spec::{BYTES_PER_LOAD, BYTES_PER_SECTOR, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
#[derive(Debug)]
#[derive(Debug, Deserialize, Serialize)]
pub enum EntryBatchData {
Complete(Vec<u8>),
/// All `PartialBatch`s are ordered based on `start_index`.
Incomplete(IncompleteData),
}
#[derive(Default, Debug, Encode, Decode)]
#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)]
pub struct IncompleteData {
pub subtrees: Vec<Subtree>,
pub known_data: Vec<PartialBatch>,
@ -57,14 +58,14 @@ impl IncompleteData {
}
}
#[derive(Default, Debug, Encode, Decode)]
#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)]
pub struct Subtree {
pub start_sector: usize,
pub subtree_height: usize,
pub root: DataRoot,
}
#[derive(PartialEq, Eq)]
#[derive(PartialEq, Eq, Deserialize, Serialize)]
pub struct PartialBatch {
/// Offset in this batch.
pub(super) start_sector: usize,

View File

@ -3,15 +3,16 @@ mod chunk_data;
mod seal;
mod serde;
use ::serde::{Deserialize, Serialize};
use std::cmp::min;
use anyhow::Result;
use anyhow::{bail, Result};
use ethereum_types::H256;
use ssz_derive::{Decode, Encode};
use crate::log_store::log_manager::data_to_merkle_leaves;
use crate::try_option;
use append_merkle::{Algorithm, MerkleTreeRead, Sha3Algorithm};
use append_merkle::MerkleTreeRead;
use shared_types::{ChunkArray, DataRoot, Merkle};
use tracing::trace;
use zgs_spec::{
@ -23,7 +24,7 @@ use super::SealAnswer;
use chunk_data::EntryBatchData;
use seal::SealInfo;
#[derive(Debug, Encode, Decode)]
#[derive(Debug, Encode, Decode, Deserialize, Serialize)]
pub struct EntryBatch {
seal: SealInfo,
// the inner data
@ -243,6 +244,15 @@ impl EntryBatch {
for subtree in self.data.get_subtree_list() {
trace!(?subtree, "get subtree, leaves={}", merkle.leaves());
if subtree.start_sector != merkle.leaves() {
println!(
"start_sector={} leaves={}",
subtree.start_sector,
merkle.leaves()
);
let subtree_size = 1 << (subtree.subtree_height - 1);
if subtree.start_sector % subtree_size != 0 {
bail!("error");
}
let leaf_data = try_option!(
self.get_unsealed_data(merkle.leaves(), subtree.start_sector - merkle.leaves())
);
@ -257,13 +267,13 @@ impl EntryBatch {
merkle.append_list(data_to_merkle_leaves(&leaf_data).expect("aligned"));
}
// TODO(zz): Optimize.
for index in 0..merkle.leaves() {
if merkle.leaf_at(index)?.is_none() {
if let Some(leaf_data) = self.get_unsealed_data(index, 1) {
merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data));
}
}
}
// for index in 0..merkle.leaves() {
// if merkle.leaf_at(index)?.is_none() {
// if let Some(leaf_data) = self.get_unsealed_data(index, 1) {
// merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data));
// }
// }
// }
Ok(Some(merkle))
}
}
@ -271,10 +281,14 @@ impl EntryBatch {
#[cfg(test)]
mod tests {
use super::{EntryBatch, SealAnswer};
use crate::log_store::load_chunk::chunk_data::EntryBatchData::Incomplete;
use crate::log_store::load_chunk::chunk_data::{IncompleteData, PartialBatch, Subtree};
use append_merkle::MerkleTreeRead;
use ethereum_types::H256;
use zgs_spec::{
BYTES_PER_SEAL, BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL,
};
const LOAD_INDEX: u64 = 1;
fn seal(
batch: &mut EntryBatch,

View File

@ -1,4 +1,5 @@
use ethereum_types::H256;
use serde::{Deserialize, Serialize};
use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
use static_assertions::const_assert;
use tracing::info;
@ -8,7 +9,7 @@ use zgs_spec::{SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
use super::bitmap::WrappedBitmap;
#[derive(Debug, DeriveEncode, DeriveDecode)]
#[derive(Debug, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct SealContextInfo {
/// The context digest for this seal group
context_digest: H256,
@ -19,9 +20,10 @@ pub struct SealContextInfo {
type ChunkSealBitmap = WrappedBitmap<SEALS_PER_LOAD>;
const_assert!(SEALS_PER_LOAD <= u128::BITS as usize);
#[derive(Debug, Default, DeriveEncode, DeriveDecode)]
#[derive(Debug, Default, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct SealInfo {
// a bitmap specify which sealing chunks have been sealed
#[serde(skip)]
bitmap: ChunkSealBitmap,
// the batch_offset (seal chunks) of the EntryBatch this seal info belongs to
load_index: u64,

View File

@ -778,16 +778,16 @@ impl LogManager {
};
let r = entry_proof(&top_proof, &sub_proof);
if r.is_err() {
let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?;
let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?.unwrap();
let db_root = self.flow_store.get_batch_root(seg_index as u64)?;
error!(
?r,
?raw_batch,
?db_root,
?seg_index,
"gen proof error: top_leaves={}, last={}",
"gen proof error: top_leaves={}, last={}, raw_batch={}",
merkle.pora_chunks_merkle.leaves(),
merkle.last_chunk_merkle.leaves()
merkle.last_chunk_merkle.leaves(),
serde_json::to_string(&raw_batch).unwrap(),
);
}
r