From ff9810e1caf6a03cd4211ea9c87ee6deb4e4462c Mon Sep 17 00:00:00 2001 From: Peilun Li Date: Fri, 6 Sep 2024 17:40:29 +0800 Subject: [PATCH 1/2] Output debug log with serde_json and fix debug root db key. --- Cargo.lock | 6 ++-- node/storage/Cargo.toml | 1 + node/storage/src/log_store/flow_store.rs | 5 ++- .../src/log_store/load_chunk/bitmap.rs | 19 +++++++++++ .../src/log_store/load_chunk/chunk_data.rs | 9 ++--- node/storage/src/log_store/load_chunk/mod.rs | 34 +++++++++++++------ node/storage/src/log_store/load_chunk/seal.rs | 6 ++-- node/storage/src/log_store/log_manager.rs | 8 ++--- 8 files changed, 65 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 232476a..470227a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6892,11 +6892,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -7236,6 +7237,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", + "serde_json", "shared_types", "static_assertions", "tempdir", diff --git a/node/storage/Cargo.toml b/node/storage/Cargo.toml index 9eff9e0..8388510 100644 --- a/node/storage/Cargo.toml +++ b/node/storage/Cargo.toml @@ -28,6 +28,7 @@ tiny-keccak = "*" itertools = "0.13.0" serde = { version = "1.0.197", features = ["derive"] } parking_lot = "0.12.3" +serde_json = "1.0.127" [dev-dependencies] tempdir = "0.3.7" diff --git a/node/storage/src/log_store/flow_store.rs b/node/storage/src/log_store/flow_store.rs index 5ff4061..cdbc672 100644 --- a/node/storage/src/log_store/flow_store.rs +++ b/node/storage/src/log_store/flow_store.rs @@ -587,7 +587,10 @@ impl FlowDBStore { fn get_batch_root(&self, batch_index: u64) -> Result> { Ok(self .kvdb - .get(COL_ENTRY_BATCH_ROOT, &batch_index.to_be_bytes())? + .get( + COL_ENTRY_BATCH_ROOT, + &encode_batch_root_key(batch_index as usize, 1), + )? .map(|v| DataRoot::from_slice(&v))) } } diff --git a/node/storage/src/log_store/load_chunk/bitmap.rs b/node/storage/src/log_store/load_chunk/bitmap.rs index 0269428..6d883b8 100644 --- a/node/storage/src/log_store/load_chunk/bitmap.rs +++ b/node/storage/src/log_store/load_chunk/bitmap.rs @@ -3,6 +3,7 @@ use std::ops::{Deref, DerefMut}; use ssz::{Decode, DecodeError, Encode}; use bitmaps::{Bitmap, Bits, BitsImpl}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Default, Debug)] pub struct WrappedBitmap(pub Bitmap) @@ -72,6 +73,24 @@ where } } +impl<'a> Deserialize<'a> for WrappedBitmap<64> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'a>, + { + Ok(Self(Bitmap::from_value(u64::deserialize(deserializer)?))) + } +} + +impl Serialize for WrappedBitmap<64> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(self.0.into_value()) + } +} + pub trait TruncateBitmap { fn truncate(&mut self, index: u16); } diff --git a/node/storage/src/log_store/load_chunk/chunk_data.rs b/node/storage/src/log_store/load_chunk/chunk_data.rs index 75e92de..1a499bf 100644 --- a/node/storage/src/log_store/load_chunk/chunk_data.rs +++ b/node/storage/src/log_store/load_chunk/chunk_data.rs @@ -1,4 +1,5 @@ use anyhow::{bail, Result}; +use serde::{Deserialize, Serialize}; use shared_types::{bytes_to_chunks, DataRoot}; use ssz_derive::{Decode, Encode}; use std::fmt::{Debug, Formatter}; @@ -6,14 +7,14 @@ use std::mem; use tracing::error; use zgs_spec::{BYTES_PER_LOAD, BYTES_PER_SECTOR, SECTORS_PER_LOAD, SECTORS_PER_SEAL}; -#[derive(Debug)] +#[derive(Debug, Deserialize, Serialize)] pub enum EntryBatchData { Complete(Vec), /// All `PartialBatch`s are ordered based on `start_index`. Incomplete(IncompleteData), } -#[derive(Default, Debug, Encode, Decode)] +#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)] pub struct IncompleteData { pub subtrees: Vec, pub known_data: Vec, @@ -57,14 +58,14 @@ impl IncompleteData { } } -#[derive(Default, Debug, Encode, Decode)] +#[derive(Default, Debug, Encode, Decode, Deserialize, Serialize)] pub struct Subtree { pub start_sector: usize, pub subtree_height: usize, pub root: DataRoot, } -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Deserialize, Serialize)] pub struct PartialBatch { /// Offset in this batch. pub(super) start_sector: usize, diff --git a/node/storage/src/log_store/load_chunk/mod.rs b/node/storage/src/log_store/load_chunk/mod.rs index 15365f9..30af81a 100644 --- a/node/storage/src/log_store/load_chunk/mod.rs +++ b/node/storage/src/log_store/load_chunk/mod.rs @@ -3,15 +3,16 @@ mod chunk_data; mod seal; mod serde; +use ::serde::{Deserialize, Serialize}; use std::cmp::min; -use anyhow::Result; +use anyhow::{bail, Result}; use ethereum_types::H256; use ssz_derive::{Decode, Encode}; use crate::log_store::log_manager::data_to_merkle_leaves; use crate::try_option; -use append_merkle::{Algorithm, MerkleTreeRead, Sha3Algorithm}; +use append_merkle::MerkleTreeRead; use shared_types::{ChunkArray, DataRoot, Merkle}; use tracing::trace; use zgs_spec::{ @@ -23,7 +24,7 @@ use super::SealAnswer; use chunk_data::EntryBatchData; use seal::SealInfo; -#[derive(Debug, Encode, Decode)] +#[derive(Debug, Encode, Decode, Deserialize, Serialize)] pub struct EntryBatch { seal: SealInfo, // the inner data @@ -243,6 +244,15 @@ impl EntryBatch { for subtree in self.data.get_subtree_list() { trace!(?subtree, "get subtree, leaves={}", merkle.leaves()); if subtree.start_sector != merkle.leaves() { + println!( + "start_sector={} leaves={}", + subtree.start_sector, + merkle.leaves() + ); + let subtree_size = 1 << (subtree.subtree_height - 1); + if subtree.start_sector % subtree_size != 0 { + bail!("error"); + } let leaf_data = try_option!( self.get_unsealed_data(merkle.leaves(), subtree.start_sector - merkle.leaves()) ); @@ -257,13 +267,13 @@ impl EntryBatch { merkle.append_list(data_to_merkle_leaves(&leaf_data).expect("aligned")); } // TODO(zz): Optimize. - for index in 0..merkle.leaves() { - if merkle.leaf_at(index)?.is_none() { - if let Some(leaf_data) = self.get_unsealed_data(index, 1) { - merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data)); - } - } - } + // for index in 0..merkle.leaves() { + // if merkle.leaf_at(index)?.is_none() { + // if let Some(leaf_data) = self.get_unsealed_data(index, 1) { + // merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data)); + // } + // } + // } Ok(Some(merkle)) } } @@ -271,10 +281,14 @@ impl EntryBatch { #[cfg(test)] mod tests { use super::{EntryBatch, SealAnswer}; + use crate::log_store::load_chunk::chunk_data::EntryBatchData::Incomplete; + use crate::log_store::load_chunk::chunk_data::{IncompleteData, PartialBatch, Subtree}; + use append_merkle::MerkleTreeRead; use ethereum_types::H256; use zgs_spec::{ BYTES_PER_SEAL, BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL, }; + const LOAD_INDEX: u64 = 1; fn seal( batch: &mut EntryBatch, diff --git a/node/storage/src/log_store/load_chunk/seal.rs b/node/storage/src/log_store/load_chunk/seal.rs index 9c6f5ff..ed028f8 100644 --- a/node/storage/src/log_store/load_chunk/seal.rs +++ b/node/storage/src/log_store/load_chunk/seal.rs @@ -1,4 +1,5 @@ use ethereum_types::H256; +use serde::{Deserialize, Serialize}; use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode}; use static_assertions::const_assert; use tracing::info; @@ -8,7 +9,7 @@ use zgs_spec::{SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL}; use super::bitmap::WrappedBitmap; -#[derive(Debug, DeriveEncode, DeriveDecode)] +#[derive(Debug, DeriveEncode, DeriveDecode, Deserialize, Serialize)] pub struct SealContextInfo { /// The context digest for this seal group context_digest: H256, @@ -19,9 +20,10 @@ pub struct SealContextInfo { type ChunkSealBitmap = WrappedBitmap; const_assert!(SEALS_PER_LOAD <= u128::BITS as usize); -#[derive(Debug, Default, DeriveEncode, DeriveDecode)] +#[derive(Debug, Default, DeriveEncode, DeriveDecode, Deserialize, Serialize)] pub struct SealInfo { // a bitmap specify which sealing chunks have been sealed + #[serde(skip)] bitmap: ChunkSealBitmap, // the batch_offset (seal chunks) of the EntryBatch this seal info belongs to load_index: u64, diff --git a/node/storage/src/log_store/log_manager.rs b/node/storage/src/log_store/log_manager.rs index 52772cd..696f481 100644 --- a/node/storage/src/log_store/log_manager.rs +++ b/node/storage/src/log_store/log_manager.rs @@ -778,16 +778,16 @@ impl LogManager { }; let r = entry_proof(&top_proof, &sub_proof); if r.is_err() { - let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?; + let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?.unwrap(); let db_root = self.flow_store.get_batch_root(seg_index as u64)?; error!( ?r, - ?raw_batch, ?db_root, ?seg_index, - "gen proof error: top_leaves={}, last={}", + "gen proof error: top_leaves={}, last={}, raw_batch={}", merkle.pora_chunks_merkle.leaves(), - merkle.last_chunk_merkle.leaves() + merkle.last_chunk_merkle.leaves(), + serde_json::to_string(&raw_batch).unwrap(), ); } r From deaf224d0f5bd6023aed3423fc7c1bdb67b9a39d Mon Sep 17 00:00:00 2001 From: Peilun Li Date: Fri, 6 Sep 2024 17:53:21 +0800 Subject: [PATCH 2/2] nit. --- node/storage/src/log_store/load_chunk/mod.rs | 30 ++++++-------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/node/storage/src/log_store/load_chunk/mod.rs b/node/storage/src/log_store/load_chunk/mod.rs index 30af81a..f62fe7c 100644 --- a/node/storage/src/log_store/load_chunk/mod.rs +++ b/node/storage/src/log_store/load_chunk/mod.rs @@ -6,13 +6,13 @@ mod serde; use ::serde::{Deserialize, Serialize}; use std::cmp::min; -use anyhow::{bail, Result}; +use anyhow::Result; use ethereum_types::H256; use ssz_derive::{Decode, Encode}; use crate::log_store::log_manager::data_to_merkle_leaves; use crate::try_option; -use append_merkle::MerkleTreeRead; +use append_merkle::{Algorithm, MerkleTreeRead, Sha3Algorithm}; use shared_types::{ChunkArray, DataRoot, Merkle}; use tracing::trace; use zgs_spec::{ @@ -244,15 +244,6 @@ impl EntryBatch { for subtree in self.data.get_subtree_list() { trace!(?subtree, "get subtree, leaves={}", merkle.leaves()); if subtree.start_sector != merkle.leaves() { - println!( - "start_sector={} leaves={}", - subtree.start_sector, - merkle.leaves() - ); - let subtree_size = 1 << (subtree.subtree_height - 1); - if subtree.start_sector % subtree_size != 0 { - bail!("error"); - } let leaf_data = try_option!( self.get_unsealed_data(merkle.leaves(), subtree.start_sector - merkle.leaves()) ); @@ -267,13 +258,13 @@ impl EntryBatch { merkle.append_list(data_to_merkle_leaves(&leaf_data).expect("aligned")); } // TODO(zz): Optimize. - // for index in 0..merkle.leaves() { - // if merkle.leaf_at(index)?.is_none() { - // if let Some(leaf_data) = self.get_unsealed_data(index, 1) { - // merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data)); - // } - // } - // } + for index in 0..merkle.leaves() { + if merkle.leaf_at(index)?.is_none() { + if let Some(leaf_data) = self.get_unsealed_data(index, 1) { + merkle.fill_leaf(index, Sha3Algorithm::leaf(&leaf_data)); + } + } + } Ok(Some(merkle)) } } @@ -281,9 +272,6 @@ impl EntryBatch { #[cfg(test)] mod tests { use super::{EntryBatch, SealAnswer}; - use crate::log_store::load_chunk::chunk_data::EntryBatchData::Incomplete; - use crate::log_store::load_chunk::chunk_data::{IncompleteData, PartialBatch, Subtree}; - use append_merkle::MerkleTreeRead; use ethereum_types::H256; use zgs_spec::{ BYTES_PER_SEAL, BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL,