Initial commit

This commit is contained in:
Peilun Li 2024-01-03 18:24:52 +08:00
commit 9058467be1
319 changed files with 71712 additions and 0 deletions

3
.github/actions-rs/grcov.yml vendored Normal file
View File

@ -0,0 +1,3 @@
branch: false
ignore-not-existing: true
llvm: true

53
.github/workflows/cc.yml vendored Normal file
View File

@ -0,0 +1,53 @@
name: code-coverage
on:
push:
branches: [ "main"]
pull_request:
branches: [ "main" ]
env:
CARGO_TERM_COLOR: always
jobs:
unittest-cov:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Cargo cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
- name: Install 1.71.0 toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: 1.71.0
override: true
- name: Run unittest
run: cargo test --all-features --no-fail-fast
env:
CARGO_INCREMENTAL: '0'
RUSTC_BOOTSTRAP: '1'
RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests'
RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests'
- id: coverage
uses: actions-rs/grcov@v0.1
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
file: ${{ steps.coverage.outputs.report }}
fail_ci_if_error: true

100
.github/workflows/rust.yml vendored Normal file
View File

@ -0,0 +1,100 @@
# Based on https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md
#
# While our "example" application has the platform-specific code,
# for simplicity we are compiling and testing everything on the Ubuntu environment only.
# For multi-OS testing see the `cross.yml` workflow.
name: rust
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: check
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v3
with:
submodules: recursive
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Run cargo check
uses: actions-rs/cargo@v1
with:
command: check
test:
name: test
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v3
with:
submodules: recursive
- name: Cargo cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Run cargo test
uses: actions-rs/cargo@v1
with:
command: test
args: --release
lints:
name: lints
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v3
with:
submodules: recursive
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
components: rustfmt, clippy
- name: Run cargo fmt
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
- name: Run cargo clippy
uses: actions-rs/cargo@v1
with:
command: clippy
args: -- -D warnings

50
.github/workflows/tests.yml vendored Normal file
View File

@ -0,0 +1,50 @@
name: functional-test
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
env:
CARGO_TERM_COLOR: always
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Cargo cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Build
run: cargo build --release --verbose
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: '3.9'
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Run tests
run: |
cd tests
python test_all.py

7
.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
.DS_Store
/target
# src/config.rs
/.idea
tests/**/__pycache__
tests/tmp/**
.vscode/*.json

6
.gitmodules vendored Normal file
View File

@ -0,0 +1,6 @@
[submodule "zerog-storage-contracts"]
path = zerog-storage-contracts
url = git@github.com:zero-gravity-labs/zerog-storage-contracts.git
[submodule "zerog-storage-client"]
path = zerog-storage-client
url = git@github.com:zero-gravity-labs/zerog-storage-client.git

8082
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

32
Cargo.toml Normal file
View File

@ -0,0 +1,32 @@
[workspace]
members = [
"common/channel",
"common/directory",
"common/hashset_delay",
"common/lighthouse_metrics",
"common/merkle_tree",
"common/task_executor",
"common/zgs_version",
"common/unused_port",
"common/append_merkle",
"node",
"node/chunk_pool",
"node/file_location_cache",
"node/log_entry_sync",
"node/miner",
"node/network",
"node/router",
"node/rpc",
"node/shared_types",
"node/storage",
"node/storage-async",
"node/sync",
]
resolver = "2"
[patch.crates-io]
discv5 = { path = "version-meld/discv5" }
eth2_ssz = { path = "version-meld/eth2_ssz" }
enr = { path = "version-meld/enr" }

201
LICENSE.lighthouse Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 Sigma Prime Pty Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

6
NOTICE Normal file
View File

@ -0,0 +1,6 @@
ZeroGStorage
Copyright 2023 ZeroGStorage
The Initial Developer of some parts of the framework, which are copied from, derived from, or
inspired by Lighthouse, is Sigma Prime Pty Ltd (https://sigmaprime.io).
Copyright 2018 - 2022 Sigma Prime Pty Ltd.

8
codecov.yml Normal file
View File

@ -0,0 +1,8 @@
coverage:
status:
project:
default:
informational: true
patch:
default:
informational: true

View File

@ -0,0 +1,14 @@
[package]
name = "append_merkle"
version = "0.1.0"
edition = "2021"
[dependencies]
anyhow = { version = "=1.0.58", features = ["backtrace"] }
ethereum-types = "0.14"
tiny-keccak = { version = "2.0.2", features = ["keccak"] }
eth2_ssz = "0.4.0"
eth2_ssz_derive = "0.3.0"
serde = { version = "1.0.137", features = ["derive"] }
lazy_static = "1.4.0"
tracing = "0.1.36"

View File

@ -0,0 +1,641 @@
mod merkle_tree;
mod proof;
mod sha3;
use anyhow::{anyhow, bail, Result};
use std::cmp::Ordering;
use std::collections::HashMap;
use std::fmt::Debug;
use std::marker::PhantomData;
use tracing::warn;
pub use crate::merkle_tree::{Algorithm, HashElement, MerkleTreeInitialData, MerkleTreeRead};
pub use proof::{Proof, RangeProof};
pub use sha3::Sha3Algorithm;
pub struct AppendMerkleTree<E: HashElement, A: Algorithm<E>> {
/// Keep all the nodes in the latest version. `layers[0]` is the layer of leaves.
layers: Vec<Vec<E>>,
/// Keep the delta nodes that can be used to construct a history tree.
/// The key is the root node of that version.
delta_nodes_map: HashMap<u64, DeltaNodes<E>>,
root_to_tx_seq_map: HashMap<E, u64>,
/// For `last_chunk_merkle` after the first chunk, this is set to `Some(10)` so that
/// `revert_to` can reset the state correctly when needed.
min_depth: Option<usize>,
/// Used to compute the correct padding hash.
/// 0 for `pora_chunk_merkle` and 10 for not-first `last_chunk_merkle`.
leaf_height: usize,
_a: PhantomData<A>,
}
impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
pub fn new(leaves: Vec<E>, leaf_height: usize, start_tx_seq: Option<u64>) -> Self {
let mut merkle = Self {
layers: vec![leaves],
delta_nodes_map: HashMap::new(),
root_to_tx_seq_map: HashMap::new(),
min_depth: None,
leaf_height,
_a: Default::default(),
};
if merkle.leaves() == 0 {
if let Some(seq) = start_tx_seq {
merkle.delta_nodes_map.insert(
seq,
DeltaNodes {
right_most_nodes: vec![],
},
);
}
return merkle;
}
// Reconstruct the whole tree.
merkle.recompute(0, 0, None);
// Commit the first version in memory.
// TODO(zz): Check when the roots become available.
merkle.commit(start_tx_seq);
merkle
}
pub fn new_with_subtrees(
initial_data: MerkleTreeInitialData<E>,
leaf_height: usize,
start_tx_seq: Option<u64>,
) -> Result<Self> {
let mut merkle = Self {
layers: vec![vec![]],
delta_nodes_map: HashMap::new(),
root_to_tx_seq_map: HashMap::new(),
min_depth: None,
leaf_height,
_a: Default::default(),
};
if initial_data.subtree_list.is_empty() {
if let Some(seq) = start_tx_seq {
merkle.delta_nodes_map.insert(
seq,
DeltaNodes {
right_most_nodes: vec![],
},
);
}
return Ok(merkle);
}
merkle.append_subtree_list(initial_data.subtree_list)?;
merkle.commit(start_tx_seq);
for (index, h) in initial_data.known_leaves {
merkle.fill_leaf(index, h);
}
Ok(merkle)
}
/// This is only used for the last chunk, so `leaf_height` is always 0 so far.
pub fn new_with_depth(leaves: Vec<E>, depth: usize, start_tx_seq: Option<u64>) -> Self {
if leaves.is_empty() {
// Create an empty merkle tree with `depth`.
let mut merkle = Self {
layers: vec![vec![]; depth],
delta_nodes_map: HashMap::new(),
root_to_tx_seq_map: HashMap::new(),
min_depth: Some(depth),
leaf_height: 0,
_a: Default::default(),
};
if let Some(seq) = start_tx_seq {
merkle.delta_nodes_map.insert(
seq,
DeltaNodes {
right_most_nodes: vec![],
},
);
}
merkle
} else {
let mut layers = vec![vec![]; depth];
layers[0] = leaves;
let mut merkle = Self {
layers,
delta_nodes_map: HashMap::new(),
root_to_tx_seq_map: HashMap::new(),
min_depth: Some(depth),
leaf_height: 0,
_a: Default::default(),
};
// Reconstruct the whole tree.
merkle.recompute(0, 0, None);
// Commit the first version in memory.
merkle.commit(start_tx_seq);
merkle
}
}
/// Return the new merkle root.
pub fn append(&mut self, new_leaf: E) {
self.layers[0].push(new_leaf);
self.recompute_after_append_leaves(self.leaves() - 1);
}
pub fn append_list(&mut self, mut leaf_list: Vec<E>) {
let start_index = self.leaves();
self.layers[0].append(&mut leaf_list);
self.recompute_after_append_leaves(start_index);
}
/// Append a leaf list by providing their intermediate node hash.
/// The appended subtree must be aligned. And it's up to the caller to
/// append the padding nodes for alignment.
/// Other nodes in the subtree will be set to `null` nodes.
/// TODO: Optimize to avoid storing the `null` nodes?
pub fn append_subtree(&mut self, subtree_depth: usize, subtree_root: E) -> Result<()> {
let start_index = self.leaves();
self.append_subtree_inner(subtree_depth, subtree_root)?;
self.recompute_after_append_subtree(start_index, subtree_depth - 1);
Ok(())
}
pub fn append_subtree_list(&mut self, subtree_list: Vec<(usize, E)>) -> Result<()> {
for (subtree_depth, subtree_root) in subtree_list {
let start_index = self.leaves();
self.append_subtree_inner(subtree_depth, subtree_root)?;
self.recompute_after_append_subtree(start_index, subtree_depth - 1);
}
Ok(())
}
/// Change the value of the last leaf and return the new merkle root.
/// This is needed if our merkle-tree in memory only keeps intermediate nodes instead of real leaves.
pub fn update_last(&mut self, updated_leaf: E) {
if self.layers[0].is_empty() {
// Special case for the first data.
self.layers[0].push(updated_leaf);
} else {
*self.layers[0].last_mut().unwrap() = updated_leaf;
}
self.recompute_after_append_leaves(self.leaves() - 1);
}
/// Fill an unknown `null` leaf with its real value.
/// Panics if the leaf changes the merkle root or the index is out of range.
/// TODO: Batch computing intermediate nodes.
pub fn fill_leaf(&mut self, index: usize, leaf: E) {
if self.layers[0][index] == E::null() {
self.layers[0][index] = leaf;
self.recompute_after_fill_leaves(index, index + 1);
} else if self.layers[0][index] != leaf {
panic!("Fill with invalid leaf")
}
}
pub fn gen_range_proof(&self, start_index: usize, end_index: usize) -> Result<RangeProof<E>> {
if end_index <= start_index {
bail!(
"invalid proof range: start={} end={}",
start_index,
end_index
);
}
// TODO(zz): Optimize range proof.
let left_proof = self.gen_proof(start_index)?;
let right_proof = self.gen_proof(end_index - 1)?;
Ok(RangeProof {
left_proof,
right_proof,
})
}
pub fn check_root(&self, root: &E) -> bool {
self.root_to_tx_seq_map.contains_key(root)
}
pub fn leaf_at(&self, position: usize) -> Result<Option<E>> {
if position >= self.leaves() {
bail!("Out of bound: position={} end={}", position, self.leaves());
}
if self.layers[0][position] != E::null() {
Ok(Some(self.layers[0][position].clone()))
} else {
// The leaf hash is unknown.
Ok(None)
}
}
/// Return a list of subtrees that can be used to rebuild the tree.
pub fn get_subtrees(&self) -> Vec<(usize, E)> {
let mut next_index = 0;
let mut subtree_list: Vec<(usize, E)> = Vec::new();
while next_index < self.leaves() {
let root_tuple = self.first_known_root_at(next_index);
let subtree_size = 1 << (root_tuple.0 - 1);
let root_start_index = next_index / subtree_size * subtree_size;
// Previous subtrees are included within the new subtree.
// Pop them out and replace with the new one.
if root_start_index < next_index {
while let Some(last) = subtree_list.pop() {
next_index -= 1 << (last.0 - 1);
if next_index == root_start_index {
break;
}
}
}
next_index += subtree_size;
subtree_list.push(root_tuple);
}
subtree_list
}
}
impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
pub fn commit(&mut self, tx_seq: Option<u64>) {
if let Some(tx_seq) = tx_seq {
if self.leaves() == 0 {
// The state is empty, so we just save the root as `null`.
// Note that this root should not be used.
self.delta_nodes_map.insert(
tx_seq,
DeltaNodes {
right_most_nodes: vec![],
},
);
return;
}
let mut right_most_nodes = Vec::new();
for layer in &self.layers {
right_most_nodes.push((layer.len() - 1, layer.last().unwrap().clone()));
}
let root = self.root().clone();
assert_eq!(root, right_most_nodes.last().unwrap().1);
self.delta_nodes_map
.insert(tx_seq, DeltaNodes::new(right_most_nodes));
self.root_to_tx_seq_map.insert(root, tx_seq);
}
}
fn before_extend_layer(&mut self, height: usize) {
if height == self.layers.len() {
self.layers.push(Vec::new());
}
}
fn recompute_after_append_leaves(&mut self, start_index: usize) {
self.recompute(start_index, 0, None)
}
fn recompute_after_append_subtree(&mut self, start_index: usize, height: usize) {
self.recompute(start_index, height, None)
}
fn recompute_after_fill_leaves(&mut self, start_index: usize, end_index: usize) {
self.recompute(start_index, 0, Some(end_index))
}
/// Given a range of changed leaf nodes and recompute the tree.
/// Since this tree is append-only, we always compute to the end.
fn recompute(
&mut self,
mut start_index: usize,
mut height: usize,
mut maybe_end_index: Option<usize>,
) {
start_index >>= height;
maybe_end_index = maybe_end_index.map(|end| end >> height);
// Loop until we compute the new root and reach `tree_depth`.
while self.layers[height].len() > 1 || height < self.layers.len() - 1 {
let next_layer_start_index = start_index >> 1;
if start_index % 2 == 1 {
start_index -= 1;
}
let mut end_index = maybe_end_index.unwrap_or(self.layers[height].len());
if end_index % 2 == 1 && end_index != self.layers[height].len() {
end_index += 1;
}
let mut i = 0;
let mut iter = self.layers[height][start_index..end_index].chunks_exact(2);
// We cannot modify the parent layer while iterating the child layer,
// so just keep the changes and update them later.
let mut parent_update = Vec::new();
while let Some([left, right]) = iter.next() {
// If either left or right is null (unknown), we cannot compute the parent hash.
// Note that if we are recompute a range of an existing tree,
// we do not need to keep these possibly null parent. This is only saved
// for the case of constructing a new tree from the leaves.
let parent = if *left == E::null() || *right == E::null() {
E::null()
} else {
A::parent(left, right)
};
parent_update.push((next_layer_start_index + i, parent));
i += 1;
}
if let [r] = iter.remainder() {
// Same as above.
let parent = if *r == E::null() {
E::null()
} else {
A::parent_single(r, height + self.leaf_height)
};
parent_update.push((next_layer_start_index + i, parent));
}
if !parent_update.is_empty() {
self.before_extend_layer(height + 1);
}
// `parent_update` is in increasing order by `parent_index`, so
// we can just overwrite `last_changed_parent_index` with new values.
let mut last_changed_parent_index = None;
for (parent_index, parent) in parent_update {
match parent_index.cmp(&self.layers[height + 1].len()) {
Ordering::Less => {
// We do not overwrite with null.
if parent != E::null() {
if self.layers[height + 1][parent_index] == E::null()
// The last node in a layer can be updated.
|| (self.layers[height + 1][parent_index] != parent
&& parent_index == self.layers[height + 1].len() - 1)
{
self.layers[height + 1][parent_index] = parent;
last_changed_parent_index = Some(parent_index);
} else if self.layers[height + 1][parent_index] != parent {
// Recompute changes a node in the middle. This should be impossible
// if the inputs are valid.
panic!("Invalid append merkle tree! height={} index={} expected={:?} get={:?}",
height + 1, parent_index, self.layers[height + 1][parent_index], parent);
}
}
}
Ordering::Equal => {
self.layers[height + 1].push(parent);
last_changed_parent_index = Some(parent_index);
}
Ordering::Greater => {
unreachable!("depth={}, parent_index={}", height, parent_index);
}
}
}
if last_changed_parent_index.is_none() {
break;
}
maybe_end_index = last_changed_parent_index.map(|i| i + 1);
height += 1;
start_index = next_layer_start_index;
}
}
fn append_subtree_inner(&mut self, subtree_depth: usize, subtree_root: E) -> Result<()> {
if subtree_depth == 0 {
bail!("Subtree depth should not be zero!");
}
if self.leaves() % (1 << (subtree_depth - 1)) != 0 {
warn!(
"The current leaves count is not aligned with the merged subtree, \
this is only possible during recovery, leaves={}",
self.leaves()
);
}
for height in 0..(subtree_depth - 1) {
self.before_extend_layer(height);
let subtree_layer_size = 1 << (subtree_depth - 1 - height);
self.layers[height].append(&mut vec![E::null(); subtree_layer_size]);
}
self.before_extend_layer(subtree_depth - 1);
self.layers[subtree_depth - 1].push(subtree_root);
Ok(())
}
#[cfg(test)]
pub fn validate(&self, proof: &Proof<E>, leaf: &E, position: usize) -> Result<bool> {
proof.validate::<A>(leaf, position)?;
Ok(self.root_to_tx_seq_map.contains_key(&proof.root()))
}
pub fn revert_to(&mut self, tx_seq: u64) -> Result<()> {
if self.layers[0].is_empty() {
// Any previous state of an empty tree is always empty.
return Ok(());
}
let delta_nodes = self
.delta_nodes_map
.get(&tx_seq)
.ok_or_else(|| anyhow!("tx_seq unavailable, root={:?}", tx_seq))?;
// Dropping the upper layers that are not in the old merkle tree.
self.layers.truncate(delta_nodes.right_most_nodes.len());
for (height, (last_index, right_most_node)) in
delta_nodes.right_most_nodes.iter().enumerate()
{
self.layers[height].truncate(*last_index + 1);
self.layers[height][*last_index] = right_most_node.clone();
}
self.clear_after(tx_seq);
Ok(())
}
pub fn at_root_version(&self, root_hash: &E) -> Result<HistoryTree<E>> {
let tx_seq = self
.root_to_tx_seq_map
.get(root_hash)
.ok_or_else(|| anyhow!("old root unavailable, root={:?}", root_hash))?;
let delta_nodes = self
.delta_nodes_map
.get(tx_seq)
.ok_or_else(|| anyhow!("tx_seq unavailable, tx_seq={:?}", tx_seq))?;
if delta_nodes.height() == 0 {
bail!("empty tree");
}
Ok(HistoryTree {
layers: &self.layers,
delta_nodes,
leaf_height: self.leaf_height,
})
}
pub fn reset(&mut self) {
self.layers = match self.min_depth {
None => vec![vec![]],
Some(depth) => vec![vec![]; depth],
};
}
fn clear_after(&mut self, tx_seq: u64) {
let mut tx_seq = tx_seq + 1;
while self.delta_nodes_map.contains_key(&tx_seq) {
if let Some(nodes) = self.delta_nodes_map.remove(&tx_seq) {
if nodes.height() != 0 {
self.root_to_tx_seq_map.remove(nodes.root());
}
}
tx_seq += 1;
}
}
/// Return the height and the root hash of the first available node from the leaf to the root.
/// The caller should ensure that `index` is within range.
fn first_known_root_at(&self, index: usize) -> (usize, E) {
let mut height = 0;
let mut index_in_layer = index;
while height < self.layers.len() {
let node = self.node(height, index_in_layer);
if !node.is_null() {
return (height + 1, node.clone());
}
height += 1;
index_in_layer /= 2;
}
unreachable!("root is always available")
}
}
#[derive(Clone, Debug)]
struct DeltaNodes<E: HashElement> {
/// The right most nodes in a layer and its position.
right_most_nodes: Vec<(usize, E)>,
}
impl<E: HashElement> DeltaNodes<E> {
fn new(right_most_nodes: Vec<(usize, E)>) -> Self {
Self { right_most_nodes }
}
fn get(&self, height: usize, position: usize) -> Result<Option<&E>> {
if height >= self.right_most_nodes.len() || position > self.right_most_nodes[height].0 {
Err(anyhow!("position out of tree range"))
} else if position == self.right_most_nodes[height].0 {
Ok(Some(&self.right_most_nodes[height].1))
} else {
Ok(None)
}
}
fn layer_len(&self, height: usize) -> usize {
self.right_most_nodes[height].0 + 1
}
fn height(&self) -> usize {
self.right_most_nodes.len()
}
fn root(&self) -> &E {
&self.right_most_nodes.last().unwrap().1
}
}
pub struct HistoryTree<'m, E: HashElement> {
/// A reference to the global tree nodes.
layers: &'m Vec<Vec<E>>,
/// The delta nodes that are difference from `layers`.
/// This could be a reference, we just take ownership for convenience.
delta_nodes: &'m DeltaNodes<E>,
leaf_height: usize,
}
impl<E: HashElement, A: Algorithm<E>> MerkleTreeRead for AppendMerkleTree<E, A> {
type E = E;
fn node(&self, layer: usize, index: usize) -> &Self::E {
&self.layers[layer][index]
}
fn height(&self) -> usize {
self.layers.len()
}
fn layer_len(&self, layer_height: usize) -> usize {
self.layers[layer_height].len()
}
fn padding_node(&self, height: usize) -> Self::E {
E::end_pad(height + self.leaf_height)
}
}
impl<'a, E: HashElement> MerkleTreeRead for HistoryTree<'a, E> {
type E = E;
fn node(&self, layer: usize, index: usize) -> &Self::E {
match self.delta_nodes.get(layer, index).expect("range checked") {
Some(node) => node,
None => &self.layers[layer][index],
}
}
fn height(&self) -> usize {
self.delta_nodes.height()
}
fn layer_len(&self, layer_height: usize) -> usize {
self.delta_nodes.layer_len(layer_height)
}
fn padding_node(&self, height: usize) -> Self::E {
E::end_pad(height + self.leaf_height)
}
}
#[macro_export]
macro_rules! ensure_eq {
($given:expr, $expected:expr) => {
ensure!(
$given == $expected,
format!(
"equal check fails! {}:{}: {}={:?}, {}={:?}",
file!(),
line!(),
stringify!($given),
$given,
stringify!($expected),
$expected,
)
);
};
}
#[cfg(test)]
mod tests {
use crate::merkle_tree::MerkleTreeRead;
use crate::sha3::Sha3Algorithm;
use crate::AppendMerkleTree;
use ethereum_types::H256;
#[test]
fn test_proof() {
let n = [1, 2, 6, 1025];
for entry_len in n {
let mut data = Vec::new();
for _ in 0..entry_len {
data.push(H256::random());
}
let mut merkle =
AppendMerkleTree::<H256, Sha3Algorithm>::new(vec![H256::zero()], 0, None);
merkle.append_list(data.clone());
merkle.commit(Some(0));
verify(&data, &merkle);
data.push(H256::random());
merkle.append(*data.last().unwrap());
merkle.commit(Some(1));
verify(&data, &merkle);
for _ in 0..6 {
data.push(H256::random());
}
merkle.append_list(data[data.len() - 6..].to_vec());
merkle.commit(Some(2));
verify(&data, &merkle);
}
}
fn verify(data: &Vec<H256>, merkle: &AppendMerkleTree<H256, Sha3Algorithm>) {
for i in 0..data.len() {
let proof = merkle.gen_proof(i + 1).unwrap();
let r = merkle.validate(&proof, &data[i], i + 1);
assert!(matches!(r, Ok(true)), "{:?}", r);
}
for i in (0..data.len()).step_by(6) {
let end = std::cmp::min(i + 3, data.len());
let range_proof = merkle.gen_range_proof(i + 1, end + 1).unwrap();
let r = range_proof.validate::<Sha3Algorithm>(&data[i..end], i + 1);
assert!(r.is_ok(), "{:?}", r);
}
}
}

View File

@ -0,0 +1,131 @@
use crate::sha3::Sha3Algorithm;
use crate::Proof;
use anyhow::{bail, Result};
use ethereum_types::H256;
use lazy_static::lazy_static;
use ssz::{Decode, Encode};
use std::fmt::Debug;
use std::hash::Hash;
use tracing::trace;
pub trait HashElement:
Clone + Debug + Eq + Hash + AsRef<[u8]> + AsMut<[u8]> + Decode + Encode + Send + Sync
{
fn end_pad(height: usize) -> Self;
fn null() -> Self;
fn is_null(&self) -> bool {
self == &Self::null()
}
}
impl HashElement for H256 {
fn end_pad(height: usize) -> Self {
ZERO_HASHES[height]
}
fn null() -> Self {
H256::repeat_byte(1)
}
}
lazy_static! {
static ref ZERO_HASHES: [H256; 64] = {
let leaf_zero_hash: H256 = Sha3Algorithm::leaf(&[0u8; 256]);
let mut list = [H256::zero(); 64];
list[0] = leaf_zero_hash;
for i in 1..list.len() {
list[i] = Sha3Algorithm::parent(&list[i - 1], &list[i - 1]);
}
list
};
}
pub trait Algorithm<E: HashElement> {
fn parent(left: &E, right: &E) -> E;
fn parent_single(r: &E, height: usize) -> E {
Self::parent(r, &E::end_pad(height))
}
fn leaf(data: &[u8]) -> E;
}
pub trait MerkleTreeRead {
type E: HashElement;
fn node(&self, layer: usize, index: usize) -> &Self::E;
fn height(&self) -> usize;
fn layer_len(&self, layer_height: usize) -> usize;
fn padding_node(&self, height: usize) -> Self::E;
fn leaves(&self) -> usize {
self.layer_len(0)
}
fn root(&self) -> &Self::E {
self.node(self.height() - 1, 0)
}
fn gen_proof(&self, leaf_index: usize) -> Result<Proof<Self::E>> {
if leaf_index >= self.leaves() {
bail!(
"leaf index out of bound: leaf_index={} total_leaves={}",
leaf_index,
self.leaves()
);
}
if self.node(0, leaf_index) == &Self::E::null() {
bail!("Not ready to generate proof for leaf_index={}", leaf_index);
}
if self.height() == 1 {
return Ok(Proof::new(
vec![self.root().clone(), self.root().clone()],
vec![],
));
}
let mut lemma: Vec<Self::E> = Vec::with_capacity(self.height()); // path + root
let mut path: Vec<bool> = Vec::with_capacity(self.height() - 2); // path - 1
let mut index_in_layer = leaf_index;
lemma.push(self.node(0, leaf_index).clone());
for height in 0..(self.height() - 1) {
trace!(
"gen_proof: height={} index={} hash={:?}",
height,
index_in_layer,
self.node(height, index_in_layer)
);
if index_in_layer % 2 == 0 {
path.push(true);
if index_in_layer + 1 == self.layer_len(height) {
// TODO: This can be skipped if the tree size is available in validation.
lemma.push(self.padding_node(height));
} else {
lemma.push(self.node(height, index_in_layer + 1).clone());
}
} else {
path.push(false);
lemma.push(self.node(height, index_in_layer - 1).clone());
}
index_in_layer >>= 1;
}
lemma.push(self.root().clone());
Ok(Proof::new(lemma, path))
}
}
/// This includes the data to reconstruct an `AppendMerkleTree` root where some nodes
/// are `null`. Other intermediate nodes will be computed based on these known nodes.
pub struct MerkleTreeInitialData<E: HashElement> {
/// A list of `(subtree_depth, root)`.
/// The subtrees are continuous so we can compute the tree root with these subtree roots.
pub subtree_list: Vec<(usize, E)>,
/// A list of `(index, leaf_hash)`.
/// These leaves are in some large subtrees of `subtree_list`. 1-node subtrees are also leaves,
/// but they will not be duplicated in `known_leaves`.
pub known_leaves: Vec<(usize, E)>,
}
impl<E: HashElement> MerkleTreeInitialData<E> {
pub fn leaves(&self) -> usize {
self.subtree_list.iter().fold(0, |acc, (subtree_depth, _)| {
acc + (1 << (subtree_depth - 1))
})
}
}

View File

@ -0,0 +1,167 @@
use crate::{ensure_eq, Algorithm, HashElement};
use anyhow::{bail, ensure, Result};
use serde::{Deserialize, Serialize};
use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
#[derive(Clone, Debug, Eq, PartialEq, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct Proof<T: HashElement> {
lemma: Vec<T>,
path: Vec<bool>,
}
impl<T: HashElement> Proof<T> {
/// Creates new MT inclusion proof
pub fn new(hash: Vec<T>, path: Vec<bool>) -> Proof<T> {
assert_eq!(hash.len() - 2, path.len());
Proof { lemma: hash, path }
}
pub fn new_empty() -> Proof<T> {
Proof {
lemma: vec![],
path: vec![],
}
}
/// Return proof target leaf
pub fn item(&self) -> T {
self.lemma.first().unwrap().clone()
}
/// Return tree root
pub fn root(&self) -> T {
self.lemma.last().unwrap().clone()
}
/// Verifies MT inclusion proof
fn validate_integrity<A: Algorithm<T>>(&self) -> bool {
let size = self.lemma.len();
if size < 2 {
return false;
}
let mut h = self.item();
for i in 1..size - 1 {
h = if self.path[i - 1] {
A::parent(&h, &self.lemma[i])
} else {
A::parent(&self.lemma[i], &h)
};
}
h == self.root()
}
pub fn validate<A: Algorithm<T>>(&self, item: &T, position: usize) -> Result<()> {
if !self.validate_integrity::<A>() {
bail!("Invalid proof");
}
if *item != self.item() {
bail!("Proof item unmatch");
}
if position != self.position() {
bail!("Proof position unmatch");
}
Ok(())
}
/// Returns the path of this proof.
pub fn path(&self) -> &[bool] {
&self.path
}
/// Returns the lemma of this proof.
pub fn lemma(&self) -> &[T] {
&self.lemma
}
pub fn position(&self) -> usize {
let mut pos = 0;
for (i, is_left) in self.path.iter().enumerate() {
if !is_left {
pos += 1 << i;
}
}
pos
}
}
#[derive(Clone, Debug, Eq, PartialEq, DeriveEncode, DeriveDecode, Deserialize, Serialize)]
pub struct RangeProof<E: HashElement> {
pub left_proof: Proof<E>,
pub right_proof: Proof<E>,
}
impl<E: HashElement> RangeProof<E> {
pub fn new_empty() -> Self {
Self {
left_proof: Proof::new_empty(),
right_proof: Proof::new_empty(),
}
}
fn validate_integrity<A: Algorithm<E>>(&self) -> bool {
self.left_proof.validate_integrity::<A>()
&& self.right_proof.validate_integrity::<A>()
&& self.left_proof.root() == self.right_proof.root()
&& self.left_proof.path().len() == self.right_proof.path().len()
}
pub fn root(&self) -> E {
self.left_proof.root()
}
pub fn validate<A: Algorithm<E>>(
&self,
range_leaves: &[E],
start_position: usize,
) -> Result<()> {
if !self.validate_integrity::<A>() {
bail!("Invalid range proof");
}
if range_leaves.is_empty() {
bail!("Empty range");
}
let end_position = start_position + range_leaves.len() - 1;
ensure_eq!(self.left_proof.item(), range_leaves[0]);
ensure_eq!(
self.right_proof.item(),
*range_leaves.last().expect("not empty")
);
ensure_eq!(self.left_proof.position(), start_position);
ensure_eq!(self.right_proof.position(), end_position);
let tree_depth = self.left_proof.path().len() + 1;
// TODO: We can avoid copying the first layer.
let mut children_layer = range_leaves.to_vec();
for height in 0..(tree_depth - 1) {
let mut parent_layer = Vec::new();
let start_index = if !self.left_proof.path()[height] {
// If the left-most node is the right child, its sibling is not within the data range and should be retrieved from the proof.
let parent = A::parent(&self.left_proof.lemma()[height + 1], &children_layer[0]);
parent_layer.push(parent);
1
} else {
// The left-most node is the left child, its sibling is just the next child.
0
};
let mut iter = children_layer[start_index..].chunks_exact(2);
while let Some([left, right]) = iter.next() {
parent_layer.push(A::parent(left, right))
}
if let [right_most] = iter.remainder() {
if self.right_proof.path()[height] {
parent_layer.push(A::parent(right_most, &self.right_proof.lemma()[height + 1]));
} else {
bail!("Unexpected error");
}
}
children_layer = parent_layer;
}
assert_eq!(children_layer.len(), 1);
let computed_root = children_layer.pop().unwrap();
ensure_eq!(computed_root, self.root());
Ok(())
}
}

View File

@ -0,0 +1,23 @@
use crate::{Algorithm, HashElement};
use tiny_keccak::{Hasher, Keccak};
pub struct Sha3Algorithm {}
impl<E: HashElement> Algorithm<E> for Sha3Algorithm {
fn parent(left: &E, right: &E) -> E {
let mut h = Keccak::v256();
let mut e = E::null();
h.update(left.as_ref());
h.update(right.as_ref());
h.finalize(e.as_mut());
e
}
fn leaf(data: &[u8]) -> E {
let mut h = Keccak::v256();
let mut e = E::null();
h.update(data.as_ref());
h.finalize(e.as_mut());
e
}
}

View File

@ -0,0 +1,7 @@
[package]
name = "channel"
version = "0.1.0"
edition = "2021"
[dependencies]
tokio = { version = "1.19.2", features = ["sync", "time"] }

View File

@ -0,0 +1,112 @@
use crate::error::Error;
use std::time::Duration;
use tokio::sync::mpsc::error::TryRecvError;
use tokio::sync::{mpsc, oneshot};
use tokio::time::timeout;
const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(3);
pub type ResponseSender<Res> = oneshot::Sender<Res>;
#[derive(Debug)]
pub enum Message<N, Req, Res> {
Notification(N),
Request(Req, ResponseSender<Res>),
}
pub struct Channel<N, Req, Res> {
_phantom: std::marker::PhantomData<(N, Req, Res)>,
}
impl<N, Req, Res> Channel<N, Req, Res> {
pub fn unbounded() -> (Sender<N, Req, Res>, Receiver<N, Req, Res>) {
let (sender, receiver) = mpsc::unbounded_channel();
(Sender { chan: sender }, Receiver { chan: receiver })
}
}
pub struct Sender<N, Req, Res> {
chan: mpsc::UnboundedSender<Message<N, Req, Res>>,
}
impl<N, Req, Res> Clone for Sender<N, Req, Res> {
fn clone(&self) -> Self {
Sender {
chan: self.chan.clone(),
}
}
}
impl<N, Req, Res> Sender<N, Req, Res> {
pub fn notify(&self, msg: N) -> Result<(), Error<N, Req, Res>> {
self.chan
.send(Message::Notification(msg))
.map_err(|e| Error::SendError(e))
}
pub async fn request(&self, request: Req) -> Result<Res, Error<N, Req, Res>> {
let (sender, receiver) = oneshot::channel();
self.chan
.send(Message::Request(request, sender))
.map_err(|e| Error::SendError(e))?;
timeout(DEFAULT_REQUEST_TIMEOUT, receiver)
.await
.map_err(|_| Error::TimeoutError)?
.map_err(|e| Error::RecvError(e))
}
}
pub struct Receiver<N, Req, Res> {
chan: mpsc::UnboundedReceiver<Message<N, Req, Res>>,
}
impl<N, Req, Res> Receiver<N, Req, Res> {
pub async fn recv(&mut self) -> Option<Message<N, Req, Res>> {
self.chan.recv().await
}
pub fn try_recv(&mut self) -> Result<Message<N, Req, Res>, TryRecvError> {
self.chan.try_recv()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Debug)]
enum Notification {}
#[derive(Debug)]
enum Request {
GetNumber,
}
#[derive(Debug, PartialEq, Eq)]
enum Response {
GetNumber(u32),
}
#[tokio::test]
async fn request_response() {
let (tx, mut rx) = Channel::<Notification, Request, Response>::unbounded();
let task1 = async move {
match rx.recv().await.expect("not dropped") {
Message::Notification(_) => {}
Message::Request(Request::GetNumber, sender) => {
sender.send(Response::GetNumber(42)).expect("not dropped");
}
}
};
let task2 = async move {
let result = tx.request(Request::GetNumber).await.expect("not dropped");
assert_eq!(result, Response::GetNumber(42));
};
tokio::join!(task1, task2);
}
}

View File

@ -0,0 +1,18 @@
use crate::Message;
use std::fmt::{Debug, Display, Formatter};
use tokio::sync::{mpsc, oneshot};
#[derive(Debug)]
pub enum Error<N, Req, Res> {
SendError(mpsc::error::SendError<Message<N, Req, Res>>),
RecvError(oneshot::error::RecvError),
TimeoutError,
}
impl<N: Debug, Req: Debug, Res: Debug> Display for Error<N, Req, Res> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "ChannelError: {:?}", self)
}
}
impl<N: Debug, Req: Debug, Res: Debug> std::error::Error for Error<N, Req, Res> {}

View File

@ -0,0 +1,5 @@
mod channel;
pub mod error;
pub mod test_util;
pub use crate::channel::{Channel, Message, Receiver, ResponseSender, Sender};

View File

@ -0,0 +1,55 @@
use std::{
fmt::Debug,
ops::{Deref, DerefMut},
time::Duration,
};
use tokio::time::timeout;
use crate::{Message, Receiver};
pub struct TestReceiver<N, Req, Res> {
recv: Receiver<N, Req, Res>,
}
impl<N, Req, Res> From<Receiver<N, Req, Res>> for TestReceiver<N, Req, Res> {
fn from(recv: Receiver<N, Req, Res>) -> Self {
Self { recv }
}
}
impl<N, Req, Res> Deref for TestReceiver<N, Req, Res> {
type Target = Receiver<N, Req, Res>;
fn deref(&self) -> &Self::Target {
&self.recv
}
}
impl<N, Req, Res> DerefMut for TestReceiver<N, Req, Res> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.recv
}
}
impl<N, Req, Res: Debug> TestReceiver<N, Req, Res> {
pub async fn expect_response(&mut self, response: Res) {
let request = timeout(Duration::from_secs(3), self.recv())
.await
.expect("Timeout to receive request")
.expect("Channel closed");
match request {
Message::Notification(..) => panic!("Unexpected message type"),
Message::Request(_, resp_sender) => {
resp_sender.send(response).expect("Channel closed");
}
}
}
pub async fn expect_responses(&mut self, responses: Vec<Res>) {
for resp in responses {
self.expect_response(resp).await;
}
}
}

View File

@ -0,0 +1,14 @@
[package]
name = "contract-interface"
version = "0.1.0"
edition = "2021"
build = "build.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
ethers = "^2"
serde_json = "1.0.82"
[features]
compile-contracts = []

View File

@ -0,0 +1,30 @@
use std::process::Command;
const INSTALL_ERROR_MESSAGE: &str =
"Install dependencies for contract fail, try to run `yarn` in folder 'zerog-storage-contracts'";
const COMPILE_ERROR_MESSAGE: &str =
"Compile solidity contracts fail, try to run `yarn compile` in folder 'zerog-storage-contracts'";
fn main() {
if cfg!(feature = "compile-contracts") {
println!("cargo:rerun-if-changed=../../zerog-storage-contracts/contracts/");
println!("cargo:rerun-if-changed=../../zerog-storage-contracts/hardhat.config.ts");
let output = Command::new("yarn")
.arg("--cwd")
.arg("../../zerog-storage-contracts")
.status()
.expect(INSTALL_ERROR_MESSAGE);
assert!(output.success(), "{}", INSTALL_ERROR_MESSAGE);
let output = Command::new("yarn")
.arg("--cwd")
.arg("../../zerog-storage-contracts")
.arg("compile")
.status()
.expect(COMPILE_ERROR_MESSAGE);
assert!(output.success(), "{}", COMPILE_ERROR_MESSAGE);
} else {
println!("cargo:rerun-if-changed=../../zerog-storage-contracts/artifacts/");
}
}

View File

@ -0,0 +1,13 @@
use ethers::prelude::abigen;
// run `cargo doc -p contract-interface --open` to read struct definition
abigen!(
ZgsFlow,
"../../zerog-storage-contracts/artifacts/contracts/dataFlow/Flow.sol/Flow.json"
);
abigen!(
PoraMine,
"../../zerog-storage-contracts/artifacts/contracts/test/PoraMineTest.sol/PoraMineTest.json"
);

View File

@ -0,0 +1,9 @@
[package]
name = "directory"
version = "0.1.0"
authors = ["pawan <pawandhananjay@gmail.com>"]
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]

View File

@ -0,0 +1,12 @@
/// Names for the default directories.
pub const DEFAULT_ROOT_DIR: &str = ".lighthouse";
pub const DEFAULT_BEACON_NODE_DIR: &str = "beacon";
pub const DEFAULT_NETWORK_DIR: &str = "network";
pub const DEFAULT_VALIDATOR_DIR: &str = "validators";
pub const DEFAULT_SECRET_DIR: &str = "secrets";
pub const DEFAULT_WALLET_DIR: &str = "wallets";
pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet";
/// Base directory name for unnamed testnets passed through the --testnet-dir flag
pub const CUSTOM_TESTNET_DIR: &str = "custom";

View File

@ -0,0 +1,12 @@
[package]
name = "hashset_delay"
version = "0.2.0"
authors = ["Sigma Prime <contact@sigmaprime.io>"]
edition = "2021"
[dependencies]
futures = "0.3.21"
tokio-util = { version = "0.6.10", features = ["time"] }
[dev-dependencies]
tokio = { version = "1.19.2", features = ["time", "rt-multi-thread", "macros"] }

View File

@ -0,0 +1,197 @@
//NOTE: This is just a specific case of a HashMapDelay.
// The code has been copied to make unique `insert` and `insert_at` functions.
/// The default delay for entries, in seconds. This is only used when `insert()` is used to add
/// entries.
const DEFAULT_DELAY: u64 = 30;
use futures::prelude::*;
use std::{
collections::HashMap,
pin::Pin,
task::{Context, Poll},
time::{Duration, Instant},
};
use tokio_util::time::delay_queue::{self, DelayQueue};
pub struct HashSetDelay<K>
where
K: std::cmp::Eq + std::hash::Hash + std::clone::Clone + Unpin,
{
/// The given entries.
entries: HashMap<K, MapEntry>,
/// A queue holding the timeouts of each entry.
expirations: DelayQueue<K>,
/// The default expiration timeout of an entry.
default_entry_timeout: Duration,
}
/// A wrapping around entries that adds the link to the entry's expiration, via a `delay_queue` key.
struct MapEntry {
/// The expiration key for the entry.
key: delay_queue::Key,
/// The actual entry.
value: Instant,
}
impl<K> Default for HashSetDelay<K>
where
K: std::cmp::Eq + std::hash::Hash + std::clone::Clone + Unpin,
{
fn default() -> Self {
HashSetDelay::new(Duration::from_secs(DEFAULT_DELAY))
}
}
impl<K> HashSetDelay<K>
where
K: std::cmp::Eq + std::hash::Hash + std::clone::Clone + Unpin,
{
/// Creates a new instance of `HashSetDelay`.
pub fn new(default_entry_timeout: Duration) -> Self {
HashSetDelay {
entries: HashMap::new(),
expirations: DelayQueue::new(),
default_entry_timeout,
}
}
/// Insert an entry into the mapping. Entries will expire after the `default_entry_timeout`.
pub fn insert(&mut self, key: K) {
self.insert_at(key, self.default_entry_timeout);
}
/// Inserts an entry that will expire at a given instant. If the entry already exists, the
/// timeout is updated.
pub fn insert_at(&mut self, key: K, entry_duration: Duration) {
if self.contains(&key) {
// update the timeout
self.update_timeout(&key, entry_duration);
} else {
let delay_key = self.expirations.insert(key.clone(), entry_duration);
let entry = MapEntry {
key: delay_key,
value: Instant::now() + entry_duration,
};
self.entries.insert(key, entry);
}
}
/// Gets a reference to an entry if it exists.
///
/// Returns None if the entry does not exist.
pub fn get(&self, key: &K) -> Option<&Instant> {
self.entries.get(key).map(|entry| &entry.value)
}
/// Returns true if the key exists, false otherwise.
pub fn contains(&self, key: &K) -> bool {
self.entries.contains_key(key)
}
/// Returns the length of the mapping.
pub fn len(&self) -> usize {
self.entries.len()
}
/// Checks if the mapping is empty.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Updates the timeout for a given key. Returns true if the key existed, false otherwise.
///
/// Panics if the duration is too far in the future.
pub fn update_timeout(&mut self, key: &K, timeout: Duration) -> bool {
if let Some(entry) = self.entries.get(key) {
self.expirations.reset(&entry.key, timeout);
true
} else {
false
}
}
/// Removes a key from the map returning the value associated with the key that was in the map.
///
/// Return false if the key was not in the map.
pub fn remove(&mut self, key: &K) -> bool {
if let Some(entry) = self.entries.remove(key) {
self.expirations.remove(&entry.key);
return true;
}
false
}
/// Retains only the elements specified by the predicate.
///
/// In other words, remove all pairs `(k, v)` such that `f(&k,&mut v)` returns false.
pub fn retain<F: FnMut(&K) -> bool>(&mut self, mut f: F) {
let expiration = &mut self.expirations;
self.entries.retain(|key, entry| {
let result = f(key);
if !result {
expiration.remove(&entry.key);
}
result
})
}
/// Removes all entries from the map.
pub fn clear(&mut self) {
self.entries.clear();
self.expirations.clear();
}
/// Returns a vector of referencing all keys in the map.
pub fn keys(&self) -> impl Iterator<Item = &K> {
self.entries.keys()
}
}
impl<K> Stream for HashSetDelay<K>
where
K: std::cmp::Eq + std::hash::Hash + std::clone::Clone + Unpin,
{
type Item = Result<K, String>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
match self.expirations.poll_expired(cx) {
Poll::Ready(Some(Ok(key))) => match self.entries.remove(key.get_ref()) {
Some(_) => Poll::Ready(Some(Ok(key.into_inner()))),
None => Poll::Ready(Some(Err("Value no longer exists in expirations".into()))),
},
Poll::Ready(Some(Err(e))) => {
Poll::Ready(Some(Err(format!("delay queue error: {:?}", e))))
}
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn should_not_panic() {
let key = 2u8;
let mut map = HashSetDelay::default();
map.insert(key);
map.update_timeout(&key, Duration::from_secs(100));
let fut = |cx: &mut Context| {
let _ = map.poll_next_unpin(cx);
let _ = map.poll_next_unpin(cx);
Poll::Ready(())
};
future::poll_fn(fut).await;
map.insert(key);
map.update_timeout(&key, Duration::from_secs(100));
}
}

View File

@ -0,0 +1,12 @@
//! This crate provides a single type (its counter-part HashMapDelay has been removed as it
//! currently is not in use in lighthouse):
//! - `HashSetDelay`
//!
//! # HashSetDelay
//!
//! This is similar to a `HashMapDelay` except the mapping maps to the expiry time. This
//! allows users to add objects and check their expiry deadlines before the `Stream`
//! consumes them.
mod hashset_delay;
pub use crate::hashset_delay::HashSetDelay;

View File

@ -0,0 +1,11 @@
[package]
name = "lighthouse_metrics"
version = "0.2.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
lazy_static = "1.4.0"
prometheus = "0.13.1"

View File

@ -0,0 +1,359 @@
#![allow(clippy::needless_doctest_main)]
//! A wrapper around the `prometheus` crate that provides a global, `lazy_static` metrics registry
//! and functions to add and use the following components (more info at
//! [Prometheus docs](https://prometheus.io/docs/concepts/metric_types/)):
//!
//! - `Histogram`: used with `start_timer(..)` and `stop_timer(..)` to record durations (e.g.,
//! block processing time).
//! - `IncCounter`: used to represent an ideally ever-growing, never-shrinking integer (e.g.,
//! number of block processing requests).
//! - `IntGauge`: used to represent an varying integer (e.g., number of attestations per block).
//!
//! ## Important
//!
//! Metrics will fail if two items have the same `name`. All metrics must have a unique `name`.
//! Because we use a global registry there is no namespace per crate, it's one big global space.
//!
//! See the [Prometheus naming best practices](https://prometheus.io/docs/practices/naming/) when
//! choosing metric names.
//!
//! ## Example
//!
//! ```rust
//! #[macro_use]
//! extern crate lazy_static;
//! use lighthouse_metrics::*;
//!
//! // These metrics are "magically" linked to the global registry defined in `lighthouse_metrics`.
//! lazy_static! {
//! pub static ref RUN_COUNT: Result<IntCounter> = try_create_int_counter(
//! "runs_total",
//! "Total number of runs"
//! );
//! pub static ref CURRENT_VALUE: Result<IntGauge> = try_create_int_gauge(
//! "current_value",
//! "The current value"
//! );
//! pub static ref RUN_TIME: Result<Histogram> =
//! try_create_histogram("run_seconds", "Time taken (measured to high precision)");
//! }
//!
//!
//! fn main() {
//! for i in 0..100 {
//! inc_counter(&RUN_COUNT);
//! let timer = start_timer(&RUN_TIME);
//!
//! for j in 0..10 {
//! set_gauge(&CURRENT_VALUE, j);
//! println!("Howdy partner");
//! }
//!
//! stop_timer(timer);
//! }
//! }
//! ```
use prometheus::{HistogramOpts, Opts};
use std::time::Duration;
use prometheus::core::{Atomic, GenericGauge, GenericGaugeVec};
pub use prometheus::{
proto::{Metric, MetricFamily, MetricType},
Encoder, Gauge, GaugeVec, Histogram, HistogramTimer, HistogramVec, IntCounter, IntCounterVec,
IntGauge, IntGaugeVec, Result, TextEncoder,
};
/// Collect all the metrics for reporting.
pub fn gather() -> Vec<prometheus::proto::MetricFamily> {
prometheus::gather()
}
/// Attempts to create an `IntCounter`, returning `Err` if the registry does not accept the counter
/// (potentially due to naming conflict).
pub fn try_create_int_counter(name: &str, help: &str) -> Result<IntCounter> {
let opts = Opts::new(name, help);
let counter = IntCounter::with_opts(opts)?;
prometheus::register(Box::new(counter.clone()))?;
Ok(counter)
}
/// Attempts to create an `IntGauge`, returning `Err` if the registry does not accept the counter
/// (potentially due to naming conflict).
pub fn try_create_int_gauge(name: &str, help: &str) -> Result<IntGauge> {
let opts = Opts::new(name, help);
let gauge = IntGauge::with_opts(opts)?;
prometheus::register(Box::new(gauge.clone()))?;
Ok(gauge)
}
/// Attempts to create a `Gauge`, returning `Err` if the registry does not accept the counter
/// (potentially due to naming conflict).
pub fn try_create_float_gauge(name: &str, help: &str) -> Result<Gauge> {
let opts = Opts::new(name, help);
let gauge = Gauge::with_opts(opts)?;
prometheus::register(Box::new(gauge.clone()))?;
Ok(gauge)
}
/// Attempts to create a `Histogram`, returning `Err` if the registry does not accept the counter
/// (potentially due to naming conflict).
pub fn try_create_histogram(name: &str, help: &str) -> Result<Histogram> {
let opts = HistogramOpts::new(name, help);
let histogram = Histogram::with_opts(opts)?;
prometheus::register(Box::new(histogram.clone()))?;
Ok(histogram)
}
/// Attempts to create a `HistogramVec`, returning `Err` if the registry does not accept the counter
/// (potentially due to naming conflict).
pub fn try_create_histogram_vec(
name: &str,
help: &str,
label_names: &[&str],
) -> Result<HistogramVec> {
let opts = HistogramOpts::new(name, help);
let histogram_vec = HistogramVec::new(opts, label_names)?;
prometheus::register(Box::new(histogram_vec.clone()))?;
Ok(histogram_vec)
}
/// Attempts to create a `IntGaugeVec`, returning `Err` if the registry does not accept the gauge
/// (potentially due to naming conflict).
pub fn try_create_int_gauge_vec(
name: &str,
help: &str,
label_names: &[&str],
) -> Result<IntGaugeVec> {
let opts = Opts::new(name, help);
let counter_vec = IntGaugeVec::new(opts, label_names)?;
prometheus::register(Box::new(counter_vec.clone()))?;
Ok(counter_vec)
}
/// Attempts to create a `GaugeVec`, returning `Err` if the registry does not accept the gauge
/// (potentially due to naming conflict).
pub fn try_create_float_gauge_vec(
name: &str,
help: &str,
label_names: &[&str],
) -> Result<GaugeVec> {
let opts = Opts::new(name, help);
let counter_vec = GaugeVec::new(opts, label_names)?;
prometheus::register(Box::new(counter_vec.clone()))?;
Ok(counter_vec)
}
/// Attempts to create a `IntCounterVec`, returning `Err` if the registry does not accept the gauge
/// (potentially due to naming conflict).
pub fn try_create_int_counter_vec(
name: &str,
help: &str,
label_names: &[&str],
) -> Result<IntCounterVec> {
let opts = Opts::new(name, help);
let counter_vec = IntCounterVec::new(opts, label_names)?;
prometheus::register(Box::new(counter_vec.clone()))?;
Ok(counter_vec)
}
/// If `int_gauge_vec.is_ok()`, returns a gauge with the given `name`.
pub fn get_int_gauge(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str]) -> Option<IntGauge> {
if let Ok(int_gauge_vec) = int_gauge_vec {
Some(int_gauge_vec.get_metric_with_label_values(name).ok()?)
} else {
None
}
}
pub fn get_gauge<P: Atomic>(
gauge_vec: &Result<GenericGaugeVec<P>>,
name: &[&str],
) -> Option<GenericGauge<P>> {
if let Ok(gauge_vec) = gauge_vec {
Some(gauge_vec.get_metric_with_label_values(name).ok()?)
} else {
None
}
}
pub fn set_gauge_entry<P: Atomic>(
gauge_vec: &Result<GenericGaugeVec<P>>,
name: &[&str],
value: P::T,
) {
if let Some(v) = get_gauge(gauge_vec, name) {
v.set(value)
};
}
/// If `int_gauge_vec.is_ok()`, sets the gauge with the given `name` to the given `value`
/// otherwise returns false.
pub fn set_int_gauge(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str], value: i64) -> bool {
if let Ok(int_gauge_vec) = int_gauge_vec {
int_gauge_vec
.get_metric_with_label_values(name)
.map(|v| {
v.set(value);
true
})
.unwrap_or_else(|_| false)
} else {
false
}
}
/// If `int_counter_vec.is_ok()`, returns a counter with the given `name`.
pub fn get_int_counter(
int_counter_vec: &Result<IntCounterVec>,
name: &[&str],
) -> Option<IntCounter> {
if let Ok(int_counter_vec) = int_counter_vec {
Some(int_counter_vec.get_metric_with_label_values(name).ok()?)
} else {
None
}
}
/// Increments the `int_counter_vec` with the given `name`.
pub fn inc_counter_vec(int_counter_vec: &Result<IntCounterVec>, name: &[&str]) {
if let Some(counter) = get_int_counter(int_counter_vec, name) {
counter.inc()
}
}
pub fn inc_counter_vec_by(int_counter_vec: &Result<IntCounterVec>, name: &[&str], amount: u64) {
if let Some(counter) = get_int_counter(int_counter_vec, name) {
counter.inc_by(amount);
}
}
/// If `histogram_vec.is_ok()`, returns a histogram with the given `name`.
pub fn get_histogram(histogram_vec: &Result<HistogramVec>, name: &[&str]) -> Option<Histogram> {
if let Ok(histogram_vec) = histogram_vec {
Some(histogram_vec.get_metric_with_label_values(name).ok()?)
} else {
None
}
}
/// Starts a timer on `vec` with the given `name`.
pub fn start_timer_vec(vec: &Result<HistogramVec>, name: &[&str]) -> Option<HistogramTimer> {
get_histogram(vec, name).map(|h| h.start_timer())
}
/// Starts a timer for the given `Histogram`, stopping when it gets dropped or given to `stop_timer(..)`.
pub fn start_timer(histogram: &Result<Histogram>) -> Option<HistogramTimer> {
if let Ok(histogram) = histogram {
Some(histogram.start_timer())
} else {
None
}
}
/// Starts a timer on `vec` with the given `name`.
pub fn observe_timer_vec(vec: &Result<HistogramVec>, name: &[&str], duration: Duration) {
if let Some(h) = get_histogram(vec, name) {
h.observe(duration_to_f64(duration))
}
}
/// Stops a timer created with `start_timer(..)`.
pub fn stop_timer(timer: Option<HistogramTimer>) {
if let Some(t) = timer {
t.observe_duration()
}
}
pub fn inc_counter(counter: &Result<IntCounter>) {
if let Ok(counter) = counter {
counter.inc();
}
}
pub fn inc_counter_by(counter: &Result<IntCounter>, value: u64) {
if let Ok(counter) = counter {
counter.inc_by(value);
}
}
pub fn set_gauge_vec(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str], value: i64) {
if let Some(gauge) = get_int_gauge(int_gauge_vec, name) {
gauge.set(value);
}
}
pub fn inc_gauge_vec(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str]) {
if let Some(gauge) = get_int_gauge(int_gauge_vec, name) {
gauge.inc();
}
}
pub fn dec_gauge_vec(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str]) {
if let Some(gauge) = get_int_gauge(int_gauge_vec, name) {
gauge.dec();
}
}
pub fn set_gauge(gauge: &Result<IntGauge>, value: i64) {
if let Ok(gauge) = gauge {
gauge.set(value);
}
}
pub fn set_float_gauge(gauge: &Result<Gauge>, value: f64) {
if let Ok(gauge) = gauge {
gauge.set(value);
}
}
pub fn set_float_gauge_vec(gauge_vec: &Result<GaugeVec>, name: &[&str], value: f64) {
if let Some(gauge) = get_gauge(gauge_vec, name) {
gauge.set(value);
}
}
pub fn inc_gauge(gauge: &Result<IntGauge>) {
if let Ok(gauge) = gauge {
gauge.inc();
}
}
pub fn dec_gauge(gauge: &Result<IntGauge>) {
if let Ok(gauge) = gauge {
gauge.dec();
}
}
pub fn maybe_set_gauge(gauge: &Result<IntGauge>, value_opt: Option<i64>) {
if let Some(value) = value_opt {
set_gauge(gauge, value)
}
}
pub fn maybe_set_float_gauge(gauge: &Result<Gauge>, value_opt: Option<f64>) {
if let Some(value) = value_opt {
set_float_gauge(gauge, value)
}
}
/// Sets the value of a `Histogram` manually.
pub fn observe(histogram: &Result<Histogram>, value: f64) {
if let Ok(histogram) = histogram {
histogram.observe(value);
}
}
pub fn observe_duration(histogram: &Result<Histogram>, duration: Duration) {
if let Ok(histogram) = histogram {
histogram.observe(duration_to_f64(duration))
}
}
fn duration_to_f64(duration: Duration) -> f64 {
// This conversion was taken from here:
//
// https://docs.rs/prometheus/0.5.0/src/prometheus/histogram.rs.html#550-555
let nanos = f64::from(duration.subsec_nanos()) / 1e9;
duration.as_secs() as f64 + nanos
}

View File

@ -0,0 +1,36 @@
[package]
name = "merkle_light"
version = "0.4.0"
authors = [
"Ivan Prisyazhnyy <john.koepi@gmail.com>"
]
edition = "2021"
description = "Light merkle tree implementation with SPV support and dependency agnostic."
license = "BSD-3-Clause"
homepage = "https://github.com/sitano/merkle_light"
repository = "https://github.com/sitano/merkle_light"
documentation = "https://sitano.github.io/merkle_light/merkle_light/index.html"
readme = "README.md"
keywords = ["merkle", "merkle-tree", "no_std"]
categories = ["data-structures", "cryptography"]
[dependencies]
rayon = "1.5.3"
[dev-dependencies]
[features]
default = ["std"]
std = []
[package.metadata.release]
sign-commit = true
upload-doc = true
doc-branch = "gh-pages"
pre-release-commit-message = "Release version {{version}}."
pro-release-commit-message = "Start next development iteration {{version}}."
tag-message = "Release version {{version}}."
doc-commit-message = "Update documentation."
dev-version-ext = "pre"

View File

@ -0,0 +1,113 @@
# merkle
[![Build Status](https://travis-ci.org/sitano/merkle_light.svg?branch=master&style=flat)](https://travis-ci.org/sitano/merkle_light)
[![Issues](http://img.shields.io/github/issues/sitano/merkle.svg?style=flat)](https://github.com/sitano/merkle_light/issues)
![License](https://img.shields.io/badge/license-bsd3-brightgreen.svg?style=flat)
[![Crates.io](https://img.shields.io/crates/v/merkle_light.svg)](https://crates.io/crates/merkle_light)
*merkle* is a lightweight Rust implementation of a [Merkle tree](https://en.wikipedia.org/wiki/Merkle_tree).
## Features
- external dependency agnostic
- `core::hash::Hasher` compatibility
- standard types hasher implementations
- `#[derive(Hashable)]` support for simple struct
- customizable merkle leaf/node hashing algorithm
- support for custom hash types (e.g. [u8; 16], [u64; 4], [u128; 2], struct)
- customizable hashing algorithm
- linear memory layout, no nodes on heap
- buildable from iterator, objects or hashes
- certificate transparency style merkle hashing support
- SPV included
## Documentation
Documentation is [available](https://sitano.github.io/merkle_light/merkle_light/index.html).
# Examples
* `test_sip.rs`: algorithm implementation example for std sip hasher, u64 hash items
* `test_xor128.rs`: custom hash example xor128
* `test_cmh.rs`: custom merkle hasher implementation example
* `crypto_bitcoin_mt.rs`: bitcoin merkle tree using crypto lib
* `crypto_chaincore_mt.rs`: chain core merkle tree using crypto lib
* `ring_bitcoin_mt.rs`: bitcoin merkle tree using ring lib
# Quick start
```
extern crate crypto;
extern crate merkle_light;
use std::fmt;
use std::hash::Hasher;
use std::iter::FromIterator;
use crypto::sha3::{Sha3, Sha3Mode};
use crypto::digest::Digest;
use merkle_light::hash::{Algorithm, Hashable};
use merkle_light::merkle::MerkleTree;
pub struct ExampleAlgorithm(Sha3);
impl ExampleAlgorithm {
pub fn new() -> ExampleAlgorithm {
ExampleAlgorithm(Sha3::new(Sha3Mode::Sha3_256))
}
}
impl Default for ExampleAlgorithm {
fn default() -> ExampleAlgorithm {
ExampleAlgorithm::new()
}
}
impl Hasher for ExampleAlgorithm {
#[inline]
fn write(&mut self, msg: &[u8]) {
self.0.input(msg)
}
#[inline]
fn finish(&self) -> u64 {
unimplemented!()
}
}
impl Algorithm<[u8; 32]> for ExampleAlgorithm {
#[inline]
fn hash(&mut self) -> [u8; 32] {
let mut h = [0u8; 32];
self.0.result(&mut h);
h
}
#[inline]
fn reset(&mut self) {
self.0.reset();
}
}
fn main() {
let mut h1 = [0u8; 32];
let mut h2 = [0u8; 32];
let mut h3 = [0u8; 32];
h1[0] = 0x11;
h2[0] = 0x22;
h3[0] = 0x33;
let t: MerkleTree<[u8; 32], ExampleAlgorithm> = MerkleTree::from_iter(vec![h1, h2, h3]);
println!("{:?}", t.root());
}
```
## Bug Reporting
Please report bugs either as pull requests or as issues in [the issue
tracker](https://github.com/sitano/merkle_light). *merkle* has a
**full disclosure** vulnerability policy. **Please do NOT attempt to report
any security vulnerability in this code privately to anybody.**
## License
See [LICENSE](LICENSE).

View File

@ -0,0 +1,170 @@
//! cargo bench --features "crypto_bench" --verbose
#![cfg(feature = "crypto_bench")]
#![feature(test)]
#![feature(rand)]
#![feature(crypto)]
mod hash512;
extern crate crypto;
extern crate merkle_light;
extern crate rand;
extern crate test;
use crypto::digest::Digest;
use crypto::sha2::Sha512;
use hash512::Hash512;
use merkle_light::hash::{Algorithm, Hashable};
use merkle_light::merkle::MerkleTree;
use rand::Rng;
use std::hash::Hasher;
use std::iter::FromIterator;
use test::Bencher;
#[derive(Copy, Clone)]
struct A(Sha512);
impl A {
fn new() -> A {
A(Sha512::new())
}
}
impl Default for A {
fn default() -> Self {
A::new()
}
}
impl Hasher for A {
#[inline]
fn write(&mut self, msg: &[u8]) {
self.0.input(msg)
}
#[inline]
fn finish(&self) -> u64 {
unimplemented!()
}
}
impl Algorithm<Hash512> for A {
#[inline]
fn hash(&mut self) -> Hash512 {
let mut h = [0u8; 64];
self.0.result(&mut h);
Hash512(h)
}
#[inline]
fn reset(&mut self) {
self.0.reset();
}
}
fn tree_5() -> Vec<Hash512> {
["one", "two", "three", "four"]
.iter()
.map(|x| {
let mut a = A::new();
Hashable::hash(x, &mut a);
a.hash()
})
.collect::<Vec<Hash512>>()
}
fn tree_160() -> Vec<Hash512> {
let mut values = vec![vec![0u8; 256]; 160];
let mut rng = rand::IsaacRng::new_unseeded();
for mut v in &mut values {
rng.fill_bytes(&mut v);
}
values
.iter()
.map(|x| {
let mut a = A::new();
a.write(x.as_ref());
a.hash()
})
.collect::<Vec<Hash512>>()
}
#[bench]
fn bench_crypto_sha512(b: &mut Bencher) {
let mut h = [0u8; 64];
b.iter(|| {
let mut x = Sha512::new();
x.input("12345".as_ref());
x.result(&mut h);
});
}
#[bench]
fn bench_crypto_sha512_from_data_5(b: &mut Bencher) {
let values = tree_5();
b.iter(|| MerkleTree::<Hash512, A>::from_iter(values.clone()));
}
#[bench]
fn bench_crypto_sha512_from_data_5_proof(b: &mut Bencher) {
let values = tree_5();
let tree: MerkleTree<Hash512, A> = MerkleTree::from_iter(values.clone());
b.iter(|| {
for i in 0..values.len() {
let proof = tree.gen_proof(i);
test::black_box(proof);
}
});
}
#[bench]
fn bench_crypto_sha512_from_data_5_proof_check(b: &mut Bencher) {
let values = tree_5();
let tree: MerkleTree<Hash512, A> = MerkleTree::from_iter(values.clone());
let proofs = (0..values.len())
.map(|i| tree.gen_proof(i))
.collect::<Vec<_>>();
b.iter(|| {
for proof in &proofs {
test::black_box(proof.validate::<A>());
}
});
}
#[bench]
fn bench_crypto_sha512_from_data_160(b: &mut Bencher) {
let values = tree_160();
b.iter(|| MerkleTree::<Hash512, A>::from_iter(values.clone()));
}
#[bench]
fn bench_crypto_sha512_from_data_160_proof(b: &mut Bencher) {
let values = tree_160();
let tree: MerkleTree<Hash512, A> = MerkleTree::from_iter(values.clone());
b.iter(|| {
for i in 0..values.len() {
let proof = tree.gen_proof(i);
test::black_box(proof);
}
});
}
#[bench]
fn bench_crypto_sha512_from_data_160_proof_check(b: &mut Bencher) {
let values = tree_160();
let tree: MerkleTree<Hash512, A> = MerkleTree::from_iter(values.clone());
let proofs = (0..values.len())
.map(|i| tree.gen_proof(i))
.collect::<Vec<_>>();
b.iter(|| {
for proof in &proofs {
test::black_box(proof.validate::<A>());
}
});
}

View File

@ -0,0 +1,58 @@
use std::cmp::Ordering;
#[derive(Copy, Clone)]
pub struct Hash512(pub [u8; 64]);
impl Default for Hash512 {
fn default() -> Self {
Hash512([0u8; 64])
}
}
impl AsRef<[u8]> for Hash512 {
fn as_ref(&self) -> &[u8] {
self.0.as_ref()
}
}
impl PartialOrd for Hash512 {
#[inline]
fn partial_cmp(&self, other: &Hash512) -> Option<Ordering> {
PartialOrd::partial_cmp(&&self.0[..], &&other.0[..])
}
#[inline]
fn lt(&self, other: &Hash512) -> bool {
PartialOrd::lt(&&self.0[..], &&other.0[..])
}
#[inline]
fn le(&self, other: &Hash512) -> bool {
PartialOrd::le(&&self.0[..], &&other.0[..])
}
#[inline]
fn ge(&self, other: &Hash512) -> bool {
PartialOrd::ge(&&self.0[..], &&other.0[..])
}
#[inline]
fn gt(&self, other: &Hash512) -> bool {
PartialOrd::gt(&&self.0[..], &&other.0[..])
}
}
impl Ord for Hash512 {
#[inline]
fn cmp(&self, other: &Hash512) -> Ordering {
Ord::cmp(&&self.0[..], &&other.0[..])
}
}
impl PartialEq for Hash512 {
fn eq(&self, other: &Hash512) -> bool {
self.0.as_ref() == other.0.as_ref()
}
}
impl Eq for Hash512 {}

View File

@ -0,0 +1,3 @@
mod hash512;
pub use hash512::hash512::Hash512;

View File

@ -0,0 +1,168 @@
//! cargo bench --features "crypto_bench" --verbose
#![cfg(feature = "crypto_bench")]
#![feature(test)]
#![feature(rand)]
mod hash512;
mod ringx;
extern crate merkle_light;
extern crate rand;
extern crate test;
use hash512::Hash512;
use merkle_light::hash::{Algorithm, Hashable};
use merkle_light::merkle::MerkleTree;
use rand::Rng;
use ringx::Context;
use ringx::SHA512;
use std::hash::Hasher;
use std::iter::FromIterator;
use test::Bencher;
#[derive(Clone)]
struct B(Context);
impl B {
fn new() -> B {
B(Context::new(&SHA512))
}
}
impl Default for B {
fn default() -> Self {
B::new()
}
}
impl Hasher for B {
#[inline]
fn write(&mut self, msg: &[u8]) {
self.0.update(msg)
}
#[inline]
fn finish(&self) -> u64 {
unimplemented!()
}
}
impl Algorithm<Hash512> for B {
#[inline]
fn hash(&mut self) -> Hash512 {
let mut h = [0u8; 64];
h.copy_from_slice(self.0.finish().as_ref());
Hash512(h)
}
#[inline]
fn reset(&mut self) {
self.0.reset();
}
}
fn tree_5() -> Vec<Hash512> {
["one", "two", "three", "four"]
.iter()
.map(|x| {
let mut a = B::new();
Hashable::hash(x, &mut a);
a.hash()
})
.collect::<Vec<Hash512>>()
}
fn tree_160() -> Vec<Hash512> {
let mut values = vec![vec![0u8; 256]; 160];
let mut rng = rand::IsaacRng::new_unseeded();
for mut v in &mut values {
rng.fill_bytes(&mut v);
}
values
.iter()
.map(|x| {
let mut a = B::new();
a.write(x.as_ref());
a.hash()
})
.collect::<Vec<Hash512>>()
}
#[bench]
fn bench_ringx_sha512(b: &mut Bencher) {
b.iter(|| {
let mut x = Context::new(&SHA512);
x.update("12345".as_ref());
x.finish();
});
}
#[bench]
fn bench_ringx_sha512_from_data_5(b: &mut Bencher) {
let values = tree_5();
b.iter(|| MerkleTree::<Hash512, B>::from_iter(values.clone()));
}
#[bench]
fn bench_ringx_sha512_from_data_5_proof(b: &mut Bencher) {
let values = tree_5();
let tree: MerkleTree<Hash512, B> = MerkleTree::from_iter(values.clone());
b.iter(|| {
for i in 0..values.len() {
let proof = tree.gen_proof(i);
test::black_box(proof);
}
});
}
#[bench]
fn bench_ringx_sha512_from_data_5_proof_check(b: &mut Bencher) {
let values = tree_5();
let tree: MerkleTree<Hash512, B> = MerkleTree::from_iter(values.clone());
let proofs = (0..values.len())
.map(|i| tree.gen_proof(i))
.collect::<Vec<_>>();
b.iter(|| {
for proof in &proofs {
test::black_box(proof.validate::<B>());
}
});
}
#[bench]
fn bench_ringx_sha512_from_data_160(b: &mut Bencher) {
let values = tree_160();
b.iter(|| MerkleTree::<Hash512, B>::from_iter(values.clone()));
}
#[bench]
fn bench_ringx_sha512_from_data_160_proof(b: &mut Bencher) {
let values = tree_160();
let tree: MerkleTree<Hash512, B> = MerkleTree::from_iter(values.clone());
b.iter(|| {
for i in 0..values.len() {
let proof = tree.gen_proof(i);
test::black_box(proof);
}
});
}
#[bench]
fn bench_ringx_sha512_from_data_160_proof_check(b: &mut Bencher) {
let values = tree_160();
let tree: MerkleTree<Hash512, B> = MerkleTree::from_iter(values.clone());
let proofs = (0..values.len())
.map(|i| tree.gen_proof(i))
.collect::<Vec<_>>();
b.iter(|| {
for proof in &proofs {
test::black_box(proof.validate::<B>());
}
});
}

View File

@ -0,0 +1,26 @@
// Copyright 2016 Brian Smith.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#[inline(always)]
pub fn init_once() {
#[cfg(not(target_os = "ios"))]
{
use std;
extern "C" {
fn GFp_cpuid_setup();
}
static INIT: std::sync::Once = std::sync::ONCE_INIT;
INIT.call_once(|| unsafe { GFp_cpuid_setup() });
}
}

View File

@ -0,0 +1,567 @@
//! Improvement version of [`ring::digest::Context`].
//!
//! [`Context.finish`] now has `(&mut self)` instead of `(mut self)`.
//! [`Context`] acquired state `reset` thing.
#![cfg(feature = "crypto_bench")]
#![allow(dead_code)]
extern crate rand;
extern crate ring;
extern crate test;
mod init;
use std::fmt;
// XXX: Replace with `const fn` when `const fn` is stable:
// https://github.com/rust-lang/rust/issues/24111
#[cfg(target_endian = "little")]
macro_rules! u32x2 {
( $first:expr, $second:expr ) => {
((($second as u64) << 32) | ($first as u64))
};
}
/// A context for multi-step (Init-Update-Finish) digest calculations.
///
/// C analog: `EVP_MD_CTX`.
pub struct Context {
state: State,
// Note that SHA-512 has a 128-bit input bit counter, but this
// implementation only supports up to 2^64-1 input bits for all algorithms,
// so a 64-bit counter is more than sufficient.
completed_data_blocks: u64,
// TODO: More explicitly force 64-bit alignment for |pending|.
pending: [u8; MAX_BLOCK_LEN],
num_pending: usize,
/// The context's algorithm.
pub algorithm: &'static Algorithm,
}
impl Context {
/// Constructs a new context.
///
/// C analogs: `EVP_DigestInit`, `EVP_DigestInit_ex`
pub fn new(algorithm: &'static Algorithm) -> Context {
init::init_once();
Context {
algorithm,
state: algorithm.initial_state,
completed_data_blocks: 0,
pending: [0u8; MAX_BLOCK_LEN],
num_pending: 0,
}
}
/// Updates the digest with all the data in `data`. `update` may be called
/// zero or more times until `finish` is called. It must not be called
/// after `finish` has been called.
///
/// C analog: `EVP_DigestUpdate`
pub fn update(&mut self, data: &[u8]) {
if data.len() < self.algorithm.block_len - self.num_pending {
self.pending[self.num_pending..(self.num_pending + data.len())].copy_from_slice(data);
self.num_pending += data.len();
return;
}
let mut remaining = data;
if self.num_pending > 0 {
let to_copy = self.algorithm.block_len - self.num_pending;
self.pending[self.num_pending..self.algorithm.block_len]
.copy_from_slice(&data[..to_copy]);
unsafe {
(self.algorithm.block_data_order)(&mut self.state, self.pending.as_ptr(), 1);
}
self.completed_data_blocks = self.completed_data_blocks.checked_add(1).unwrap();
remaining = &remaining[to_copy..];
self.num_pending = 0;
}
let num_blocks = remaining.len() / self.algorithm.block_len;
let num_to_save_for_later = remaining.len() % self.algorithm.block_len;
if num_blocks > 0 {
unsafe {
(self.algorithm.block_data_order)(&mut self.state, remaining.as_ptr(), num_blocks);
}
self.completed_data_blocks = self
.completed_data_blocks
.checked_add(polyfill::slice::u64_from_usize(num_blocks))
.unwrap();
}
if num_to_save_for_later > 0 {
self.pending[..num_to_save_for_later]
.copy_from_slice(&remaining[(remaining.len() - num_to_save_for_later)..]);
self.num_pending = num_to_save_for_later;
}
}
/// Finalizes the digest calculation and returns the digest value. `finish`
/// consumes the context so it cannot be (mis-)used after `finish` has been
/// called.
///
/// C analogs: `EVP_DigestFinal`, `EVP_DigestFinal_ex`
pub fn finish(&mut self) -> Digest {
// We know |num_pending < self.algorithm.block_len|, because we would
// have processed the block otherwise.
let mut padding_pos = self.num_pending;
self.pending[padding_pos] = 0x80;
padding_pos += 1;
if padding_pos > self.algorithm.block_len - self.algorithm.len_len {
polyfill::slice::fill(&mut self.pending[padding_pos..self.algorithm.block_len], 0);
unsafe {
(self.algorithm.block_data_order)(&mut self.state, self.pending.as_ptr(), 1);
}
// We don't increase |self.completed_data_blocks| because the
// padding isn't data, and so it isn't included in the data length.
padding_pos = 0;
}
polyfill::slice::fill(
&mut self.pending[padding_pos..(self.algorithm.block_len - 8)],
0,
);
// Output the length, in bits, in big endian order.
let mut completed_data_bits: u64 = self
.completed_data_blocks
.checked_mul(polyfill::slice::u64_from_usize(self.algorithm.block_len))
.unwrap()
.checked_add(polyfill::slice::u64_from_usize(self.num_pending))
.unwrap()
.checked_mul(8)
.unwrap();
for b in (&mut self.pending[(self.algorithm.block_len - 8)..self.algorithm.block_len])
.into_iter()
.rev()
{
*b = completed_data_bits as u8;
completed_data_bits /= 0x100;
}
unsafe {
(self.algorithm.block_data_order)(&mut self.state, self.pending.as_ptr(), 1);
}
Digest {
algorithm: self.algorithm,
value: (self.algorithm.format_output)(&self.state),
}
}
/// The algorithm that this context is using.
#[inline(always)]
pub fn algorithm(&self) -> &'static Algorithm {
self.algorithm
}
/// Reset context state.
pub fn reset(&mut self) {
self.state = self.algorithm.initial_state;
self.pending = [0u8; MAX_BLOCK_LEN];
self.completed_data_blocks = 0;
self.num_pending = 0;
}
}
// XXX: This should just be `#[derive(Clone)]` but that doesn't work because
// `[u8; 128]` doesn't implement `Clone`.
impl Clone for Context {
fn clone(&self) -> Context {
Context {
state: self.state,
pending: self.pending,
completed_data_blocks: self.completed_data_blocks,
num_pending: self.num_pending,
algorithm: self.algorithm,
}
}
}
/// Returns the digest of `data` using the given digest algorithm.
///
/// C analog: `EVP_Digest`
///
/// # Examples:
///
/// ```
/// # #[cfg(feature = "use_heap")]
/// # fn main() {
/// use ring::{digest, test};
///
/// let expected_hex =
/// "09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b";
/// let expected: Vec<u8> = test::from_hex(expected_hex).unwrap();
/// let actual = digest::digest(&digest::SHA256, b"hello, world");
///
/// assert_eq!(&expected, &actual.as_ref());
/// # }
///
/// # #[cfg(not(feature = "use_heap"))]
/// # fn main() { }
/// ```
pub fn digest(algorithm: &'static Algorithm, data: &[u8]) -> Digest {
let mut ctx = Context::new(algorithm);
ctx.update(data);
ctx.finish()
}
/// A calculated digest value.
///
/// Use `as_ref` to get the value as a `&[u8]`.
#[derive(Clone, Copy)]
pub struct Digest {
value: Output,
algorithm: &'static Algorithm,
}
impl Digest {
/// The algorithm that was used to calculate the digest value.
#[inline(always)]
pub fn algorithm(&self) -> &'static Algorithm {
self.algorithm
}
}
impl AsRef<[u8]> for Digest {
#[inline(always)]
fn as_ref(&self) -> &[u8] {
&(polyfill::slice::u64_as_u8(&self.value))[..self.algorithm.output_len]
}
}
impl fmt::Debug for Digest {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{:?}:", self.algorithm)?;
for byte in self.as_ref() {
write!(fmt, "{:02x}", byte)?;
}
Ok(())
}
}
/// A digest algorithm.
///
/// C analog: `EVP_MD`
pub struct Algorithm {
/// C analog: `EVP_MD_size`
pub output_len: usize,
/// The size of the chaining value of the digest function, in bytes. For
/// non-truncated algorithms (SHA-1, SHA-256, SHA-512), this is equal to
/// `output_len`. For truncated algorithms (e.g. SHA-384, SHA-512/256),
/// this is equal to the length before truncation. This is mostly helpful
/// for determining the size of an HMAC key that is appropriate for the
/// digest algorithm.
pub chaining_len: usize,
/// C analog: `EVP_MD_block_size`
pub block_len: usize,
/// The length of the length in the padding.
pub len_len: usize,
pub block_data_order: unsafe extern "C" fn(state: &mut State, data: *const u8, num: usize),
pub format_output: fn(input: &State) -> Output,
pub initial_state: State,
pub id: AlgorithmID,
}
#[derive(Eq, PartialEq)]
#[allow(non_camel_case_types)]
pub enum AlgorithmID {
SHA256,
SHA512,
SHA512_256,
}
impl PartialEq for Algorithm {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for Algorithm {}
impl fmt::Debug for Algorithm {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
// This would have to change if/when we add other algorithms with the
// same lengths.
let (n, suffix) =
if self.output_len == SHA512_256_OUTPUT_LEN && self.block_len == SHA512_BLOCK_LEN {
(512, "_256")
} else if self.output_len == 20 {
(1, "")
} else {
(self.output_len * 8, "")
};
write!(fmt, "SHA{}{}", n, suffix)
}
}
/// SHA-256 as specified in [FIPS 180-4].
///
/// [FIPS 180-4]: http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf
pub static SHA256: Algorithm = Algorithm {
output_len: SHA256_OUTPUT_LEN,
chaining_len: SHA256_OUTPUT_LEN,
block_len: 512 / 8,
len_len: 64 / 8,
block_data_order: GFp_sha256_block_data_order,
format_output: sha256_format_output,
initial_state: [
u32x2!(0x6a09e667u32, 0xbb67ae85u32),
u32x2!(0x3c6ef372u32, 0xa54ff53au32),
u32x2!(0x510e527fu32, 0x9b05688cu32),
u32x2!(0x1f83d9abu32, 0x5be0cd19u32),
0,
0,
0,
0,
],
id: AlgorithmID::SHA256,
};
/// SHA-512 as specified in [FIPS 180-4].
///
/// [FIPS 180-4]: http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf
pub static SHA512: Algorithm = Algorithm {
output_len: SHA512_OUTPUT_LEN,
chaining_len: SHA512_OUTPUT_LEN,
block_len: SHA512_BLOCK_LEN,
len_len: SHA512_LEN_LEN,
block_data_order: GFp_sha512_block_data_order,
format_output: sha512_format_output,
initial_state: [
0x6a09e667f3bcc908,
0xbb67ae8584caa73b,
0x3c6ef372fe94f82b,
0xa54ff53a5f1d36f1,
0x510e527fade682d1,
0x9b05688c2b3e6c1f,
0x1f83d9abfb41bd6b,
0x5be0cd19137e2179,
],
id: AlgorithmID::SHA512,
};
/// SHA-512/256 as specified in [FIPS 180-4].
///
/// This is *not* the same as just truncating the output of SHA-512, as
/// SHA-512/256 has its own initial state distinct from SHA-512's initial
/// state.
///
/// [FIPS 180-4]: http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf
pub static SHA512_256: Algorithm = Algorithm {
output_len: SHA512_256_OUTPUT_LEN,
chaining_len: SHA512_OUTPUT_LEN,
block_len: SHA512_BLOCK_LEN,
len_len: SHA512_LEN_LEN,
block_data_order: GFp_sha512_block_data_order,
format_output: sha512_format_output,
initial_state: [
0x22312194fc2bf72c,
0x9f555fa3c84c64c2,
0x2393b86b6f53b151,
0x963877195940eabd,
0x96283ee2a88effe3,
0xbe5e1e2553863992,
0x2b0199fc2c85b8aa,
0x0eb72ddc81c52ca2,
],
id: AlgorithmID::SHA512_256,
};
// We use u64 to try to ensure 64-bit alignment/padding.
pub type State = [u64; MAX_CHAINING_LEN / 8];
pub type Output = [u64; MAX_OUTPUT_LEN / 8];
/// The maximum block length (`Algorithm::block_len`) of all the algorithms in
/// this module.
pub const MAX_BLOCK_LEN: usize = 1024 / 8;
/// The maximum output length (`Algorithm::output_len`) of all the algorithms
/// in this module.
pub const MAX_OUTPUT_LEN: usize = 512 / 8;
/// The maximum chaining length (`Algorithm::chaining_len`) of all the
/// algorithms in this module.
pub const MAX_CHAINING_LEN: usize = MAX_OUTPUT_LEN;
mod polyfill {
pub mod slice {
use std::slice::from_raw_parts;
// https://internals.rust-lang.org/t/
// safe-trasnsmute-for-slices-e-g-u64-u32-particularly-simd-types/2871
#[inline(always)]
pub fn u64_as_u32(src: &[u64]) -> &[u32] {
unsafe { from_raw_parts(src.as_ptr() as *const u32, src.len() * 2) }
}
#[inline(always)]
pub fn u64_from_usize(x: usize) -> u64 {
x as u64
}
// https://internals.rust-lang.org/t/
// stabilizing-basic-functions-on-arrays-and-slices/2868
#[inline(always)]
pub fn fill(dest: &mut [u8], value: u8) {
for d in dest {
*d = value;
}
}
// https://internals.rust-lang.org/t/
// safe-trasnsmute-for-slices-e-g-u64-u32-particularly-simd-types/2871
#[inline(always)]
pub fn u64_as_u8(src: &[u64]) -> &[u8] {
unsafe { from_raw_parts(src.as_ptr() as *const u8, src.len() * 8) }
}
}
}
pub fn sha256_format_output(input: &State) -> Output {
let input = &polyfill::slice::u64_as_u32(input)[..8];
[
u32x2!(input[0].to_be(), input[1].to_be()),
u32x2!(input[2].to_be(), input[3].to_be()),
u32x2!(input[4].to_be(), input[5].to_be()),
u32x2!(input[6].to_be(), input[7].to_be()),
0,
0,
0,
0,
]
}
pub fn sha512_format_output(input: &State) -> Output {
[
input[0].to_be(),
input[1].to_be(),
input[2].to_be(),
input[3].to_be(),
input[4].to_be(),
input[5].to_be(),
input[6].to_be(),
input[7].to_be(),
]
}
/// The length of the output of SHA-256, in bytes.
pub const SHA256_OUTPUT_LEN: usize = 256 / 8;
/// The length of the output of SHA-512, in bytes.
pub const SHA512_OUTPUT_LEN: usize = 512 / 8;
/// The length of the output of SHA-512/256, in bytes.
pub const SHA512_256_OUTPUT_LEN: usize = 256 / 8;
/// The length of a block for SHA-512-based algorithms, in bytes.
const SHA512_BLOCK_LEN: usize = 1024 / 8;
/// The length of the length field for SHA-512-based algorithms, in bytes.
const SHA512_LEN_LEN: usize = 128 / 8;
extern "C" {
fn GFp_sha256_block_data_order(state: &mut State, data: *const u8, num: usize);
fn GFp_sha512_block_data_order(state: &mut State, data: *const u8, num: usize);
}
#[cfg(test)]
pub mod test_util {
use super::*;
pub static ALL_ALGORITHMS: [&'static Algorithm; 3] = [&SHA256, &SHA512, &SHA512_256];
}
/*
#[cfg(test)]
mod tests {
mod max_input {
use super::super::super::digest;
macro_rules! max_input_tests {
( $algorithm_name:ident ) => {
#[allow(non_snake_case)]
mod $algorithm_name {
use super::super::super::super::digest;
#[test]
fn max_input_test() {
super::max_input_test(&digest::$algorithm_name);
}
#[test]
#[should_panic]
fn too_long_input_test_block() {
super::too_long_input_test_block(
&digest::$algorithm_name);
}
#[test]
#[should_panic]
fn too_long_input_test_byte() {
super::too_long_input_test_byte(
&digest::$algorithm_name);
}
}
}
}
fn max_input_test(alg: &'static digest::Algorithm) {
let mut context = nearly_full_context(alg);
let next_input = vec![0u8; alg.block_len - 1];
context.update(&next_input);
let _ = context.finish(); // no panic
}
fn too_long_input_test_block(alg: &'static digest::Algorithm) {
let mut context = nearly_full_context(alg);
let next_input = vec![0u8; alg.block_len];
context.update(&next_input);
let _ = context.finish(); // should panic
}
fn too_long_input_test_byte(alg: &'static digest::Algorithm) {
let mut context = nearly_full_context(alg);
let next_input = vec![0u8; alg.block_len - 1];
context.update(&next_input); // no panic
context.update(&[0]);
let _ = context.finish(); // should panic
}
fn nearly_full_context(alg: &'static digest::Algorithm)
-> digest::Context {
// All implementations currently support up to 2^64-1 bits
// of input; according to the spec, SHA-384 and SHA-512
// support up to 2^128-1, but that's not implemented yet.
let max_bytes = 1u64 << (64 - 3);
let max_blocks = max_bytes / (alg.block_len as u64);
digest::Context {
algorithm: alg,
state: alg.initial_state,
completed_data_blocks: max_blocks - 1,
pending: [0u8; digest::MAX_BLOCK_LEN],
num_pending: 0,
}
}
max_input_tests!(SHA256);
max_input_tests!(SHA512);
}
}*/

View File

@ -0,0 +1,3 @@
disallowed-names = [
"unreadable_literal"
]

View File

@ -0,0 +1,4 @@
// This is just for no_std testing
#[allow(unused_imports)]
fn main() {}

View File

@ -0,0 +1,135 @@
//! Hash infrastructure for items in Merkle Tree.
use core::hash::Hasher;
/// A hashable type.
///
/// Types implementing `Hashable` are able to be [`hash`]ed with an instance of
/// [`Hasher`].
///
/// ## Implementing `Hashable`
///
/// You can derive `Hashable` with `#[derive(Hashable)]` if all fields implement `Hashable`.
/// The resulting hash will be the combination of the values from calling
/// [`hash`] on each field.
///
/// ```text
/// #[macro_use]
/// extern crate merkle_light_derive;
/// extern crate merkle_light;
///
/// use merkle_light::hash::Hashable;
///
/// fn main() {
/// #[derive(Hashable)]
/// struct Foo {
/// name: String,
/// country: String,
/// }
/// }
/// ```
///
/// If you need more control over how a value is hashed, you can of course
/// implement the `Hashable` trait yourself:
///
/// ```
/// extern crate merkle_light;
///
/// use merkle_light::hash::Hashable;
/// use std::hash::Hasher;
/// use std::collections::hash_map::DefaultHasher;
///
/// fn main() {
/// struct Person {
/// id: u32,
/// name: String,
/// phone: u64,
/// }
///
/// impl<H: Hasher> Hashable<H> for Person {
/// fn hash(&self, state: &mut H) {
/// self.id.hash(state);
/// self.name.hash(state);
/// self.phone.hash(state);
/// }
/// }
///
/// let foo = Person{
/// id: 1,
/// name: String::from("blah"),
/// phone: 2,
/// };
///
/// let mut hr = DefaultHasher::new();
/// foo.hash(&mut hr);
/// assert_eq!(hr.finish(), 7101638158313343130)
/// }
/// ```
///
/// ## `Hashable` and `Eq`
///
/// When implementing both `Hashable` and [`Eq`], it is important that the following
/// property holds:
///
/// ```text
/// k1 == k2 -> hash(k1) == hash(k2)
/// ```
///
/// In other words, if two keys are equal, their hashes must also be equal.
pub trait Hashable<H: Hasher> {
/// Feeds this value into the given [`Hasher`].
///
/// [`Hasher`]: trait.Hasher.html
fn hash(&self, state: &mut H);
/// Feeds a slice of this type into the given [`Hasher`].
///
/// [`Hasher`]: trait.Hasher.html
fn hash_slice(data: &[Self], state: &mut H)
where
Self: Sized,
{
for piece in data {
piece.hash(state);
}
}
}
/// A trait for hashing an arbitrary stream of bytes for calculating merkle tree
/// nodes.
///
/// T is a hash item must be of known size at compile time, globally ordered, with
/// default value as a neutral element of the hash space. Neutral element is
/// interpreted as 0 or nil and required for evaluation of merkle tree.
///
/// [`Algorithm`] breaks the [`Hasher`] contract at `finish()`, but that is intended.
/// This trait extends [`Hasher`] with `hash -> T` and `reset` state methods,
/// plus implements default behavior of evaluation of MT interior nodes.
pub trait Algorithm<T>: Hasher + Default
where
T: Clone + AsRef<[u8]>,
{
/// Returns the hash value for the data stream written so far.
fn hash(&mut self) -> T;
/// Reset Hasher state.
#[inline]
fn reset(&mut self) {
*self = Self::default();
}
/// Returns hash value for MT leaf (prefix 0x00).
#[inline]
fn leaf(&mut self, leaf: T) -> T {
self.write(leaf.as_ref());
self.hash()
}
/// Returns hash value for MT interior node (prefix 0x01).
#[inline]
fn node(&mut self, left: T, right: T) -> T {
self.write(left.as_ref());
self.write(right.as_ref());
self.hash()
}
}

View File

@ -0,0 +1,188 @@
extern crate alloc;
use crate::hash::Hashable;
use alloc::string::String;
use alloc::vec::Vec;
use core::hash::Hasher;
use core::mem;
use core::slice;
macro_rules! impl_write {
($(($ty:ident, $meth:ident),)*) => {$(
impl<H: Hasher> Hashable<H> for $ty {
fn hash(&self, state: &mut H) {
state.$meth(*self)
}
#[allow(trivial_casts, unsafe_code)]
fn hash_slice(data: &[$ty], state: &mut H) {
let newlen = data.len() * mem::size_of::<$ty>();
let ptr = data.as_ptr() as *const u8;
state.write(unsafe { slice::from_raw_parts(ptr, newlen) })
}
}
)*}
}
impl_write! {
(u8, write_u8),
(u16, write_u16),
(u32, write_u32),
(u64, write_u64),
(usize, write_usize),
(i8, write_i8),
(i16, write_i16),
(i32, write_i32),
(i64, write_i64),
(isize, write_isize),
// unstable: (u128, write_u128),
// unstable: (i128, write_i128),
}
macro_rules! impl_array {
($ty:ident $($N:expr)+) => {$(
impl<H: Hasher> Hashable<H> for [$ty; $N] {
fn hash(&self, state: &mut H) {
state.write(self.as_ref())
}
#[allow(trivial_casts, unsafe_code)]
fn hash_slice(data: &[[$ty; $N]], state: &mut H) {
let newlen = data.len() * mem::size_of::<[$ty; $N]>();
let ptr = data.as_ptr() as *const u8;
state.write(unsafe { slice::from_raw_parts(ptr, newlen) })
}
}
)*}
}
impl_array! { u8
1 2 3 4 5 6 7 8 9 10
11 12 13 14 15 16 17 18 19 20
21 22 23 24 25 26 27 28 29 30
31 32 33 34 35 36 37 38 39 40
41 42 43 44 45 46 47 48 49 50
51 52 53 54 55 56 57 58 59 60
61 62 63 64
}
impl<H: Hasher> Hashable<H> for Vec<u8> {
fn hash(&self, state: &mut H) {
state.write(self.as_ref());
}
}
impl<H: Hasher> Hashable<H> for bool {
fn hash(&self, state: &mut H) {
state.write_u8(*self as u8)
}
}
impl<H: Hasher> Hashable<H> for char {
fn hash(&self, state: &mut H) {
state.write_u32(*self as u32)
}
}
impl<H: Hasher> Hashable<H> for str {
fn hash(&self, state: &mut H) {
state.write(self.as_bytes());
// empty str nope: state.write_u8(0xff)
}
}
impl<H: Hasher> Hashable<H> for String {
fn hash(&self, state: &mut H) {
state.write(self.as_bytes());
// empty str nope: state.write_u8(0xff)
}
}
macro_rules! impl_hash_tuple {
() => (
impl<H: Hasher> Hashable<H> for () {
fn hash(&self, _: &mut H) {}
}
);
( $($name:ident)+) => (
impl<Z: Hasher, $($name: Hashable<Z>),*> Hashable<Z> for ($($name,)*)
where
last_type!($($name,)+): ?Sized
{
#[allow(non_snake_case)]
fn hash(&self, state: &mut Z) {
let ($(ref $name,)*) = *self;
$($name.hash(state);)*
}
}
);
}
macro_rules! last_type {
($a:ident,) => { $a };
($a:ident, $($rest_a:ident,)+) => { last_type!($($rest_a,)+) };
}
impl_hash_tuple! {}
impl_hash_tuple! { A }
impl_hash_tuple! { A B }
impl_hash_tuple! { A B C }
impl_hash_tuple! { A B C D }
impl_hash_tuple! { A B C D E }
impl_hash_tuple! { A B C D E F }
impl_hash_tuple! { A B C D E F G }
impl_hash_tuple! { A B C D E F G H }
impl_hash_tuple! { A B C D E F G H I }
impl_hash_tuple! { A B C D E F G H I J }
impl_hash_tuple! { A B C D E F G H I J K }
impl_hash_tuple! { A B C D E F G H I J K L }
impl<H: Hasher, T: Hashable<H>> Hashable<H> for [T] {
fn hash(&self, state: &mut H) {
self.len().hash(state);
Hashable::hash_slice(self, state)
}
}
impl<'a, H: Hasher, T: ?Sized + Hashable<H>> Hashable<H> for &'a T {
fn hash(&self, state: &mut H) {
(**self).hash(state);
}
}
impl<'a, H: Hasher, T: ?Sized + Hashable<H>> Hashable<H> for &'a mut T {
fn hash(&self, state: &mut H) {
(**self).hash(state);
}
}
impl<H: Hasher, T: ?Sized> Hashable<H> for *const T {
#[allow(trivial_casts, unsafe_code)]
fn hash(&self, state: &mut H) {
if mem::size_of::<Self>() == mem::size_of::<usize>() {
// Thin pointer
state.write_usize(*self as *const () as usize);
} else {
// Fat pointer
let (a, b) = unsafe { *(self as *const Self as *const (usize, usize)) };
state.write_usize(a);
state.write_usize(b);
}
}
}
impl<H: Hasher, T: ?Sized> Hashable<H> for *mut T {
#[allow(trivial_casts, unsafe_code)]
fn hash(&self, state: &mut H) {
if mem::size_of::<Self>() == mem::size_of::<usize>() {
// Thin pointer
state.write_usize(*self as *const () as usize);
} else {
// Fat pointer
let (a, b) = unsafe { *(self as *const Self as *const (usize, usize)) };
state.write_usize(a);
state.write_usize(b);
}
}
}

View File

@ -0,0 +1,178 @@
//! light _Merkle Tree_ implementation.
//!
//! Merkle tree (MT) implemented as a full binary tree allocated as a vec
//! of statically sized hashes to give hashes more locality. MT specialized
//! to the extent of hashing algorithm and hash item. [`Hashable`] trait is
//! compatible to the `std::hash::Hasher` and supports custom hash algorithms.
//! Implementation does not depend on any external crypto libraries, and tries
//! to be as performant as possible.
//!
//! This tree implementation uses encoding scheme as in _Certificate Transparency_
//! by default. Encoding scheme for leafs and nodes can be overridden though.
//! [RFC 6962](https://tools.ietf.org/html/rfc6962):
//!
//! ```text
//! MTH({d(0)}) = ALG(0x00 || d(0)).
//! For n > 1, let k be the largest power of two smaller than n (i.e.,
//! k < n <= 2k). The Merkle tree Hash of an n-element list D[n] is then
//! defined recursively as
//! MTH(D[n]) = ALG(0x01 || MTH(D[0:k]) || MTH(D[k:n])),
//! ```
//!
//! Link: [](https://en.wikipedia.org/wiki/Merkle_tree)
//!
//! # Implementation choices
//!
//! Main idea is the whole code must obtain specialization at compile time with
//! minimum allocations calls, hashes must be of fixed size arrays known at
//! compile time, hash algorithm must be a trait and must not depend on any
//! external cryptographic libraries and the lib itself must somehow mimic std Rust api.
//!
//! Standard way in Rust is to hash objects with a `std::hash::Hasher`, and mainly
//! that is the reason behind the choice of the abstractions:
//!
//! `Object : Hashable<H> -> Hasher + Algorithm <- Merkle Tree`
//!
//! Custom [`merkle::hash::Hashable`] trait allows implementations differ
//! from [`std::collection`] related hashes, different implementations for
//! different hashing algorithms / schemas and conforms object-safety trait rules.
//!
//! [`Algorithm`] complements [`Hasher`] to be reusable and follows the idea
//! that the result hash is a mapping of the data stream.
//!
//! [`Algorithm.hash`] had to change its signature to be `&mut self` (`&self`) because
//! most of the cryptographic digest algorithms breaks current state on finalization
//! into unusable. `ring` libra tho contains interfaces incompatible to
//! `start-update-finish-reset` lifecycle. It requires either `cloning()` its state
//! on finalization, or `Cell`-ing via unsafe.
//!
//! Turning back to having [`Algorithm.write(&mut self, &[u8])`] instead of
//! `write(T)` allows to relax [`Algorithm`] trait [`Hasher`] constraint, even tho
//! works together well still.
//!
//! # Interface
//!
//! ```text
//! - build_tree (items) -> tree
//! - get_root -> hash
//! - gen_proof -> proof
//! - validate_proof (proof, leaf, root) -> bool
//! ```
//!
//! # Examples
//!
//! [`test_cmh.rs`]: custom merkle hasher implementation example
//! [`crypto_bitcoin_mt.rs`]: bitcoin merkle tree using crypto lib
//! [`crypto_chaincore_mt.rs`]: chain core merkle tree using crypto lib
//! [`ring_bitcoin_mt.rs`]: bitcoin merkle tree using ring lib
//!
//! # Quick start
//!
//! ```
//! #[cfg(feature = "chaincore")]
//! extern crate crypto;
//! extern crate merkle_light;
//!
//! #[cfg(feature = "chaincore")]
//! mod example {
//! use std::fmt;
//! use std::hash::Hasher;
//! use std::iter::FromIterator;
//! use crypto::sha3::{Sha3, Sha3Mode};
//! use crypto::digest::Digest;
//! use merkle_light::hash::{Algorithm, Hashable};
//!
//! pub struct ExampleAlgorithm(Sha3);
//!
//! impl ExampleAlgorithm {
//! pub fn new() -> ExampleAlgorithm {
//! ExampleAlgorithm(Sha3::new(Sha3Mode::Sha3_256))
//! }
//! }
//!
//! impl Default for ExampleAlgorithm {
//! fn default() -> ExampleAlgorithm {
//! ExampleAlgorithm::new()
//! }
//! }
//!
//! impl Hasher for ExampleAlgorithm {
//! #[inline]
//! fn write(&mut self, msg: &[u8]) {
//! self.0.input(msg)
//! }
//!
//! #[inline]
//! fn finish(&self) -> u64 {
//! unimplemented!()
//! }
//! }
//!
//! impl Algorithm<[u8; 32]> for ExampleAlgorithm {
//! #[inline]
//! fn hash(&mut self) -> [u8; 32] {
//! let mut h = [0u8; 32];
//! self.0.result(&mut h);
//! h
//! }
//!
//! #[inline]
//! fn reset(&mut self) {
//! self.0.reset();
//! }
//! }
//! }
//!
//! fn main() {
//! #[cfg(feature = "chaincore")]
//! {
//! use example::ExampleAlgorithm;
//! use merkle_light::merkle::MerkleTree;
//! use std::iter::FromIterator;
//!
//! let mut h1 = [0u8; 32];
//! let mut h2 = [0u8; 32];
//! let mut h3 = [0u8; 32];
//! h1[0] = 0x11;
//! h2[0] = 0x22;
//! h3[0] = 0x33;
//!
//! let t: MerkleTree<[u8; 32], ExampleAlgorithm> = MerkleTree::from_iter(vec![h1, h2, h3]);
//! println!("{:?}", t.root());
//! }
//! }
//! ```
#![deny(
missing_docs,
unused_qualifications,
missing_debug_implementations,
missing_copy_implementations,
trivial_casts,
trivial_numeric_casts,
unsafe_code,
unstable_features,
unused_import_braces
)]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(feature = "nightly", allow(unstable_features))]
/// Hash infrastructure for items in Merkle tree.
pub mod hash;
/// Common implementations for [`Hashable`].
mod hash_impl;
/// Merkle tree inclusion proof
pub mod proof;
/// Merkle tree abstractions, implementation and algorithms.
pub mod merkle;
/// Tests data.
#[cfg(test)]
mod test_item;
/// Tests for Merkle Hasher Customization
#[cfg(test)]
mod test_cmh;

View File

@ -0,0 +1,305 @@
extern crate alloc;
use crate::hash::{Algorithm, Hashable};
use crate::proof::Proof;
use alloc::vec::Vec;
use core::iter::FromIterator;
use core::marker::PhantomData;
use core::ops;
use rayon::prelude::*;
use std::collections::BTreeMap;
use std::fmt::Debug;
/// Merkle Tree.
///
/// All leafs and nodes are stored in a linear array (vec).
///
/// A merkle tree is a tree in which every non-leaf node is the hash of its
/// children nodes. A diagram depicting how it works:
///
/// ```text
/// root = h1234 = h(h12 + h34)
/// / \
/// h12 = h(h1 + h2) h34 = h(h3 + h4)
/// / \ / \
/// h1 = h(tx1) h2 = h(tx2) h3 = h(tx3) h4 = h(tx4)
/// ```
///
/// In memory layout:
///
/// ```text
/// [h1 h2 h3 h4 h12 h34 root]
/// ```
///
/// Merkle root is always the last element in the array.
///
/// The number of inputs is not always a power of two which results in a
/// balanced tree structure as above. In that case, parent nodes with no
/// children are also zero and parent nodes with only a single left node
/// are calculated by concatenating the left node with itself before hashing.
/// Since this function uses nodes that are pointers to the hashes, empty nodes
/// will be nil.
///
/// TODO: Ord
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct MerkleTree<T: Ord + Clone + AsRef<[u8]> + Sync + Send, A: Algorithm<T>> {
data: Vec<T>,
leafs: usize,
height: usize,
link_map: BTreeMap<usize, usize>,
_a: PhantomData<A>,
}
impl<T: Ord + Clone + Debug + Default + AsRef<[u8]> + Sync + Send, A: Algorithm<T>>
MerkleTree<T, A>
{
/// Creates new merkle from a sequence of hashes.
pub fn new<I: IntoIterator<Item = T>>(data: I) -> MerkleTree<T, A> {
Self::from_iter(data)
}
/// Creates new merkle tree from a list of hashable objects.
pub fn from_data<O: Hashable<A>, I: IntoIterator<Item = O>>(data: I) -> MerkleTree<T, A> {
let mut a = A::default();
Self::from_iter(data.into_iter().map(|x| {
a.reset();
x.hash(&mut a);
a.hash()
}))
}
fn build(&mut self) {
let mut width = self.leafs;
// build tree
let mut layer_start: usize = 0;
let mut layer_end: usize = width;
while width > 1 {
// if there is odd num of elements, fill in a NULL.
if width & 1 == 1 {
self.data.push(Self::null_node());
width += 1;
layer_end += 1;
}
let layer: Vec<_> = (layer_start..layer_end)
.into_par_iter()
.step_by(2)
.map(|i| {
let mut a = A::default();
// If the right child is not NULL, the left child is ensured to be not NULL.
let mut link_map_update = None;
let h = if self.data[i + 1] != Self::null_node() {
a.node(self.data[i].clone(), self.data[i + 1].clone())
} else {
// If a child is NULL, the parent should be a linking node to the actual node hash.
let parent_index = (i - layer_start) / 2 + layer_end;
if self.data[i] == Self::null_node() {
// If both are NULL, the left child must be a linking node.
let linked_to = *self.link_map.get(&i).unwrap();
link_map_update = Some((parent_index, linked_to, Some(i)));
Self::null_node()
} else {
match self.link_map.get(&(i + 1)) {
// Right child is linked to a hash, so we just compute the parent hash.
Some(index) => {
assert_ne!(self.data[*index], Self::null_node());
a.node(self.data[i].clone(), self.data[*index].clone())
}
// Right child is NULL, so link the parent to the left child which has a hash stored.
None => {
link_map_update = Some((parent_index, i, None));
Self::null_node()
}
}
}
};
(h, link_map_update)
})
.collect();
for (node, maybe_link_map_update) in layer {
self.data.push(node);
if let Some((from, to, maybe_remove)) = maybe_link_map_update {
self.link_map.insert(from, to);
if let Some(remove) = maybe_remove {
self.link_map.remove(&remove);
}
}
}
layer_start = layer_end;
width >>= 1;
layer_end += width;
}
}
/// Generate merkle tree inclusion proof for leaf `i`
pub fn gen_proof(&self, i: usize) -> Proof<T> {
if self.leafs == 1 {
assert_eq!(i, 0);
return Proof::new(vec![self.root()], vec![]);
}
assert!(i < self.leafs); // i in [0 .. self.leafs)
let mut lemma: Vec<T> = Vec::with_capacity(self.height + 1); // path + root
let mut path: Vec<bool> = Vec::with_capacity(self.height - 1); // path - 1
let mut base = 0;
let mut j = i;
// level 1 width
let mut width = self.leafs;
if width & 1 == 1 {
width += 1;
}
lemma.push(self.data[j].clone());
while base + 1 < self.len() {
let proof_hash_index = if j & 1 == 0 {
// j is left
let right_index = base + j + 1;
if self.data[right_index] == Self::null_node() {
match self.link_map.get(&right_index) {
// A link node, so the proof uses the linked hash.
Some(index) => {
assert_ne!(self.data[*index], Self::null_node());
Some(*index)
}
// A NULL node, just skip.
None => None,
}
} else {
Some(right_index)
}
} else {
// j is right
Some(base + j - 1)
};
if let Some(index) = proof_hash_index {
lemma.push(self.data[index].clone());
path.push(j & 1 == 0);
}
base += width;
width >>= 1;
if width & 1 == 1 {
width += 1;
}
j >>= 1;
}
// root is final
lemma.push(self.root());
Proof::new(lemma, path)
}
/// Returns merkle root
pub fn root(&self) -> T {
self.data[self.data.len() - 1].clone()
}
/// Returns number of elements in the tree.
pub fn len(&self) -> usize {
self.data.len()
}
/// Returns `true` if the vector contains no elements.
pub fn is_empty(&self) -> bool {
self.data.is_empty()
}
/// Returns height of the tree
pub fn height(&self) -> usize {
self.height
}
/// Returns original number of elements the tree was built upon.
pub fn leafs(&self) -> usize {
self.leafs
}
/// Extracts a slice containing the entire vector.
///
/// Equivalent to `&s[..]`.
pub fn as_slice(&self) -> &[T] {
self
}
fn null_node() -> T {
T::default()
}
}
impl<T: Ord + Clone + Debug + Default + AsRef<[u8]> + Sync + Send, A: Algorithm<T>> FromIterator<T>
for MerkleTree<T, A>
{
/// Creates new merkle tree from an iterator over hashable objects.
fn from_iter<I: IntoIterator<Item = T>>(into: I) -> Self {
let iter = into.into_iter();
let mut data: Vec<T> = match iter.size_hint().1 {
Some(e) => {
let pow = next_pow2(e);
let size = 2 * pow - 1;
Vec::with_capacity(size)
}
None => Vec::new(),
};
// leafs
let mut a = A::default();
for item in iter {
a.reset();
data.push(a.leaf(item));
}
let leafs = data.len();
let pow = next_pow2(leafs);
let size = 2 * pow - 1;
// assert!(leafs > 1);
let mut mt: MerkleTree<T, A> = MerkleTree {
data,
leafs,
height: log2_pow2(size + 1),
link_map: Default::default(),
_a: PhantomData,
};
mt.build();
mt
}
}
impl<T: Ord + Clone + AsRef<[u8]> + Sync + Send, A: Algorithm<T>> ops::Deref for MerkleTree<T, A> {
type Target = [T];
fn deref(&self) -> &[T] {
self.data.deref()
}
}
/// `next_pow2` returns next highest power of two from a given number if
/// it is not already a power of two.
///
/// [](http://locklessinc.com/articles/next_pow2/)
/// [](https://stackoverflow.com/questions/466204/rounding-up-to-next-power-of-2/466242#466242)
pub fn next_pow2(mut n: usize) -> usize {
n -= 1;
n |= n >> 1;
n |= n >> 2;
n |= n >> 4;
n |= n >> 8;
n |= n >> 16;
#[cfg(target_pointer_width = "64")]
{
n |= n >> 32;
}
n + 1
}
/// find power of 2 of a number which is power of 2
pub fn log2_pow2(n: usize) -> usize {
n.trailing_zeros() as usize
}

View File

@ -0,0 +1,80 @@
extern crate alloc;
use crate::hash::Algorithm;
use alloc::vec::Vec;
/// Merkle tree inclusion proof for data element, for which item = Leaf(Hash(Data Item)).
///
/// Lemma layout:
///
/// ```text
/// [ item h1x h2y h3z ... root ]
/// ```
///
/// Proof validation is positioned hash against lemma path to match root hash.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Proof<T: Eq + Clone + AsRef<[u8]>> {
lemma: Vec<T>,
path: Vec<bool>,
}
impl<T: Eq + Clone + AsRef<[u8]>> Proof<T> {
/// Creates new MT inclusion proof
pub fn new(hash: Vec<T>, path: Vec<bool>) -> Proof<T> {
if hash.len() > 2 {
assert_eq!(hash.len() - 2, path.len());
} else {
assert_eq!(hash.len(), 1);
assert_eq!(path, vec![]);
}
Proof { lemma: hash, path }
}
/// Return proof target leaf
pub fn item(&self) -> T {
self.lemma.first().unwrap().clone()
}
/// Return tree root
pub fn root(&self) -> T {
self.lemma.last().unwrap().clone()
}
/// Verifies MT inclusion proof
pub fn validate<A: Algorithm<T>>(&self) -> bool {
let size = self.lemma.len();
// Special case for a single node.
if size == 1 && self.path == vec![] {
return true;
}
if size < 2 {
return false;
}
let mut h = self.item();
let mut a = A::default();
for i in 1..size - 1 {
a.reset();
h = if self.path[i - 1] {
a.node(h, self.lemma[i].clone())
} else {
a.node(self.lemma[i].clone(), h)
};
}
h == self.root()
}
/// Returns the path of this proof.
pub fn path(&self) -> &[bool] {
&self.path
}
/// Returns the lemma of this proof.
pub fn lemma(&self) -> &[T] {
&self.lemma
}
}

View File

@ -0,0 +1,76 @@
#![cfg(test)]
use crate::hash::{Algorithm, Hashable};
use crate::merkle::MerkleTree;
use crate::test_item::Item;
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use std::iter::FromIterator;
/// Custom merkle hash util test
#[derive(Debug, Clone, Default)]
struct CMH(DefaultHasher);
impl CMH {
pub fn new() -> CMH {
CMH(DefaultHasher::new())
}
}
impl Hasher for CMH {
#[inline]
fn write(&mut self, msg: &[u8]) {
self.0.write(msg)
}
#[inline]
fn finish(&self) -> u64 {
self.0.finish()
}
}
impl Algorithm<Item> for CMH {
#[inline]
fn hash(&mut self) -> Item {
Item(self.finish())
}
#[inline]
fn reset(&mut self) {
*self = CMH::default()
}
#[inline]
fn leaf(&mut self, leaf: Item) -> Item {
Item(leaf.0 & 0xff)
}
#[inline]
fn node(&mut self, left: Item, right: Item) -> Item {
self.write(&[1u8]);
self.write(left.as_ref());
self.write(&[2u8]);
self.write(right.as_ref());
Item(self.hash().0 & 0xffff)
}
}
#[test]
fn test_custom_merkle_hasher() {
let mut a = CMH::new();
let mt: MerkleTree<Item, CMH> = MerkleTree::from_iter([1, 2, 3, 4, 5].iter().map(|x| {
a.reset();
x.hash(&mut a);
a.hash()
}));
assert_eq!(
mt.as_slice()
.iter()
.take(mt.leafs())
.filter(|&&x| x.0 > 255)
.count(),
0
);
assert_eq!(mt.as_slice().iter().filter(|&&x| x.0 > 65535).count(), 0);
}

View File

@ -0,0 +1,39 @@
#![cfg(test)]
#![allow(unsafe_code)]
use crate::hash::{Algorithm, Hashable};
use std::mem;
use std::slice;
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default, Debug)]
pub struct Item(pub u64);
impl AsRef<[u8]> for Item {
fn as_ref(&self) -> &[u8] {
unsafe { slice::from_raw_parts(mem::transmute(&self.0), 8) }
}
}
impl PartialEq<u64> for Item {
fn eq(&self, other: &u64) -> bool {
self.0 == *other
}
}
impl From<u64> for Item {
fn from(x: u64) -> Self {
Item(x)
}
}
impl From<Item> for u64 {
fn from(val: Item) -> Self {
val.0
}
}
impl<A: Algorithm<Item>> Hashable<A> for Item {
fn hash(&self, state: &mut A) {
state.write_u64(self.0)
}
}

View File

@ -0,0 +1,13 @@
[package]
name = "merkle_tree"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
merkle_light = { path = "../../common/merkle_light" }
tiny-keccak = { version = "2.0.2", features = ["keccak"] }
[dev-dependencies]
hex = "0.4.3"

View File

@ -0,0 +1,88 @@
use merkle_light::hash::Algorithm;
use std::hash::Hasher;
use tiny_keccak::{Hasher as KeccakHasher, Keccak};
// TODO: Option here is only used for compatibility with `tiny_keccak` and `merkle_light`.
#[derive(Clone)]
pub struct RawLeafSha3Algorithm(Option<Keccak>);
impl RawLeafSha3Algorithm {
fn new() -> RawLeafSha3Algorithm {
RawLeafSha3Algorithm(Some(Keccak::v256()))
}
}
impl Default for RawLeafSha3Algorithm {
fn default() -> RawLeafSha3Algorithm {
RawLeafSha3Algorithm::new()
}
}
impl Hasher for RawLeafSha3Algorithm {
#[inline]
fn write(&mut self, msg: &[u8]) {
self.0.as_mut().unwrap().update(msg)
}
#[inline]
fn finish(&self) -> u64 {
unimplemented!()
}
}
pub type CryptoSHA256Hash = [u8; 32];
impl Algorithm<CryptoSHA256Hash> for RawLeafSha3Algorithm {
#[inline]
fn hash(&mut self) -> CryptoSHA256Hash {
let mut h = [0u8; 32];
self.0.take().unwrap().finalize(&mut h);
h
}
fn leaf(&mut self, leaf: CryptoSHA256Hash) -> CryptoSHA256Hash {
// Leave the leaf node untouched so we can save the subtree root
// just as the leaf node for the top tree.
// `LEAF` is prepended for `Chunk` hash computation.
leaf
}
#[inline]
fn node(&mut self, left: CryptoSHA256Hash, right: CryptoSHA256Hash) -> CryptoSHA256Hash {
self.write(left.as_ref());
self.write(right.as_ref());
self.hash()
}
}
#[cfg(test)]
mod tests {
use crate::RawLeafSha3Algorithm;
use merkle_light::{hash::Algorithm, merkle::MerkleTree};
use std::hash::Hasher;
#[test]
fn test_root() {
let results = [
[
86, 124, 71, 168, 121, 121, 77, 212, 137, 162, 16, 222, 193, 125, 49, 204, 89, 25,
188, 66, 125, 19, 141, 113, 106, 129, 7, 224, 37, 226, 219, 203,
],
[
41, 66, 83, 171, 49, 203, 249, 13, 187, 190, 247, 85, 167, 95, 241, 96, 29, 167,
144, 227, 92, 54, 95, 83, 14, 124, 26, 28, 169, 4, 220, 248,
],
];
for (test_index, n_chunk) in [6, 7].into_iter().enumerate() {
let mut data = Vec::with_capacity(n_chunk);
for _ in 0..n_chunk {
let mut a = RawLeafSha3Algorithm::default();
a.write(&[0; 256]);
data.push(a.hash());
}
let mt = MerkleTree::<_, RawLeafSha3Algorithm>::new(data);
println!("{:?} {}", mt.root(), hex::encode(mt.root()));
assert_eq!(results[test_index], mt.root());
}
}
}

6
common/spec/Cargo.toml Normal file
View File

@ -0,0 +1,6 @@
[package]
name = "zgs_spec"
version = "0.1.0"
edition = "2021"
[dependencies]

18
common/spec/src/lib.rs Normal file
View File

@ -0,0 +1,18 @@
pub const KB: usize = 1024;
pub const MB: usize = 1024 * KB;
pub const GB: usize = 1024 * MB;
pub const TB: usize = 1024 * GB;
pub const BYTES_PER_SECTOR: usize = 256;
pub const BYTES_PER_SEAL: usize = 4 * KB;
pub const BYTES_PER_SCRATCHPAD: usize = 64 * KB;
pub const BYTES_PER_LOAD: usize = 256 * KB;
pub const BYTES_PER_PRICING: usize = 8 * GB;
pub const BYTES_PER_MAX_MINING_RANGE: usize = 8 * TB;
pub const SECTORS_PER_LOAD: usize = BYTES_PER_LOAD / BYTES_PER_SECTOR;
pub const SECTORS_PER_SEAL: usize = BYTES_PER_SEAL / BYTES_PER_SECTOR;
pub const SECTORS_PER_PRICING: usize = BYTES_PER_PRICING / BYTES_PER_SECTOR;
pub const SECTORS_PER_MAX_MINING_RANGE: usize = BYTES_PER_MAX_MINING_RANGE / BYTES_PER_SECTOR;
pub const SEALS_PER_LOAD: usize = BYTES_PER_LOAD / BYTES_PER_SEAL;

View File

@ -0,0 +1,13 @@
[package]
name = "task_executor"
version = "0.1.0"
authors = ["Sigma Prime <contact@sigmaprime.io>"]
edition = "2021"
[dependencies]
exit-future = "0.2.0"
futures = "0.3.21"
lazy_static = "1.4.0"
lighthouse_metrics = { path = "../lighthouse_metrics" }
tokio = { version = "1.19.2", features = ["rt"] }
tracing = "0.1.35"

View File

@ -0,0 +1,369 @@
#[macro_use]
extern crate tracing;
mod metrics;
pub mod test_utils;
use futures::channel::mpsc::Sender;
use futures::prelude::*;
use std::sync::Weak;
use tokio::runtime::{Handle, Runtime};
pub use tokio::task::JoinHandle;
/// Provides a reason when node is shut down.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum ShutdownReason {
/// The node shut down successfully.
Success(&'static str),
/// The node shut down due to an error condition.
Failure(&'static str),
}
impl ShutdownReason {
pub fn message(&self) -> &'static str {
match self {
ShutdownReason::Success(msg) => msg,
ShutdownReason::Failure(msg) => msg,
}
}
}
/// Provides a `Handle` by either:
///
/// 1. Holding a `Weak<Runtime>` and calling `Runtime::handle`.
/// 2. Directly holding a `Handle` and cloning it.
///
/// This enum allows the `TaskExecutor` to work in production where a `Weak<Runtime>` is directly
/// accessible and in testing where the `Runtime` is hidden outside our scope.
#[derive(Clone)]
pub enum HandleProvider {
Runtime(Weak<Runtime>),
Handle(Handle),
}
impl From<Handle> for HandleProvider {
fn from(handle: Handle) -> Self {
HandleProvider::Handle(handle)
}
}
impl From<Weak<Runtime>> for HandleProvider {
fn from(weak_runtime: Weak<Runtime>) -> Self {
HandleProvider::Runtime(weak_runtime)
}
}
impl HandleProvider {
/// Returns a `Handle` to a `Runtime`.
///
/// May return `None` if the weak reference to the `Runtime` has been dropped (this generally
/// means Lighthouse is shutting down).
pub fn handle(&self) -> Option<Handle> {
match self {
HandleProvider::Runtime(weak_runtime) => weak_runtime
.upgrade()
.map(|runtime| runtime.handle().clone()),
HandleProvider::Handle(handle) => Some(handle.clone()),
}
}
}
/// A wrapper over a runtime handle which can spawn async and blocking tasks.
#[derive(Clone)]
pub struct TaskExecutor {
/// The handle to the runtime on which tasks are spawned
handle_provider: HandleProvider,
/// The receiver exit future which on receiving shuts down the task
exit: exit_future::Exit,
/// Sender given to tasks, so that if they encounter a state in which execution cannot
/// continue they can request that everything shuts down.
///
/// The task must provide a reason for shutting down.
signal_tx: Sender<ShutdownReason>,
}
impl TaskExecutor {
/// Create a new task executor.
///
/// ## Note
///
/// This function should only be used during testing. In production, prefer to obtain an
/// instance of `Self` via a `environment::RuntimeContext` (see the `lighthouse/environment`
/// crate).
pub fn new<T: Into<HandleProvider>>(
handle: T,
exit: exit_future::Exit,
signal_tx: Sender<ShutdownReason>,
) -> Self {
Self {
handle_provider: handle.into(),
exit,
signal_tx,
}
}
/// Clones the task executor adding a service name.
pub fn clone_with_name(&self) -> Self {
TaskExecutor {
handle_provider: self.handle_provider.clone(),
exit: self.exit.clone(),
signal_tx: self.signal_tx.clone(),
}
}
/// A convenience wrapper for `Self::spawn` which ignores a `Result` as long as both `Ok`/`Err`
/// are of type `()`.
///
/// The purpose of this function is to create a compile error if some function which previously
/// returned `()` starts returning something else. Such a case may otherwise result in
/// accidental error suppression.
pub fn spawn_ignoring_error(
&self,
task: impl Future<Output = Result<(), ()>> + Send + 'static,
name: &'static str,
) {
self.spawn(task.map(|_| ()), name)
}
/// Spawn a task to monitor the completion of another task.
///
/// If the other task exits by panicking, then the monitor task will shut down the executor.
fn spawn_monitor<R: Send>(
&self,
task_handle: impl Future<Output = Result<R, tokio::task::JoinError>> + Send + 'static,
name: &'static str,
) {
let mut shutdown_sender = self.shutdown_sender();
if let Some(handle) = self.handle() {
handle.spawn(async move {
let timer = metrics::start_timer_vec(&metrics::TASKS_HISTOGRAM, &[name]);
if let Err(join_error) = task_handle.await {
if let Ok(panic) = join_error.try_into_panic() {
let message = panic.downcast_ref::<&str>().unwrap_or(&"<none>");
error!(
task_name = name,
message = message,
advice = "Please check above for a backtrace and notify \
the developers",
"Task panic. This is a bug!",
);
let _ = shutdown_sender
.try_send(ShutdownReason::Failure("Panic (fatal error)"));
}
}
drop(timer);
});
} else {
debug!("Couldn't spawn monitor task. Runtime shutting down")
}
}
/// Spawn a future on the tokio runtime.
///
/// The future is wrapped in an `exit_future::Exit`. The task is cancelled when the corresponding
/// exit_future `Signal` is fired/dropped.
///
/// The future is monitored via another spawned future to ensure that it doesn't panic. In case
/// of a panic, the executor will be shut down via `self.signal_tx`.
///
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn(&self, task: impl Future<Output = ()> + Send + 'static, name: &'static str) {
if let Some(task_handle) = self.spawn_handle(task, name) {
self.spawn_monitor(task_handle, name)
}
}
/// Spawn a future on the tokio runtime. This function does not wrap the task in an `exit_future::Exit`
/// like [spawn](#method.spawn).
/// The caller of this function is responsible for wrapping up the task with an `exit_future::Exit` to
/// ensure that the task gets canceled appropriately.
/// This function generates prometheus metrics on number of tasks and task duration.
///
/// This is useful in cases where the future to be spawned needs to do additional cleanup work when
/// the task is completed/canceled (e.g. writing local variables to disk) or the task is created from
/// some framework which does its own cleanup (e.g. a hyper server).
pub fn spawn_without_exit(
&self,
task: impl Future<Output = ()> + Send + 'static,
name: &'static str,
) {
if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) {
let int_gauge_1 = int_gauge.clone();
let future = task.then(move |_| {
int_gauge_1.dec();
futures::future::ready(())
});
int_gauge.inc();
if let Some(handle) = self.handle() {
handle.spawn(future);
} else {
debug!("Couldn't spawn task. Runtime shutting down");
}
}
}
/// Spawn a blocking task on a dedicated tokio thread pool wrapped in an exit future.
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn_blocking<F>(&self, task: F, name: &'static str)
where
F: FnOnce() + Send + 'static,
{
if let Some(task_handle) = self.spawn_blocking_handle(task, name) {
self.spawn_monitor(task_handle, name)
}
}
/// Spawn a future on the tokio runtime wrapped in an `exit_future::Exit` returning an optional
/// join handle to the future.
/// The task is canceled when the corresponding exit_future `Signal` is fired/dropped.
///
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn_handle<R: Send + 'static>(
&self,
task: impl Future<Output = R> + Send + 'static,
name: &'static str,
) -> Option<tokio::task::JoinHandle<Option<R>>> {
let exit = self.exit.clone();
if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) {
// Task is shutdown before it completes if `exit` receives
let int_gauge_1 = int_gauge.clone();
let future = future::select(Box::pin(task), exit).then(move |either| {
let result = match either {
future::Either::Left((value, _)) => {
trace!(task = name, "Async task completed");
Some(value)
}
future::Either::Right(_) => {
debug!(task = name, "Async task shutdown, exit received");
None
}
};
int_gauge_1.dec();
futures::future::ready(result)
});
int_gauge.inc();
if let Some(handle) = self.handle() {
Some(handle.spawn(future))
} else {
debug!("Couldn't spawn task. Runtime shutting down");
None
}
} else {
None
}
}
/// Spawn a blocking task on a dedicated tokio thread pool wrapped in an exit future returning
/// a join handle to the future.
/// If the runtime doesn't exist, this will return None.
/// The Future returned behaves like the standard JoinHandle which can return an error if the
/// task failed.
/// This function generates prometheus metrics on number of tasks and task duration.
pub fn spawn_blocking_handle<F, R>(
&self,
task: F,
name: &'static str,
) -> Option<impl Future<Output = Result<R, tokio::task::JoinError>>>
where
F: FnOnce() -> R + Send + 'static,
R: Send + 'static,
{
let timer = metrics::start_timer_vec(&metrics::BLOCKING_TASKS_HISTOGRAM, &[name]);
metrics::inc_gauge_vec(&metrics::BLOCKING_TASKS_COUNT, &[name]);
let join_handle = if let Some(handle) = self.handle() {
handle.spawn_blocking(task)
} else {
debug!("Couldn't spawn task. Runtime shutting down");
return None;
};
let future = async move {
let result = match join_handle.await {
Ok(result) => {
trace!(task = name, "Blocking task completed");
Ok(result)
}
Err(e) => {
debug!(error = %e, "Blocking task ended unexpectedly");
Err(e)
}
};
drop(timer);
metrics::dec_gauge_vec(&metrics::BLOCKING_TASKS_COUNT, &[name]);
result
};
Some(future)
}
/// Block the current (non-async) thread on the completion of some future.
///
/// ## Warning
///
/// This method is "dangerous" since calling it from an async thread will result in a panic! Any
/// use of this outside of testing should be very deeply considered as Lighthouse has been
/// burned by this function in the past.
///
/// Determining what is an "async thread" is rather challenging; just because a function isn't
/// marked as `async` doesn't mean it's not being called from an `async` function or there isn't
/// a `tokio` context present in the thread-local storage due to some `rayon` funkiness. Talk to
/// @paulhauner if you plan to use this function in production. He has put metrics in here to
/// track any use of it, so don't think you can pull a sneaky one on him.
pub fn block_on_dangerous<F: Future>(
&self,
future: F,
name: &'static str,
) -> Option<F::Output> {
let timer = metrics::start_timer_vec(&metrics::BLOCK_ON_TASKS_HISTOGRAM, &[name]);
metrics::inc_gauge_vec(&metrics::BLOCK_ON_TASKS_COUNT, &[name]);
let handle = self.handle()?;
let exit = self.exit.clone();
debug!(name = "name", "Starting block_on task");
handle.block_on(async {
let output = tokio::select! {
output = future => {
debug!(
name = "name",
"Completed block_on task"
);
Some(output)
},
_ = exit => {
debug!(
name = "name",
"Cancelled block_on task"
);
None
}
};
metrics::dec_gauge_vec(&metrics::BLOCK_ON_TASKS_COUNT, &[name]);
drop(timer);
output
})
}
/// Returns a `Handle` to the current runtime.
pub fn handle(&self) -> Option<Handle> {
self.handle_provider.handle()
}
/// Returns a copy of the `exit_future::Exit`.
pub fn exit(&self) -> exit_future::Exit {
self.exit.clone()
}
/// Get a channel to request shutting down.
pub fn shutdown_sender(&self) -> Sender<ShutdownReason> {
self.signal_tx.clone()
}
}

View File

@ -0,0 +1,36 @@
/// Handles async task metrics
use lazy_static::lazy_static;
pub use lighthouse_metrics::*;
lazy_static! {
pub static ref ASYNC_TASKS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
"async_tasks_count",
"Total number of async tasks spawned using spawn",
&["async_task_count"]
);
pub static ref BLOCKING_TASKS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
"blocking_tasks_count",
"Total number of async tasks spawned using spawn_blocking",
&["blocking_task_count"]
);
pub static ref BLOCKING_TASKS_HISTOGRAM: Result<HistogramVec> = try_create_histogram_vec(
"blocking_tasks_histogram",
"Time taken by blocking tasks",
&["blocking_task_hist"]
);
pub static ref BLOCK_ON_TASKS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
"block_on_tasks_count",
"Total number of block_on_dangerous tasks spawned",
&["name"]
);
pub static ref BLOCK_ON_TASKS_HISTOGRAM: Result<HistogramVec> = try_create_histogram_vec(
"block_on_tasks_histogram",
"Time taken by block_on_dangerous tasks",
&["name"]
);
pub static ref TASKS_HISTOGRAM: Result<HistogramVec> = try_create_histogram_vec(
"async_tasks_time_histogram",
"Time taken by async tasks",
&["async_task_hist"]
);
}

View File

@ -0,0 +1,51 @@
use crate::TaskExecutor;
use std::sync::Arc;
use tokio::runtime;
/// Whilst the `TestRuntime` is not necessarily useful in itself, it provides the necessary
/// components for creating a `TaskExecutor` during tests.
///
/// May create its own runtime or use an existing one.
///
/// ## Warning
///
/// This struct should never be used in production, only testing.
pub struct TestRuntime {
runtime: Option<Arc<tokio::runtime::Runtime>>,
_runtime_shutdown: exit_future::Signal,
pub task_executor: TaskExecutor,
}
impl Default for TestRuntime {
/// If called *inside* an existing runtime, instantiates `Self` using a handle to that runtime. If
/// called *outside* any existing runtime, create a new `Runtime` and keep it alive until the
/// `Self` is dropped.
fn default() -> Self {
let (runtime_shutdown, exit) = exit_future::signal();
let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
let runtime = Arc::new(
runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap(),
);
let handle = runtime.handle().clone();
let task_executor = TaskExecutor::new(handle, exit, shutdown_tx);
Self {
runtime: Some(runtime),
_runtime_shutdown: runtime_shutdown,
task_executor,
}
}
}
impl Drop for TestRuntime {
fn drop(&mut self) {
if let Some(runtime) = self.runtime.take() {
Arc::try_unwrap(runtime).unwrap().shutdown_background()
}
}
}

View File

@ -0,0 +1,8 @@
[package]
name = "unused_port"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]

View File

@ -0,0 +1,55 @@
use std::net::{TcpListener, UdpSocket};
#[derive(Copy, Clone)]
pub enum Transport {
Tcp,
Udp,
}
/// A convenience function for `unused_port(Transport::Tcp)`.
pub fn unused_tcp_port() -> Result<u16, String> {
unused_port(Transport::Tcp)
}
/// A convenience function for `unused_port(Transport::Tcp)`.
pub fn unused_udp_port() -> Result<u16, String> {
unused_port(Transport::Udp)
}
/// A bit of hack to find an unused port.
///
/// Does not guarantee that the given port is unused after the function exits, just that it was
/// unused before the function started (i.e., it does not reserve a port).
///
/// ## Notes
///
/// It is possible that users are unable to bind to the ports returned by this function as the OS
/// has a buffer period where it doesn't allow binding to the same port even after the socket is
/// closed. We might have to use SO_REUSEADDR socket option from `std::net2` crate in that case.
pub fn unused_port(transport: Transport) -> Result<u16, String> {
let local_addr = match transport {
Transport::Tcp => {
let listener = TcpListener::bind("127.0.0.1:0").map_err(|e| {
format!("Failed to create TCP listener to find unused port: {:?}", e)
})?;
listener.local_addr().map_err(|e| {
format!(
"Failed to read TCP listener local_addr to find unused port: {:?}",
e
)
})?
}
Transport::Udp => {
let socket = UdpSocket::bind("127.0.0.1:0")
.map_err(|e| format!("Failed to create UDP socket to find unused port: {:?}", e))?;
socket.local_addr().map_err(|e| {
format!(
"Failed to read UDP socket local_addr to find unused port: {:?}",
e
)
})?
}
};
Ok(local_addr.port())
}

View File

@ -0,0 +1,9 @@
[package]
name = "zgs_seal"
version = "0.1.0"
edition = "2021"
[dependencies]
ethereum-types = "0.14"
zgs_spec = {path = "../spec"}
tiny-keccak = { version = "2.0.2", features = ["keccak"] }

View File

@ -0,0 +1,62 @@
use ethereum_types::H256;
use tiny_keccak::{Hasher, Keccak};
pub fn compute_first_mask_seed(
miner_id: &H256,
context_digest: &H256,
start_sector: u64,
) -> [u8; 96] {
let mut output = [0u8; 96];
output[0..32].copy_from_slice(&miner_id.0);
output[32..64].copy_from_slice(&context_digest.0);
output[88..96].clone_from_slice(&start_sector.to_be_bytes());
output
}
fn keccak(input: impl AsRef<[u8]>) -> [u8; 32] {
let mut hasher = Keccak::v256();
let mut output = [0u8; 32];
hasher.update(input.as_ref());
hasher.finalize(&mut output);
output
}
pub fn seal_with_mask(data: &mut [u8], first_mask: [u8; 32]) {
assert!(data.len() % 32 == 0);
let mut mask = first_mask;
for word in data.chunks_mut(32) {
word.iter_mut().zip(mask.iter()).for_each(|(x, y)| *x ^= *y);
mask = keccak(&*word);
}
}
pub fn unseal_with_mask(data: &mut [u8], first_mask: [u8; 32]) {
assert!(data.len() % 32 == 0);
let mut mask = first_mask;
data.chunks_exact_mut(32).for_each(|x| {
let next_mask = keccak(&*x);
x.iter_mut()
.zip(mask.iter())
.for_each(|(x, mask)| *x ^= *mask);
mask = next_mask;
})
}
pub fn seal_with_mask_seed(data: &mut [u8], first_mask_seed: impl AsRef<[u8]>) {
seal_with_mask(data, keccak(first_mask_seed))
}
pub fn unseal_with_mask_seed(data: &mut [u8], first_mask_seed: impl AsRef<[u8]>) {
unseal_with_mask(data, keccak(first_mask_seed))
}
pub fn seal(data: &mut [u8], miner_id: &H256, context_digest: &H256, start_sector: u64) {
let first_mask_seed = compute_first_mask_seed(miner_id, context_digest, start_sector);
seal_with_mask_seed(data, first_mask_seed)
}
pub fn unseal(data: &mut [u8], miner_id: &H256, context_digest: &H256, start_sector: u64) {
let first_mask_seed = compute_first_mask_seed(miner_id, context_digest, start_sector);
unseal_with_mask_seed(data, first_mask_seed)
}

View File

@ -0,0 +1,14 @@
[package]
name = "zgs_version"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
git-version = "0.3.5"
target_info = "0.1.0"
[dev-dependencies]
regex = "1.5.6"

View File

@ -0,0 +1,47 @@
use git_version::git_version;
use target_info::Target;
/// Returns the current version of this build of Lighthouse.
///
/// A plus-sign (`+`) is appended to the git commit if the tree is dirty.
///
/// ## Example
///
/// `Lighthouse/v1.5.1-67da032+`
pub const VERSION: &str = git_version!(
args = [
"--always",
"--dirty=+",
"--abbrev=7",
// NOTE: using --match instead of --exclude for compatibility with old Git
"--match=thiswillnevermatchlol"
],
prefix = "zgs/v0.0.1-",
fallback = "unknown"
);
/// Returns `VERSION`, but with platform information appended to the end.
///
/// ## Example
///
/// `zgs/v0.0.1-67da032+/x86_64-linux`
pub fn version_with_platform() -> String {
format!("{}/{}-{}", VERSION, Target::arch(), Target::os())
}
#[cfg(test)]
mod test {
use super::*;
use regex::Regex;
#[test]
fn version_formatting() {
let re =
Regex::new(r"^zgs/v[0-9]+\.[0-9]+\.[0-9]+(-rc.[0-9])?-[[:xdigit:]]{7}\+?$").unwrap();
assert!(
re.is_match(VERSION),
"version doesn't match regex: {}",
VERSION
);
}
}

70
doc/install.md Normal file
View File

@ -0,0 +1,70 @@
# Install
ZeroGStorage requires Rust 1.71.0 and Go to build.
## Install Rust
We recommend installing Rust through [rustup](https://www.rustup.rs/).
* Linux
Install Rust
```shell
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
rustup install 1.65.0
```
Other dependencies
* Ubuntu
```shell
sudo apt-get install clang cmake build-essential
```
* Mac
Install Rust
```shell
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
rustup install 1.65.0
```
```shell
brew install llvm cmake
```
* Windows
Download and run the rustup installer from [this link](https://static.rust-lang.org/rustup/dist/i686-pc-windows-gnu/rustup-init.exe).
Install LLVM, pre-built binaries can be downloaded from [this link](https://releases.llvm.org/download.html).
## Install Go
* Linux
```shell
# Download the Go installer
wget https://go.dev/dl/go1.19.3.linux-amd64.tar.gz
# Extract the archive
sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.19.3.linux-amd64.tar.gz
# Add /usr/local/go/bin to the PATH environment variable by adding the following line to your ~/.profile.
export PATH=$PATH:/usr/local/go/bin
```
* Mac
Download the Go installer from https://go.dev/dl/go1.19.3.darwin-amd64.pkg.
Open the package file you downloaded and follow the prompts to install Go.
* Windows
Download the Go installer from https://go.dev/dl/go1.19.3.windows-amd64.msi.
Open the MSI file you downloaded and follow the prompts to install Go.
## Build from source
```shell
# Download code
$ git clone https://github.com/zero-gravity-labs/zerog-storage-rust.git
$ cd zerog-storage-rust
$ git submodule update --init
# Build in release mode
$ cargo build --release
```

91
doc/run.md Normal file
View File

@ -0,0 +1,91 @@
# Run
## Deploy contract: Token, Flow and Mine contracts
### Setup Environment
Install dependencies Node.js, yarn, hardhat.
* Linux
* Ubuntu
```shell
# node >=12.18
sudo apt install npm
sudo npm install --global yarn
sudo npm install --global hardhat
```
* Mac
```shell
brew install node
sudo npm install --global yarn
sudo npm install --global hardhat
```
* Windows
Download and install node from [here](https://nodejs.org/en/download/)
```shell
npm install --global yarn
npm install --global hardhat
```
### Download contract source code
```shell
git clone https://github.com/zero-gravity-labs/zerog-storage-contracts.git
cd zerog-storage-contracts
```
Add target network to your hardhat.config.js, i.e.
```shell
# example
networks: {
targetNetwork: {
url: "******",
accounts: [
"******",
],
},
},
```
### Compile
```shell
yarn
yarn compile
```
### Deploy contract
```shell
npx hardhat run scripts/deploy.ts --network targetnetwork
```
Keep contracts addresses
## Run ZeroGStorage
Update coinfig run/config.toml as required:
```shell
# p2p port
network_libp2p_port
# rpc endpoint
rpc_listen_address
# peer nodes
network_libp2p_nodes
# flow contract address
log_contract_address
# mine contract address
mine_contract_address
# layer one blockchain rpc endpoint
blockchain_rpc_endpoint
```
Run node
```shell
cd run
../target/release/zgs_node --config config.toml
```

1
node/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/target

42
node/Cargo.toml Normal file
View File

@ -0,0 +1,42 @@
[package]
name = "zgs_node"
version = "0.1.0"
edition = "2021"
build = "build.rs"
[dependencies]
anyhow = { version = "=1.0.58", features = ["backtrace"] }
clap = { version = "3.2.5", features = ["cargo"] }
ctrlc = "3.2.2"
error-chain = "0.12.4"
ethereum-types = "0.14"
exit-future = "0.2.0"
futures = "0.3.21"
file_location_cache = { path = "file_location_cache" }
zgs_version = { path = "../common/zgs_version" }
log_entry_sync = { path = "./log_entry_sync" }
miner = { path = "./miner" }
network = { path = "./network" }
router = { path = "./router" }
rpc = { path = "./rpc" }
shared_types = { path = "./shared_types" }
storage = { path = "./storage" }
storage-async = { path = "./storage-async" }
sync = { path = "./sync" }
task_executor = { path = "../common/task_executor" }
tokio = { version = "1.19.2", features = ["full"] }
tokio-stream = { version = "0.1.9", features = ["sync"] }
toml = "0.5.9"
tracing = "0.1.35"
tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
tracing-appender = { version = "0.2.2" }
chunk_pool = { path = "./chunk_pool" }
itertools = "0.10.5"
serde = { version = "1.0.137", features = ["derive"] }
duration-str = "0.5.1"
config = "0.13.1"
[dependencies.libp2p]
version = "0.45.1"
default-features = true
features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns-tokio", "tcp-tokio", "plaintext", "secp256k1"]

13
node/build.rs Normal file
View File

@ -0,0 +1,13 @@
use std::process::Command;
fn main() {
println!("cargo:rerun-if-changed=../zerog-storage-client");
let status = Command::new("go")
.current_dir("../zerog-storage-client")
.args(vec!["build", "-o", "../target"])
.status()
.unwrap();
println!("build zerog-storage-client with status {}", status);
}

View File

@ -0,0 +1,15 @@
[package]
name = "chunk_pool"
version = "0.1.0"
edition = "2021"
[dependencies]
anyhow = { version = "=1.0.58", features = ["backtrace"] }
shared_types = { path = "../shared_types" }
storage-async = { path = "../storage-async" }
log_entry_sync = { path = "../log_entry_sync" }
network = { path = "../network" }
tokio = { version = "1.19.2", features = ["sync"] }
async-lock = "2.5.0"
hashlink = "0.8.0"
tracing = "0.1.35"

View File

@ -0,0 +1,93 @@
use super::mem_pool::MemoryChunkPool;
use crate::mem_pool::FileID;
use anyhow::Result;
use network::NetworkMessage;
use shared_types::ChunkArray;
use std::sync::Arc;
use storage_async::Store;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
/// Handle the cached file when uploaded completely and verified from blockchain.
/// Generally, the file will be persisted into log store.
pub struct ChunkPoolHandler {
receiver: UnboundedReceiver<FileID>,
mem_pool: Arc<MemoryChunkPool>,
log_store: Store,
sender: UnboundedSender<NetworkMessage>,
}
impl ChunkPoolHandler {
pub(crate) fn new(
receiver: UnboundedReceiver<FileID>,
mem_pool: Arc<MemoryChunkPool>,
log_store: Store,
sender: UnboundedSender<NetworkMessage>,
) -> Self {
ChunkPoolHandler {
receiver,
mem_pool,
log_store,
sender,
}
}
/// Writes memory cached chunks into store and finalize transaction.
/// Note, a separate thread should be spawned to call this method.
pub async fn handle(&mut self) -> Result<bool> {
let id = match self.receiver.recv().await {
Some(id) => id,
None => return Ok(false),
};
debug!(?id, "Received task to finalize transaction");
// TODO(qhz): remove from memory pool after transaction finalized,
// when store support to write chunks with reference.
if let Some(file) = self.mem_pool.remove_cached_file(&id.root).await {
// If there is still cache of chunks, write them into store
let mut segments: Vec<ChunkArray> = file.segments.into_values().collect();
while let Some(seg) = segments.pop() {
if !self
.log_store
.put_chunks_with_tx_hash(id.tx_id.seq, id.tx_id.hash, seg)
.await?
{
return Ok(false);
}
}
}
if !self
.log_store
.finalize_tx_with_hash(id.tx_id.seq, id.tx_id.hash)
.await?
{
return Ok(false);
}
debug!(?id, "Transaction finalized");
// always remove file from pool after transaction finalized
self.mem_pool.remove_file(&id.root).await;
let msg = NetworkMessage::AnnounceLocalFile { tx_id: id.tx_id };
if let Err(e) = self.sender.send(msg) {
error!(
"Failed to send NetworkMessage::AnnounceLocalFile message, tx_seq={}, err={}",
id.tx_id.seq, e
);
}
Ok(true)
}
pub async fn run(mut self) {
info!("Worker started to finalize transactions");
loop {
if let Err(e) = self.handle().await {
warn!("Failed to write chunks or finalize transaction, {:?}", e);
}
}
}
}

View File

@ -0,0 +1,38 @@
#[macro_use]
extern crate tracing;
mod handler;
mod mem_pool;
pub use handler::ChunkPoolHandler;
pub use mem_pool::{FileID, MemoryChunkPool, SegmentInfo};
use std::sync::Arc;
use std::time::Duration;
#[derive(Clone, Copy, Debug)]
pub struct Config {
pub write_window_size: usize,
pub max_cached_chunks_all: usize,
pub max_writings: usize,
pub expiration_time_secs: u64,
}
impl Config {
pub fn expiration_time(&self) -> Duration {
Duration::from_secs(self.expiration_time_secs)
}
}
pub fn unbounded(
config: Config,
log_store: storage_async::Store,
network_send: tokio::sync::mpsc::UnboundedSender<network::NetworkMessage>,
) -> (Arc<MemoryChunkPool>, ChunkPoolHandler) {
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let mem_pool = Arc::new(MemoryChunkPool::new(config, log_store.clone(), sender));
let handler = ChunkPoolHandler::new(receiver, mem_pool.clone(), log_store, network_send);
(mem_pool, handler)
}

View File

@ -0,0 +1,158 @@
use super::FileID;
use crate::{Config, SegmentInfo};
use anyhow::{bail, Result};
use hashlink::LinkedHashMap;
use shared_types::{bytes_to_chunks, ChunkArray, DataRoot, Transaction, CHUNK_SIZE};
use std::collections::HashMap;
use std::ops::Add;
use std::time::{Duration, Instant};
/// Used to cache chunks in memory pool and persist into db once log entry
/// retrieved from blockchain.
pub struct MemoryCachedFile {
pub id: FileID,
/// Window to control the cache of each file
pub segments: HashMap<usize, ChunkArray>,
/// Total number of chunks for the cache file, which is updated from log entry.
pub total_chunks: usize,
/// Used for garbage collection. It is updated when new segment uploaded.
expired_at: Instant,
/// Number of chunks that's currently cached for this file
pub cached_chunk_num: usize,
}
impl MemoryCachedFile {
fn new(root: DataRoot, timeout: Duration) -> Self {
MemoryCachedFile {
id: FileID {
root,
tx_id: Default::default(),
},
segments: HashMap::default(),
total_chunks: 0,
expired_at: Instant::now().add(timeout),
cached_chunk_num: 0,
}
}
/// Updates file with transaction once log entry retrieved from blockchain.
/// So that, write memory cached segments into database.
pub fn update_with_tx(&mut self, tx: &Transaction) {
self.total_chunks = bytes_to_chunks(tx.size as usize);
self.id.tx_id = tx.id();
}
fn update_expiration_time(&mut self, timeout: Duration) {
self.expired_at = Instant::now().add(timeout);
}
#[allow(unused)]
fn is_completed(&self) -> bool {
self.total_chunks > 0 && self.cached_chunk_num >= self.total_chunks
}
fn should_flush(&self) -> bool {
self.total_chunks > 0 && self.cached_chunk_num > 0
}
}
/// ChunkPoolCache is used to cache small files that log entry not retrieved
/// from L1 blockchain yet.
pub struct ChunkPoolCache {
config: Config,
/// All cached files.
/// Note, file root is used as key instead of `tx_seq`, since log entry
/// not retrieved yet.
files: LinkedHashMap<DataRoot, MemoryCachedFile>,
/// Total number of chunks that cached in the memory pool.
pub total_chunks: usize,
}
impl ChunkPoolCache {
pub fn new(config: Config) -> Self {
ChunkPoolCache {
config,
files: LinkedHashMap::default(),
total_chunks: 0,
}
}
pub fn get_file(&self, root: &DataRoot) -> Option<&MemoryCachedFile> {
self.files.get(root)
}
pub fn get_file_mut(&mut self, root: &DataRoot) -> Option<&mut MemoryCachedFile> {
self.files.get_mut(root)
}
pub fn remove_file(&mut self, root: &DataRoot) -> Option<MemoryCachedFile> {
let file = self.files.remove(root)?;
self.update_total_chunks_when_remove_file(&file);
Some(file)
}
/// Remove files that no new segment uploaded for a long time.
///
/// Note, when log sync delayed, files may be also garbage collected if the
/// entire file uploaded. Because, it is hard to check if log sync delayed
/// or user upload an invalid file, e.g. for attack purpose.
///
/// Once garbage collected, user could simply upload the entire file again,
/// which is fast enough due to small file size.
fn garbage_collect(&mut self) {
let now = Instant::now();
while let Some((_, file)) = self.files.front() {
if file.expired_at > now {
return;
}
if let Some((r, f)) = self.files.pop_front() {
self.update_total_chunks_when_remove_file(&f);
debug!("Garbage collected for file {}", r);
}
}
}
fn update_total_chunks_when_remove_file(&mut self, file: &MemoryCachedFile) {
assert!(self.total_chunks >= file.cached_chunk_num);
self.total_chunks -= file.cached_chunk_num;
}
/// Caches the specified segment in memory.
///
/// Returns if there are cached segments and log entry also retrieved.
pub fn cache_segment(&mut self, seg_info: SegmentInfo) -> Result<bool> {
// always GC at first
self.garbage_collect();
let file = self
.files
.entry(seg_info.root)
.or_insert_with(|| MemoryCachedFile::new(seg_info.root, self.config.expiration_time()));
// Segment already cached in memory. Directly return OK
if file.segments.contains_key(&seg_info.seg_index) {
return Ok(file.should_flush());
}
// Otherwise, just cache segment in memory
let num_chunks = seg_info.seg_data.len() / CHUNK_SIZE;
// Limits the cached chunks in the memory pool.
if self.total_chunks + num_chunks > self.config.max_cached_chunks_all {
bail!(
"exceeds the maximum cached chunks of whole pool: {}",
self.config.max_cached_chunks_all
);
}
// Cache segment and update the counter for cached chunks.
self.total_chunks += num_chunks;
file.cached_chunk_num += num_chunks;
file.update_expiration_time(self.config.expiration_time());
file.segments.insert(seg_info.seg_index, seg_info.into());
Ok(file.should_flush())
}
}

View File

@ -0,0 +1,332 @@
use super::chunk_cache::{ChunkPoolCache, MemoryCachedFile};
use super::chunk_write_control::ChunkPoolWriteCtrl;
use super::FileID;
use crate::Config;
use anyhow::{bail, Result};
use async_lock::Mutex;
use log_entry_sync::LogSyncEvent;
use shared_types::{
bytes_to_chunks, compute_segment_size, ChunkArray, DataRoot, Transaction, CHUNK_SIZE,
};
use std::sync::Arc;
use storage_async::Store;
use tokio::sync::broadcast::{error::RecvError, Receiver};
use tokio::sync::mpsc::UnboundedSender;
struct Inner {
config: Config,
segment_cache: ChunkPoolCache,
write_control: ChunkPoolWriteCtrl,
}
impl Inner {
fn new(config: Config) -> Self {
Inner {
config,
segment_cache: ChunkPoolCache::new(config),
write_control: ChunkPoolWriteCtrl::new(config),
}
}
fn after_flush_cache(&mut self) {
assert!(self.write_control.total_writings > 0);
self.write_control.total_writings -= 1;
}
/// Return the tx seq and all segments that belong to the root.
fn get_all_cached_segments_to_write(
&mut self,
root: &DataRoot,
) -> Result<(FileID, Vec<ChunkArray>)> {
// Limits the number of writing threads.
if self.write_control.total_writings >= self.config.max_writings {
bail!("too many data writing: {}", self.config.max_writings);
}
let file = match self.segment_cache.remove_file(root) {
Some(f) => f,
None => bail!("file not found to write into store {:?}", root),
};
let id = file.id;
let segs = file.segments.into_values().collect();
self.write_control.total_writings += 1;
Ok((id, segs))
}
}
pub struct SegmentInfo {
pub root: DataRoot,
pub seg_data: Vec<u8>,
pub seg_index: usize,
pub chunks_per_segment: usize,
}
impl From<SegmentInfo> for ChunkArray {
fn from(seg_info: SegmentInfo) -> Self {
let start_index = seg_info.seg_index * seg_info.chunks_per_segment;
ChunkArray {
data: seg_info.seg_data,
start_index: start_index as u64,
}
}
}
/// Caches data chunks in memory before the entire file uploaded to storage node
/// and data root verified on blockchain.
pub struct MemoryChunkPool {
inner: Mutex<Inner>,
log_store: Store,
sender: UnboundedSender<FileID>,
}
impl MemoryChunkPool {
pub(crate) fn new(config: Config, log_store: Store, sender: UnboundedSender<FileID>) -> Self {
MemoryChunkPool {
inner: Mutex::new(Inner::new(config)),
log_store,
sender,
}
}
pub fn validate_segment_size(&self, segment: &Vec<u8>) -> Result<()> {
if segment.is_empty() {
bail!("data is empty");
}
if segment.len() % CHUNK_SIZE != 0 {
bail!("invalid data length");
}
Ok(())
}
pub async fn cache_chunks(&self, seg_info: SegmentInfo) -> Result<()> {
let root = seg_info.root;
debug!("cache_chunks, root={:?} index={}", root, seg_info.seg_index);
let should_flush = self
.inner
.lock()
.await
.segment_cache
.cache_segment(seg_info)?;
// store and finalize the cached file if completed
if should_flush {
debug!("cache_chunk: flush cached chunks");
self.write_all_cached_chunks_and_finalize(root).await?;
}
Ok(())
}
pub async fn write_chunks(
&self,
seg_info: SegmentInfo,
file_id: FileID,
file_size: usize,
) -> Result<()> {
let total_chunks = bytes_to_chunks(file_size);
debug!(
"Begin to write segment, root={}, segment_size={}, segment_index={}",
seg_info.root,
seg_info.seg_data.len(),
seg_info.seg_index,
);
//Write the segment in window
let (total_segments, _) = compute_segment_size(total_chunks, seg_info.chunks_per_segment);
self.inner.lock().await.write_control.write_segment(
file_id,
seg_info.seg_index,
total_segments,
)?;
// Write memory cached segments into store.
// TODO(qhz): error handling
// 1. Push the failed segment back to front. (enhance store to return Err(ChunkArray))
// 2. Put the incompleted segments back to memory pool.
let seg = ChunkArray {
data: seg_info.seg_data,
start_index: (seg_info.seg_index * seg_info.chunks_per_segment) as u64,
};
match self
.log_store
.put_chunks_with_tx_hash(file_id.tx_id.seq, file_id.tx_id.hash, seg)
.await
{
Ok(true) => {}
Ok(false) => {
self.inner
.lock()
.await
.write_control
.on_write_failed(&seg_info.root, seg_info.seg_index);
// remove the file if transaction reverted
self.inner
.lock()
.await
.write_control
.remove_file(&seg_info.root);
bail!("Transaction reverted, please upload again");
}
Err(e) => {
self.inner
.lock()
.await
.write_control
.on_write_failed(&seg_info.root, seg_info.seg_index);
return Err(e);
}
}
let all_uploaded = self
.inner
.lock()
.await
.write_control
.on_write_succeeded(&seg_info.root, seg_info.seg_index);
// Notify to finalize transaction asynchronously.
if all_uploaded {
if let Err(e) = self.sender.send(file_id) {
// Channel receiver will not be dropped until program exit.
bail!("channel send error: {}", e);
}
debug!("Queue to finalize transaction for file {}", seg_info.root);
}
Ok(())
}
/// Updates the cached file info when log entry retrieved from blockchain.
pub async fn update_file_info(&self, tx: &Transaction) -> Result<bool> {
let mut inner = self.inner.lock().await;
// Do nothing if file not uploaded yet.
let file = match inner.segment_cache.get_file_mut(&tx.data_merkle_root) {
Some(f) => f,
None => return Ok(false),
};
// Update the file info with transaction.
file.update_with_tx(tx);
// File partially uploaded and it's up to user thread
// to write chunks into store and finalize transaction.
if file.cached_chunk_num < file.total_chunks {
return Ok(true);
}
// Otherwise, notify to write all memory cached chunks and finalize transaction.
let file_id = FileID {
root: tx.data_merkle_root,
tx_id: tx.id(),
};
if let Err(e) = self.sender.send(file_id) {
// Channel receiver will not be dropped until program exit.
bail!("channel send error: {}", e);
}
Ok(true)
}
pub async fn monitor_log_entry(chunk_pool: Arc<Self>, mut receiver: Receiver<LogSyncEvent>) {
info!("Start to monitor log entry");
loop {
match receiver.recv().await {
Ok(LogSyncEvent::ReorgDetected { .. }) => {}
Ok(LogSyncEvent::Reverted { .. }) => {}
Ok(LogSyncEvent::TxSynced { tx }) => {
if let Err(_e) = chunk_pool.update_file_info(&tx).await {
error!(
"Failed to update file info. tx seq={}, tx_root={}",
tx.seq, tx.data_merkle_root
);
}
}
Err(RecvError::Closed) => {
// program terminated
info!("Completed to monitor log entry");
return;
}
Err(RecvError::Lagged(lagged)) => {
error!(%lagged, "Lagged messages: (Lagged)");
}
}
}
}
pub(crate) async fn remove_cached_file(&self, root: &DataRoot) -> Option<MemoryCachedFile> {
self.inner.lock().await.segment_cache.remove_file(root)
}
pub(crate) async fn remove_file(&self, root: &DataRoot) -> bool {
let mut inner = self.inner.lock().await;
inner.segment_cache.remove_file(root).is_some()
|| inner.write_control.remove_file(root).is_some()
}
pub async fn check_already_has_cache(&self, root: &DataRoot) -> bool {
self.inner
.lock()
.await
.segment_cache
.get_file(root)
.is_some()
}
async fn write_all_cached_chunks_and_finalize(&self, root: DataRoot) -> Result<()> {
let (file, mut segments) = self
.inner
.lock()
.await
.get_all_cached_segments_to_write(&root)?;
while let Some(seg) = segments.pop() {
// TODO(qhz): error handling
// 1. Push the failed segment back to front. (enhance store to return Err(ChunkArray))
// 2. Put the incompleted segments back to memory pool.
match self
.log_store
.put_chunks_with_tx_hash(file.tx_id.seq, file.tx_id.hash, seg)
.await
{
Ok(true) => {}
Ok(false) => {
self.inner.lock().await.after_flush_cache();
bail!("Transaction reverted, please upload again");
}
Err(e) => {
self.inner.lock().await.after_flush_cache();
return Err(e);
}
}
}
self.inner.lock().await.after_flush_cache();
if let Err(e) = self.sender.send(file) {
// Channel receiver will not be dropped until program exit.
bail!("channel send error: {}", e);
}
Ok(())
}
pub async fn get_uploaded_seg_num(&self, root: &DataRoot) -> (usize, bool) {
let inner = self.inner.lock().await;
if let Some(file) = inner.segment_cache.get_file(root) {
(file.cached_chunk_num, true)
} else if let Some(file) = inner.write_control.get_file(root) {
(file.uploaded_seg_num(), false)
} else {
(0, false)
}
}
}

View File

@ -0,0 +1,200 @@
use super::FileID;
use crate::Config;
use anyhow::{bail, Result};
use shared_types::DataRoot;
use std::collections::HashMap;
/// The segment status in sliding window
#[derive(PartialEq, Eq, Debug)]
enum SlotStatus {
Writing, // segment in writing
Finished, // segment uploaded in store
}
/// Sliding window is used to control the concurrent uploading process of a file.
/// Bounded window allows segments to be uploaded concurrenly, while having a capacity
/// limit on writing threads per file. Meanwhile, the left_boundary field records
/// how many segments have been uploaded.
struct CtrlWindow {
size: usize,
left_boundary: usize,
slots: HashMap<usize, SlotStatus>,
}
impl CtrlWindow {
fn new(size: usize) -> Self {
CtrlWindow {
size,
left_boundary: 0,
slots: HashMap::default(),
}
}
/// Check if the specified slot by `index` has been already uploaded.
/// Note, this function do not check about the right boundary.
fn check_duplicate(&self, index: usize) -> bool {
index < self.left_boundary || self.slots.contains_key(&index)
}
/// Should call check_duplicate and handle the duplicated case before calling this function.
/// This function assumes that there are no duplicate slots in the window.
fn start_writing(&mut self, index: usize) -> Result<()> {
assert!(index >= self.left_boundary);
if index >= self.left_boundary + self.size {
bail!(
"index exceeds window limit, index = {}, left_boundary = {}, window_size = {}",
index,
self.left_boundary,
self.size
);
}
assert!(!self.slots.contains_key(&index));
self.slots.insert(index, SlotStatus::Writing);
Ok(())
}
fn rollback_writing(&mut self, index: usize) {
let slot_status = self.slots.remove(&index);
assert_eq!(slot_status, Some(SlotStatus::Writing));
}
fn finish_writing(&mut self, index: usize) {
let old_status = self.slots.insert(index, SlotStatus::Finished);
assert_eq!(old_status, Some(SlotStatus::Writing));
// move forward if leftmost slot completed
let mut left_boundary = self.left_boundary;
while let Some(&SlotStatus::Finished) = self.slots.get(&left_boundary) {
self.slots.remove(&left_boundary);
left_boundary += 1;
}
self.left_boundary = left_boundary;
}
}
/// To track the file uploading progress.
pub struct FileWriteCtrl {
pub id: FileID,
total_segments: usize,
window: CtrlWindow,
}
impl FileWriteCtrl {
fn new(id: FileID, total_segments: usize, window_size: usize) -> Self {
FileWriteCtrl {
id,
total_segments,
window: CtrlWindow::new(window_size),
}
}
pub fn uploaded_seg_num(&self) -> usize {
self.window.left_boundary
}
}
/// ChunkPoolWriteCtrl is used to track uploading progress for all files,
/// and limits the maximum number of threads to write segments into store.
pub struct ChunkPoolWriteCtrl {
config: Config,
/// Windows to control writing processes of files
files: HashMap<DataRoot, FileWriteCtrl>,
/// Total number of threads that are writing chunks into store.
pub total_writings: usize,
}
impl ChunkPoolWriteCtrl {
pub fn new(config: Config) -> Self {
ChunkPoolWriteCtrl {
files: HashMap::default(),
total_writings: 0,
config,
}
}
pub fn get_file(&self, root: &DataRoot) -> Option<&FileWriteCtrl> {
self.files.get(root)
}
pub fn remove_file(&mut self, root: &DataRoot) -> Option<FileWriteCtrl> {
self.files.remove(root)
}
pub fn write_segment(
&mut self,
id: FileID,
seg_index: usize,
total_segments: usize,
) -> Result<()> {
let file_ctrl = self.files.entry(id.root).or_insert_with(|| {
FileWriteCtrl::new(id, total_segments, self.config.write_window_size)
});
// ensure the tx_id not changed during file uploading
if file_ctrl.id != id {
self.files.remove(&id.root);
bail!("Transaction reverted when uploading segments, please try again");
}
if file_ctrl.total_segments != total_segments {
bail!(
"file size in segment doesn't match with file size declared in previous segment. Previous total segments:{}, current total segments:{}s",
file_ctrl.total_segments,
total_segments
);
}
// Segment already uploaded.
if file_ctrl.window.check_duplicate(seg_index) {
bail!("segment has already been uploaded or is being uploaded");
}
// Limits the number of writing threads.
if self.total_writings >= self.config.max_writings {
bail!("too many data writing: {}", self.config.max_writings);
}
file_ctrl.window.start_writing(seg_index)?;
self.total_writings += 1;
Ok(())
}
pub fn on_write_succeeded(&mut self, root: &DataRoot, seg_index: usize) -> bool {
let file_ctrl = match self.files.get_mut(root) {
Some(w) => w,
None => return false,
};
file_ctrl.window.finish_writing(seg_index);
assert!(self.total_writings > 0);
self.total_writings -= 1;
debug!(
"Succeeded to write segment, root={}, seg_index={}, total_writings={}",
root, seg_index, self.total_writings
);
// All chunks of file written into store.
file_ctrl.window.left_boundary >= file_ctrl.total_segments
}
pub fn on_write_failed(&mut self, root: &DataRoot, seg_index: usize) {
let file_ctrl = match self.files.get_mut(root) {
Some(w) => w,
None => return,
};
//Rollback the segment status if failed
file_ctrl.window.rollback_writing(seg_index);
assert!(self.total_writings > 0);
self.total_writings -= 1;
}
}

View File

@ -0,0 +1,15 @@
mod chunk_cache;
mod chunk_pool_inner;
mod chunk_write_control;
pub use chunk_pool_inner::MemoryChunkPool;
pub use chunk_pool_inner::SegmentInfo;
use shared_types::DataRoot;
use shared_types::TxID;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct FileID {
pub root: DataRoot,
pub tx_id: TxID,
}

View File

@ -0,0 +1,13 @@
[package]
name = "file_location_cache"
version = "0.1.0"
edition = "2021"
[dependencies]
hashlink = "0.8.0"
network = { path = "../network" }
parking_lot = "0.12.1"
rand = "0.8.5"
tracing = "0.1.35"
priority-queue = "1.2.3"
shared_types = { path = "../shared_types" }

View File

@ -0,0 +1,532 @@
use crate::Config;
use network::types::SignedAnnounceFile;
use network::PeerId;
use parking_lot::Mutex;
use priority_queue::PriorityQueue;
use rand::seq::IteratorRandom;
use shared_types::{timestamp_now, TxID};
use std::cmp::Reverse;
use std::collections::HashMap;
/// Caches limited announcements of specified file from different peers.
struct AnnouncementCache {
/// Maximum number of announcements in cache.
capacity: usize,
/// Timeout in seconds to expire the cached announcement.
/// This is because file may be removed from the announced
/// storage node.
timeout_secs: u32,
/// All cached announcements that mapped from peer id to announcement.
/// Note, only cache the latest announcement for each peer.
items: HashMap<PeerId, SignedAnnounceFile>,
/// All announcements are prioritized by timestamp.
/// The top element is the oldest announcement.
priorities: PriorityQueue<PeerId, Reverse<u32>>,
}
impl AnnouncementCache {
fn new(capacity: usize, timeout_secs: u32) -> Self {
assert!(capacity > 0);
AnnouncementCache {
capacity,
timeout_secs,
items: Default::default(),
priorities: Default::default(),
}
}
/// Returns the priority of the oldest announcement if any.
fn peek_priority(&self) -> Option<Reverse<u32>> {
let (_, &Reverse(ts)) = self.priorities.peek()?;
Some(Reverse(ts))
}
/// Removes the oldest announcement if any.
fn pop(&mut self) -> Option<SignedAnnounceFile> {
let (peer_id, _) = self.priorities.pop()?;
self.items.remove(&peer_id)
}
fn len(&self) -> usize {
self.items.len()
}
/// Garbage collects expired announcements.
fn garbage_collect(&mut self) -> usize {
let mut collected = 0;
let now = timestamp_now();
while let Some((_, &Reverse(ts))) = self.priorities.peek() {
if ts + self.timeout_secs > now {
break;
}
self.pop();
collected += 1;
}
collected
}
fn do_insert_or_update(&mut self, announcement: SignedAnnounceFile) {
let peer_id = announcement.peer_id.clone().into();
self.priorities
.push(peer_id, Reverse(announcement.timestamp));
self.items.insert(peer_id, announcement);
}
/// Insert the specified `announcement` into cache.
fn insert(&mut self, announcement: SignedAnnounceFile) {
self.garbage_collect();
let peer_id = announcement.peer_id.clone().into();
if let Some(existing) = self.items.get(&peer_id) {
// ignore older announcement
if announcement.timestamp <= existing.timestamp {
return;
}
}
// insert or update
self.do_insert_or_update(announcement);
// remove oldest one if capacity exceeded
if self.items.len() > self.capacity {
self.pop();
}
}
/// Randomly pick an announcement if any.
fn random(&mut self) -> (Option<SignedAnnounceFile>, usize) {
let collected = self.garbage_collect();
// all announcements garbage collected
if self.items.is_empty() {
return (None, collected);
}
let choosed = self
.items
.iter()
.choose(&mut rand::thread_rng())
.map(|(_, item)| item.clone());
(choosed, collected)
}
/// Returns all announcements.
fn all(&mut self) -> (Vec<SignedAnnounceFile>, usize) {
let collected = self.garbage_collect();
let result = self.items.values().cloned().collect();
(result, collected)
}
}
/// Caches announcements for different files.
struct FileCache {
/// Cache configuration.
config: Config,
/// Total number of announcements cached for all files.
total_announcements: usize,
/// All cached files that mapped from `tx_id` to `AnnouncementCache`.
files: HashMap<TxID, AnnouncementCache>,
/// All files are prioritized by timestamp.
/// The top element is the `AnnouncementCache` that has the oldest announcement.
priorities: PriorityQueue<TxID, Reverse<u32>>,
}
impl FileCache {
fn new(config: Config) -> Self {
FileCache {
config,
total_announcements: 0,
files: Default::default(),
priorities: Default::default(),
}
}
/// Insert the specified `announcement` into cache.
fn insert(&mut self, announcement: SignedAnnounceFile) {
let tx_id = announcement.tx_id;
let item = self.files.entry(tx_id).or_insert_with(|| {
AnnouncementCache::new(
self.config.max_entries_per_file,
self.config.entry_expiration_time_secs,
)
});
assert!(self.total_announcements >= item.len());
self.total_announcements -= item.len();
item.insert(announcement);
if let Some(priority) = item.peek_priority() {
self.priorities.push(tx_id, priority);
}
self.total_announcements += item.len();
if self.total_announcements > self.config.max_entries_total {
self.pop();
}
}
/// Removes the oldest file announcement.
fn pop(&mut self) -> Option<SignedAnnounceFile> {
let (&tx_id, _) = self.priorities.peek()?;
let item = self.files.get_mut(&tx_id)?;
let result = item.pop()?;
self.update_on_announcement_cache_changed(&tx_id, 1);
Some(result)
}
/// Randomly pick a announcement of specified file by `tx_id`.
fn random(&mut self, tx_id: TxID) -> Option<SignedAnnounceFile> {
let item = self.files.get_mut(&tx_id)?;
let (result, collected) = item.random();
self.update_on_announcement_cache_changed(&tx_id, collected);
result
}
fn update_on_announcement_cache_changed(&mut self, tx_id: &TxID, removed: usize) {
if removed == 0 {
return;
}
assert!(self.total_announcements >= removed);
self.total_announcements -= removed;
let item = match self.files.get_mut(tx_id) {
Some(v) => v,
None => return,
};
if let Some(priority) = item.peek_priority() {
// update priority if changed
self.priorities.change_priority(tx_id, priority);
} else {
// remove entry if empty
self.files.remove(tx_id);
self.priorities.remove(tx_id);
}
}
/// Returns all the announcements of specified file by `tx_id`.
fn all(&mut self, tx_id: TxID) -> Option<Vec<SignedAnnounceFile>> {
let item = self.files.get_mut(&tx_id)?;
let (result, collected) = item.all();
self.update_on_announcement_cache_changed(&tx_id, collected);
Some(result)
}
}
pub struct FileLocationCache {
cache: Mutex<FileCache>,
}
impl Default for FileLocationCache {
fn default() -> Self {
FileLocationCache {
cache: Mutex::new(FileCache::new(Default::default())),
}
}
}
impl FileLocationCache {
pub fn insert(&self, announcement: SignedAnnounceFile) {
self.cache.lock().insert(announcement);
}
pub fn get_one(&self, tx_id: TxID) -> Option<SignedAnnounceFile> {
self.cache.lock().random(tx_id)
}
pub fn get_all(&self, tx_id: TxID) -> Vec<SignedAnnounceFile> {
self.cache.lock().all(tx_id).unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use std::cmp::Reverse;
use network::{types::SignedAnnounceFile, PeerId};
use shared_types::{timestamp_now, TxID};
use crate::{test_util::AnnounceFileBuilder, Config};
use super::{AnnouncementCache, FileCache};
fn create_file(peer_id: Option<PeerId>, timestamp: u32) -> SignedAnnounceFile {
let builder = AnnounceFileBuilder::default().with_timestamp(timestamp);
if let Some(id) = peer_id {
builder.with_peer_id(id).build()
} else {
builder.build()
}
}
#[test]
fn test_annoucement_cache_peek_priority() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
// empty by default
assert_eq!(cache.peek_priority(), None);
// one file with timestamp `now - 5`
let t1 = now - 5;
cache.insert(create_file(None, t1));
assert_eq!(cache.peek_priority(), Some(Reverse(t1)));
// newly file with timestamp `now - 4`
let t2 = now - 4;
cache.insert(create_file(None, t2));
assert_eq!(cache.peek_priority(), Some(Reverse(t1)));
// old file with timestamp `now - 6`
let t3 = now - 6;
cache.insert(create_file(None, t3));
assert_eq!(cache.peek_priority(), Some(Reverse(t3)));
}
#[test]
fn test_annoucement_cache_pop_len() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
// empty by default
assert_eq!(cache.pop(), None);
assert_eq!(cache.len(), 0);
cache.insert(create_file(None, now - 2));
cache.insert(create_file(None, now - 3));
cache.insert(create_file(None, now - 1));
assert_eq!(cache.len(), 3);
// pop from oldest to newest
assert_eq!(cache.pop().unwrap().timestamp, now - 3);
assert_eq!(cache.pop().unwrap().timestamp, now - 2);
assert_eq!(cache.pop().unwrap().timestamp, now - 1);
assert_eq!(cache.pop(), None);
assert_eq!(cache.len(), 0);
}
#[test]
fn test_annoucement_cache_garbage_collect() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
assert_eq!(cache.garbage_collect(), 0);
cache.do_insert_or_update(create_file(None, now - 5000));
cache.do_insert_or_update(create_file(None, now - 5001));
cache.do_insert_or_update(create_file(None, now - 2000));
cache.do_insert_or_update(create_file(None, now + 10));
// gc for expired only
assert_eq!(cache.garbage_collect(), 2);
assert_eq!(cache.len(), 2);
assert_eq!(cache.peek_priority(), Some(Reverse(now - 2000)));
}
#[test]
fn test_annoucement_cache_insert_gc() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
// prepare expired items
cache.do_insert_or_update(create_file(None, now - 5000));
cache.do_insert_or_update(create_file(None, now - 5001));
// insert with gc
cache.insert(create_file(None, now - 1));
assert_eq!(cache.len(), 1);
assert_eq!(cache.peek_priority(), Some(Reverse(now - 1)));
}
#[test]
fn test_annoucement_cache_insert_ignore_older() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
let peer_id = PeerId::random();
// insert `now - 2`
cache.insert(create_file(Some(peer_id), now - 2));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 2)));
assert_eq!(cache.len(), 1);
// ignore for older announcement of the same peer
cache.insert(create_file(Some(peer_id), now - 3));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 2)));
assert_eq!(cache.len(), 1);
// however, older announcement allowed from other peer
cache.insert(create_file(None, now - 3));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 3)));
assert_eq!(cache.len(), 2);
}
#[test]
fn test_annoucement_cache_insert_overwrite() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
let peer_id = PeerId::random();
// insert `now - 2`
cache.insert(create_file(Some(peer_id), now - 2));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 2)));
assert_eq!(cache.len(), 1);
// overwrite with newly item
cache.insert(create_file(Some(peer_id), now - 1));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 1)));
assert_eq!(cache.len(), 1);
}
#[test]
fn test_annoucement_cache_insert_cap_exceeded() {
let mut cache = AnnouncementCache::new(3, 3600);
let now = timestamp_now();
cache.insert(create_file(None, now - 2));
cache.insert(create_file(None, now - 3));
cache.insert(create_file(None, now - 4));
// oldest `now - 5` will be removed
cache.insert(create_file(None, now - 5));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 4)));
assert_eq!(cache.len(), 3);
// oldest `now - 4` will be removed
cache.insert(create_file(None, now - 1));
assert_eq!(cache.peek_priority(), Some(Reverse(now - 3)));
assert_eq!(cache.len(), 3);
}
#[test]
fn test_annoucement_cache_random() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
assert_eq!(cache.random().0, None);
cache.insert(create_file(None, now - 1));
assert_eq!(cache.random().0.unwrap().timestamp, now - 1);
cache.insert(create_file(None, now - 2));
cache.insert(create_file(None, now - 3));
let picked = cache.random().0.unwrap().timestamp;
assert!(picked >= now - 3 && picked < now);
}
#[test]
fn test_annoucement_cache_all() {
let mut cache = AnnouncementCache::new(100, 3600);
let now = timestamp_now();
assert_eq!(cache.all().0, vec![]);
cache.insert(create_file(None, now - 1));
cache.insert(create_file(None, now - 2));
cache.insert(create_file(None, now - 3));
assert_all_files(cache.all().0, vec![now - 3, now - 2, now - 1])
}
fn assert_all_files(files: Vec<SignedAnnounceFile>, sorted_timestamps: Vec<u32>) {
let mut timestamps: Vec<u32> = files.iter().map(|f| f.timestamp).collect();
timestamps.sort();
assert_eq!(timestamps, sorted_timestamps);
}
fn create_file_cache(total_entries: usize, file_entries: usize, timeout: u32) -> FileCache {
FileCache::new(Config {
max_entries_total: total_entries,
max_entries_per_file: file_entries,
entry_expiration_time_secs: timeout,
})
}
fn create_file_2(tx_id: TxID, peer_id: PeerId, timestamp: u32) -> SignedAnnounceFile {
AnnounceFileBuilder::default()
.with_tx_id(tx_id)
.with_peer_id(peer_id)
.with_timestamp(timestamp)
.build()
}
fn assert_file(file: &SignedAnnounceFile, tx_id: TxID, peer_id: PeerId, timestamp: u32) {
assert_eq!(file.tx_id, tx_id);
assert_eq!(PeerId::from(file.peer_id.clone()), peer_id);
assert_eq!(file.timestamp, timestamp);
}
#[test]
fn test_file_cache_insert_pop() {
let mut cache = create_file_cache(100, 3, 3600);
let now = timestamp_now();
assert_eq!(cache.total_announcements, 0);
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let tx1 = TxID::random_hash(1);
let tx2 = TxID::random_hash(2);
cache.insert(create_file_2(tx1, peer1, now - 1));
assert_eq!(cache.total_announcements, 1);
cache.insert(create_file_2(tx2, peer1, now - 2));
assert_eq!(cache.total_announcements, 2);
cache.insert(create_file_2(tx1, peer2, now - 3));
assert_eq!(cache.total_announcements, 3);
assert_file(&cache.pop().unwrap(), tx1, peer2, now - 3);
assert_eq!(cache.total_announcements, 2);
assert_file(&cache.pop().unwrap(), tx2, peer1, now - 2);
assert_eq!(cache.total_announcements, 1);
assert_file(&cache.pop().unwrap(), tx1, peer1, now - 1);
assert_eq!(cache.total_announcements, 0);
assert_eq!(cache.pop(), None);
}
#[test]
fn test_file_cache_insert_cap_exceeded() {
let mut cache = create_file_cache(3, 3, 3600);
let now = timestamp_now();
let tx1 = TxID::random_hash(1);
cache.insert(create_file_2(tx1, PeerId::random(), now - 7));
cache.insert(create_file_2(tx1, PeerId::random(), now - 8));
cache.insert(create_file_2(tx1, PeerId::random(), now - 9));
assert_eq!(cache.total_announcements, 3);
// insert more files and cause to max entries limited
let tx2 = TxID::random_hash(2);
cache.insert(create_file_2(tx2, PeerId::random(), now - 1));
assert_all_files(cache.all(tx1).unwrap_or_default(), vec![now - 8, now - 7]);
cache.insert(create_file_2(tx2, PeerId::random(), now - 2));
assert_all_files(cache.all(tx1).unwrap_or_default(), vec![now - 7]);
cache.insert(create_file_2(tx2, PeerId::random(), now - 3));
assert_all_files(cache.all(tx1).unwrap_or_default(), vec![]);
assert_all_files(
cache.all(tx2).unwrap_or_default(),
vec![now - 3, now - 2, now - 1],
);
}
}

View File

@ -0,0 +1,20 @@
mod file_location_cache;
pub mod test_util;
pub use crate::file_location_cache::FileLocationCache;
pub struct Config {
pub max_entries_total: usize,
pub max_entries_per_file: usize,
pub entry_expiration_time_secs: u32,
}
impl Default for Config {
fn default() -> Self {
Config {
max_entries_total: 4096,
max_entries_per_file: 4,
entry_expiration_time_secs: 3600,
}
}
}

View File

@ -0,0 +1,47 @@
use network::{
libp2p::identity,
types::{AnnounceFile, SignedAnnounceFile},
Multiaddr, PeerId,
};
use shared_types::{timestamp_now, TxID};
#[derive(Default)]
pub struct AnnounceFileBuilder {
tx_id: Option<TxID>,
peer_id: Option<PeerId>,
timestamp: Option<u32>,
}
impl AnnounceFileBuilder {
pub fn with_tx_id(mut self, tx_id: TxID) -> Self {
self.tx_id = Some(tx_id);
self
}
pub fn with_peer_id(mut self, peer_id: PeerId) -> Self {
self.peer_id = Some(peer_id);
self
}
pub fn with_timestamp(mut self, timestamp: u32) -> Self {
self.timestamp = Some(timestamp);
self
}
pub fn build(self) -> SignedAnnounceFile {
let tx_id = self.tx_id.unwrap_or_else(|| TxID::random_hash(0));
let peer_id = self.peer_id.unwrap_or_else(PeerId::random);
let at: Multiaddr = "/ip4/127.0.0.1/tcp/10000".parse().unwrap();
let timestamp = self.timestamp.unwrap_or_else(timestamp_now);
let msg = AnnounceFile {
tx_id,
peer_id: peer_id.into(),
at: at.into(),
timestamp,
};
let keypair = identity::Keypair::generate_secp256k1();
msg.into_signed(&keypair).unwrap()
}
}

View File

@ -0,0 +1,24 @@
[package]
name = "log_entry_sync"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = { version = "=1.0.58", features = ["backtrace"] }
append_merkle = { path = "../../common/append_merkle" }
async-trait = "0.1.56"
ethereum-types = "0.14"
futures = "0.3.21"
jsonrpsee = { version = "0.14.0", features = ["full"] }
shared_types = { path = "../shared_types" }
task_executor = { path = "../../common/task_executor" }
tokio = "1.19.2"
ethers = { version = "^2", features = ["ws"] }
serde_json = "1.0.82"
storage = { path = "../storage" }
contract-interface = { path = "../../common/contract-interface" }
futures-core = "0.3.28"
futures-util = "0.3.28"
thiserror = "1.0.44"

View File

@ -0,0 +1,10 @@
extern crate core;
pub(crate) mod rpc_proxy;
mod sync_manager;
pub use rpc_proxy::ContractAddress;
pub use sync_manager::{
config::{CacheConfig, LogSyncConfig},
LogSyncEvent, LogSyncManager,
};

View File

@ -0,0 +1,30 @@
use crate::rpc_proxy::{ContractAddress, EvmRpcProxy, SubEvent, SubFilter};
use async_trait::async_trait;
use ethers::prelude::{Bytes, Middleware, Provider};
use ethers::providers::Ws;
use ethers::types::TransactionRequest;
use jsonrpsee::core::client::Subscription;
pub struct EthClient {
client: Provider<Ws>,
}
impl EthClient {
#[allow(unused)]
pub async fn new(url: &str) -> anyhow::Result<EthClient> {
let client = Provider::new(Ws::connect(url).await?);
Ok(Self { client })
}
}
#[async_trait]
impl EvmRpcProxy for EthClient {
async fn call(&self, to: ContractAddress, data: Bytes) -> anyhow::Result<Bytes> {
let request = TransactionRequest::new().to(to).data(data);
Ok(self.client.call(&request.into(), None).await?)
}
async fn sub_events(&self, _filter: SubFilter) -> Subscription<SubEvent> {
todo!()
}
}

View File

@ -0,0 +1,38 @@
use anyhow::Result;
use async_trait::async_trait;
use ethereum_types::{H160, H256};
use ethers::prelude::Bytes;
use jsonrpsee::core::client::Subscription;
// TODO: Define accounts/filter/events as associated types?
// TODO: Define an abstraction suitable for other chains.
#[async_trait]
pub trait EvmRpcProxy {
async fn call(&self, to: ContractAddress, data: Bytes) -> Result<Bytes>;
async fn sub_events(&self, filter: SubFilter) -> Subscription<SubEvent>;
}
pub type ContractAddress = H160;
pub type Topic = H256;
#[allow(unused)]
pub struct SubFilter {
to: Option<ContractAddress>,
topics: Vec<Topic>,
}
#[allow(unused)]
pub struct SubEvent {
/// Address
pub address: ContractAddress,
/// Topics
pub topics: Vec<Topic>,
/// Data
pub data: Bytes,
}
pub(crate) mod eth;

View File

@ -0,0 +1,65 @@
use crate::rpc_proxy::ContractAddress;
pub struct LogSyncConfig {
pub rpc_endpoint_url: String,
pub contract_address: ContractAddress,
pub cache_config: CacheConfig,
/// The block number where we start to sync data.
/// This is usually the block number when Zgs contract is deployed.
pub start_block_number: u64,
/// The number of blocks needed for confirmation on the blockchain.
/// This is used to rollback to a stable height if reorg happens during node restart.
/// TODO(zz): Some blockchains have better confirmation/finalization mechanisms.
pub confirmation_block_count: u64,
/// Maximum number of event logs to poll at a time.
pub log_page_size: u64,
// blockchain provider retry params
// the number of retries after a connection times out
pub rate_limit_retries: u32,
// the nubmer of retries for rate limited responses
pub timeout_retries: u32,
// the duration to wait before retry, in ms
pub initial_backoff: u64,
// the duration between each paginated getLogs RPC call, in ms.
// This is set to avoid triggering the throttling mechanism in the RPC server.
pub recover_query_delay: u64,
}
#[derive(Clone)]
pub struct CacheConfig {
/// The data with a size larger than this will not be cached.
/// This is reasonable because uploading
pub max_data_size: usize,
pub tx_seq_ttl: usize,
}
impl LogSyncConfig {
#[allow(clippy::too_many_arguments)]
pub fn new(
rpc_endpoint_url: String,
contract_address: ContractAddress,
start_block_number: u64,
confirmation_block_count: u64,
cache_config: CacheConfig,
log_page_size: u64,
rate_limit_retries: u32,
timeout_retries: u32,
initial_backoff: u64,
recover_query_delay: u64,
) -> Self {
Self {
rpc_endpoint_url,
contract_address,
cache_config,
start_block_number,
confirmation_block_count,
log_page_size,
rate_limit_retries,
timeout_retries,
initial_backoff,
recover_query_delay,
}
}
}

View File

@ -0,0 +1,58 @@
use crate::sync_manager::config::CacheConfig;
use shared_types::DataRoot;
use std::cmp;
use std::collections::HashMap;
struct CachedData {
/// Used for garbage collection.
last_seen_tx_seq: u64,
/// Complete data for a given DataRoot.
data: Vec<u8>,
}
pub struct DataCache {
root_to_data: HashMap<DataRoot, CachedData>,
config: CacheConfig,
}
impl DataCache {
pub fn new(config: CacheConfig) -> Self {
Self {
root_to_data: HashMap::new(),
config,
}
}
pub fn add_data(&mut self, root: DataRoot, tx_seq: u64, data: Vec<u8>) -> bool {
if data.len() > self.config.max_data_size {
// large data are not cached.
return false;
}
// TODO: insert partial data and merge here.
self.root_to_data
.entry(root)
.and_modify(|cached| {
cached.last_seen_tx_seq = cmp::max(tx_seq, cached.last_seen_tx_seq)
})
.or_insert(CachedData {
last_seen_tx_seq: tx_seq,
data,
});
true
}
/// Remove and return the data of a given `DataRoot`.
/// If two completed reverted transactions have the same root and both appear later,
/// the second one will have its data copied in `put_tx`.
pub fn pop_data(&mut self, root: &DataRoot) -> Option<Vec<u8>> {
self.root_to_data.remove(root).map(|e| e.data)
}
/// Remove timeout data entries according to TTL.
pub fn garbage_collect(&mut self, latest_tx_seq: u64) {
// We won't keep too many data, so it's okay to just iterate here.
self.root_to_data.retain(|_, cached| {
cached.last_seen_tx_seq + self.config.tx_seq_ttl as u64 >= latest_tx_seq
})
}
}

View File

@ -0,0 +1,356 @@
use crate::rpc_proxy::ContractAddress;
use crate::sync_manager::log_query::LogQuery;
use crate::sync_manager::{repeat_run_and_log, RETRY_WAIT_MS};
use anyhow::{anyhow, bail, Result};
use append_merkle::{Algorithm, Sha3Algorithm};
use contract_interface::{ZgsFlow, SubmissionNode, SubmitFilter};
use ethers::abi::RawLog;
use ethers::prelude::{BlockNumber, EthLogDecode, Http, Log, Middleware, Provider, U256};
use ethers::providers::{FilterKind, HttpRateLimitRetryPolicy, RetryClient, RetryClientBuilder};
use ethers::types::H256;
use futures::StreamExt;
use jsonrpsee::tracing::{debug, error, info};
use shared_types::{DataRoot, Transaction};
use std::collections::{BTreeMap, VecDeque};
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use task_executor::TaskExecutor;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
pub struct LogEntryFetcher {
contract_address: ContractAddress,
log_page_size: u64,
provider: Arc<Provider<RetryClient<Http>>>,
confirmation_delay: u64,
}
impl LogEntryFetcher {
pub async fn new(
url: &str,
contract_address: ContractAddress,
log_page_size: u64,
confirmation_delay: u64,
rate_limit_retries: u32,
timeout_retries: u32,
initial_backoff: u64,
) -> Result<Self> {
let provider = Arc::new(Provider::new(
RetryClientBuilder::default()
.rate_limit_retries(rate_limit_retries)
.timeout_retries(timeout_retries)
.initial_backoff(Duration::from_millis(initial_backoff))
.build(Http::from_str(url)?, Box::new(HttpRateLimitRetryPolicy)),
));
// TODO: `error` types are removed from the ABI json file.
Ok(Self {
contract_address,
provider,
log_page_size,
confirmation_delay,
})
}
pub fn start_recover(
&self,
start_block_number: u64,
end_block_number: u64,
executor: &TaskExecutor,
log_query_delay: Duration,
) -> UnboundedReceiver<LogFetchProgress> {
let provider = self.provider.clone();
let (recover_tx, recover_rx) = tokio::sync::mpsc::unbounded_channel();
let contract = ZgsFlow::new(self.contract_address, provider.clone());
let log_page_size = self.log_page_size;
executor.spawn(
async move {
let mut progress = start_block_number;
let mut filter = contract
.submit_filter()
.from_block(progress)
.to_block(end_block_number)
.filter;
let mut stream = LogQuery::new(&provider, &filter, log_query_delay)
.with_page_size(log_page_size);
debug!(
"start_recover starts, start={} end={}",
start_block_number, end_block_number
);
while let Some(maybe_log) = stream.next().await {
match maybe_log {
Ok(log) => {
let sync_progress =
if log.block_hash.is_some() && log.block_number.is_some() {
let synced_block = LogFetchProgress::SyncedBlock((
log.block_number.unwrap().as_u64(),
log.block_hash.unwrap(),
));
progress = log.block_number.unwrap().as_u64();
Some(synced_block)
} else {
None
};
debug!("recover: progress={:?}", sync_progress);
match SubmitFilter::decode_log(&RawLog {
topics: log.topics,
data: log.data.to_vec(),
}) {
Ok(event) => {
if let Err(e) = recover_tx
.send(submission_event_to_transaction(event))
.and_then(|_| match sync_progress {
Some(b) => recover_tx.send(b),
None => Ok(()),
})
{
error!("send error: e={:?}", e);
}
}
Err(e) => {
error!("log decode error: e={:?}", e);
}
}
}
Err(e) => {
error!("log query error: e={:?}", e);
filter = filter.from_block(progress);
stream = LogQuery::new(&provider, &filter, log_query_delay)
.with_page_size(log_page_size);
tokio::time::sleep(Duration::from_millis(RETRY_WAIT_MS)).await;
}
}
}
},
"log recover",
);
recover_rx
}
pub fn start_watch(
&self,
start_block_number: u64,
executor: &TaskExecutor,
) -> UnboundedReceiver<LogFetchProgress> {
let (watch_tx, watch_rx) = tokio::sync::mpsc::unbounded_channel();
let contract = ZgsFlow::new(self.contract_address, self.provider.clone());
let provider = self.provider.clone();
let mut log_confirmation_queue = LogConfirmationQueue::new(self.confirmation_delay);
executor.spawn(
async move {
let mut filter = contract
.submit_filter()
.from_block(start_block_number)
.filter;
debug!("start_watch starts, start={}", start_block_number);
let mut filter_id =
repeat_run_and_log(|| provider.new_filter(FilterKind::Logs(&filter))).await;
let mut progress = start_block_number;
loop {
match Self::watch_loop(
provider.as_ref(),
filter_id,
&watch_tx,
&mut log_confirmation_queue,
)
.await
{
Err(e) => {
error!("log sync watch error: e={:?}", e);
filter = filter.from_block(progress);
filter_id = repeat_run_and_log(|| {
provider.new_filter(FilterKind::Logs(&filter))
})
.await;
}
Ok(Some(p)) => {
progress = p;
info!("log sync to block number {:?}", progress);
}
Ok(None) => {
error!(
"log sync gets entries without progress? old_progress={}",
progress
)
}
}
tokio::time::sleep(Duration::from_millis(RETRY_WAIT_MS)).await;
}
},
"log watch",
);
watch_rx
}
async fn watch_loop(
provider: &Provider<RetryClient<Http>>,
filter_id: U256,
watch_tx: &UnboundedSender<LogFetchProgress>,
log_confirmation_queue: &mut LogConfirmationQueue,
) -> Result<Option<u64>> {
debug!("get block");
let latest_block = provider
.get_block(BlockNumber::Latest)
.await?
.ok_or_else(|| anyhow!("None for latest block"))?;
debug!("get filter changes");
let logs: Vec<Log> = provider.get_filter_changes(filter_id).await?;
if let Some(reverted) = log_confirmation_queue.push(logs)? {
watch_tx.send(LogFetchProgress::Reverted(reverted))?;
}
debug!("get filter end");
for log in log_confirmation_queue.confirm_logs(latest_block.number.unwrap().as_u64()) {
assert!(!log.removed.unwrap_or(false));
// TODO(zz): Log parse error means logs might be lost here.
let tx = SubmitFilter::decode_log(&RawLog {
topics: log.topics,
data: log.data.to_vec(),
})?;
watch_tx.send(submission_event_to_transaction(tx))?;
}
let progress = if latest_block.hash.is_some() && latest_block.number.is_some() {
Some((
latest_block.number.unwrap().as_u64(),
latest_block.hash.unwrap(),
))
} else {
None
};
if let Some(p) = &progress {
watch_tx.send(LogFetchProgress::SyncedBlock(*p))?;
}
Ok(progress.map(|p| p.0))
}
pub fn provider(&self) -> &Provider<RetryClient<Http>> {
self.provider.as_ref()
}
}
struct LogConfirmationQueue {
/// Keep the unconfirmed new logs.
/// The key is the block number and the value is the set of needed logs in that block.
queue: VecDeque<(u64, Vec<Log>)>,
latest_block_number: u64,
confirmation_delay: u64,
}
impl LogConfirmationQueue {
fn new(confirmation_delay: u64) -> Self {
Self {
queue: VecDeque::new(),
latest_block_number: 0,
confirmation_delay,
}
}
/// Push a set of new logs.
/// We assumes that these logs are in order, and removed logs are returned first.
///
/// Return `Ok(Some(tx_seq))` of the first reverted tx_seq if chain reorg happens.
/// `Err` is returned if assumptions are violated (like the log have missing fields).
fn push(&mut self, logs: Vec<Log>) -> Result<Option<u64>> {
let mut revert_to = None;
// First merge logs according to the block number.
let mut block_logs: BTreeMap<u64, Vec<Log>> = BTreeMap::new();
let mut removed_block_logs = BTreeMap::new();
for log in logs {
let set = if log.removed.unwrap_or(false) {
&mut removed_block_logs
} else {
&mut block_logs
};
let block_number = log
.block_number
.ok_or_else(|| anyhow!("block number missing"))?
.as_u64();
set.entry(block_number).or_default().push(log);
}
// Handle revert if it happens.
for (block_number, removed_logs) in &removed_block_logs {
if revert_to.is_none() {
let reverted_index = match self.queue.binary_search_by_key(block_number, |e| e.0) {
Ok(x) => x,
Err(x) => x,
};
self.queue.truncate(reverted_index);
let first = removed_logs.first().expect("not empty");
let first_reverted_tx_seq = SubmitFilter::decode_log(&RawLog {
topics: first.topics.clone(),
data: first.data.to_vec(),
})?
.submission_index
.as_u64();
revert_to = Some(first_reverted_tx_seq);
} else {
// Other removed logs should have larger tx seq, so no need to process them.
break;
}
}
// Add new logs to the queue.
for (block_number, new_logs) in block_logs {
if block_number <= self.queue.back().map(|e| e.0).unwrap_or(0) {
bail!("reverted without being notified");
}
self.queue.push_back((block_number, new_logs));
}
Ok(revert_to)
}
/// Pass in the latest block number and return the confirmed logs.
fn confirm_logs(&mut self, latest_block_number: u64) -> Vec<Log> {
self.latest_block_number = latest_block_number;
let mut confirmed_logs = Vec::new();
while let Some((block_number, _)) = self.queue.front() {
if *block_number
> self
.latest_block_number
.wrapping_sub(self.confirmation_delay)
{
break;
}
let (_, mut logs) = self.queue.pop_front().unwrap();
confirmed_logs.append(&mut logs);
}
confirmed_logs
}
}
#[derive(Debug)]
pub enum LogFetchProgress {
SyncedBlock((u64, H256)),
Transaction(Transaction),
Reverted(u64),
}
fn submission_event_to_transaction(e: SubmitFilter) -> LogFetchProgress {
LogFetchProgress::Transaction(Transaction {
stream_ids: vec![],
data: vec![],
data_merkle_root: nodes_to_root(&e.submission.nodes),
merkle_nodes: e
.submission
.nodes
.iter()
// the submission height is the height of the root node starting from height 0.
.map(|SubmissionNode { root, height }| (height.as_usize() + 1, root.into()))
.collect(),
start_entry_index: e.start_pos.as_u64(),
size: e.submission.length.as_u64(),
seq: e.submission_index.as_u64(),
})
}
fn nodes_to_root(node_list: &Vec<SubmissionNode>) -> DataRoot {
let mut root: DataRoot = node_list.last().expect("not empty").root.into();
for next_node in node_list[..node_list.len() - 1].iter().rev() {
root = Sha3Algorithm::parent(&next_node.root.into(), &root);
}
root
}

View File

@ -0,0 +1,183 @@
use ethers::prelude::{Filter, JsonRpcClient, Log, Middleware, Provider, ProviderError, U64};
use futures_core::stream::Stream;
use jsonrpsee::tracing::trace;
use std::future::Future;
use std::time::Duration;
use std::{
collections::VecDeque,
pin::Pin,
task::{Context, Poll},
};
use thiserror::Error;
pub(crate) type PinBoxFut<'a, T> =
Pin<Box<dyn Future<Output = Result<T, ProviderError>> + Send + 'a>>;
/// A log query provides streaming access to historical logs via a paginated
/// request. For streaming access to future logs, use [`Middleware::watch`] or
/// [`Middleware::subscribe_logs`]
pub struct LogQuery<'a, P> {
provider: &'a Provider<P>,
filter: Filter,
from_block: Option<U64>,
page_size: u64,
current_logs: VecDeque<Log>,
last_block: Option<U64>,
state: LogQueryState<'a>,
delay: Duration,
}
enum LogQueryState<'a> {
Initial,
LoadLastBlock(PinBoxFut<'a, U64>),
LoadLogs(PinBoxFut<'a, Vec<Log>>),
Consume,
}
impl<'a, P> LogQuery<'a, P>
where
P: JsonRpcClient,
{
/// Instantiate a new `LogQuery`
pub fn new(provider: &'a Provider<P>, filter: &Filter, delay: Duration) -> Self {
Self {
provider,
filter: filter.clone(),
from_block: filter.get_from_block(),
page_size: 10000,
current_logs: VecDeque::new(),
last_block: None,
state: LogQueryState::Initial,
delay,
}
}
/// set page size for pagination
pub fn with_page_size(mut self, page_size: u64) -> Self {
self.page_size = page_size;
self
}
}
macro_rules! rewake_with_new_state {
($ctx:ident, $this:ident, $new_state:expr) => {
$this.state = $new_state;
$ctx.waker().wake_by_ref();
return Poll::Pending
};
}
/// Errors while querying for logs
#[derive(Error, Debug)]
pub enum LogQueryError<E> {
/// Error loading latest block
#[error(transparent)]
LoadLastBlockError(E),
/// Error loading logs from block range
#[error(transparent)]
LoadLogsError(E),
}
impl<'a, P> Stream for LogQuery<'a, P>
where
P: JsonRpcClient,
{
type Item = Result<Log, LogQueryError<ProviderError>>;
fn poll_next(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let delay = self.delay;
match &mut self.state {
LogQueryState::Initial => {
if !self.filter.is_paginatable() {
// if not paginatable, load logs and consume
let filter = self.filter.clone();
let provider = self.provider;
#[allow(clippy::redundant_async_block)]
let fut = Box::pin(async move {
tokio::time::sleep(delay).await;
provider.get_logs(&filter).await
});
rewake_with_new_state!(ctx, self, LogQueryState::LoadLogs(fut));
} else {
// if paginatable, load last block
let fut = self.provider.get_block_number();
rewake_with_new_state!(ctx, self, LogQueryState::LoadLastBlock(fut));
}
}
LogQueryState::LoadLastBlock(fut) => {
match futures_util::ready!(fut.as_mut().poll(ctx)) {
Ok(last_block) => {
trace!("log_query: last_block={}", last_block);
self.last_block = Some(last_block);
// this is okay because we will only enter this state when the filter is
// paginatable i.e. from block is set
let from_block = self.filter.get_from_block().unwrap();
let to_block = from_block + self.page_size;
self.from_block = Some(to_block + 1);
let filter = self
.filter
.clone()
.from_block(from_block)
.to_block(to_block);
let provider = self.provider;
// load first page of logs
#[allow(clippy::redundant_async_block)]
let fut = Box::pin(async move {
tokio::time::sleep(delay).await;
provider.get_logs(&filter).await
});
rewake_with_new_state!(ctx, self, LogQueryState::LoadLogs(fut));
}
Err(err) => Poll::Ready(Some(Err(LogQueryError::LoadLastBlockError(err)))),
}
}
LogQueryState::LoadLogs(fut) => match futures_util::ready!(fut.as_mut().poll(ctx)) {
Ok(logs) => {
self.current_logs = VecDeque::from(logs);
rewake_with_new_state!(ctx, self, LogQueryState::Consume);
}
Err(err) => Poll::Ready(Some(Err(LogQueryError::LoadLogsError(err)))),
},
LogQueryState::Consume => {
let log = self.current_logs.pop_front();
if log.is_none() {
// consumed all the logs
if !self.filter.is_paginatable() {
Poll::Ready(None)
} else {
// load new logs if there are still more pages to go through
// can safely assume this will always be set in this state
let from_block = self.from_block.unwrap();
let to_block = from_block + self.page_size;
// no more pages to load, and everything is consumed
// can safely assume this will always be set in this state
if from_block > self.last_block.unwrap() {
return Poll::Ready(None);
}
// load next page
self.from_block = Some(to_block + 1);
let filter = self
.filter
.clone()
.from_block(from_block)
.to_block(to_block);
let provider = self.provider;
#[allow(clippy::redundant_async_block)]
let fut = Box::pin(async move {
tokio::time::sleep(delay).await;
provider.get_logs(&filter).await
});
rewake_with_new_state!(ctx, self, LogQueryState::LoadLogs(fut));
}
} else {
Poll::Ready(log.map(Ok))
}
}
}
}
}

View File

@ -0,0 +1,319 @@
use crate::sync_manager::config::LogSyncConfig;
use crate::sync_manager::data_cache::DataCache;
use crate::sync_manager::log_entry_fetcher::{LogEntryFetcher, LogFetchProgress};
use anyhow::{bail, Result};
use ethers::prelude::Middleware;
use futures::FutureExt;
use jsonrpsee::tracing::{debug, error, trace, warn};
use shared_types::{ChunkArray, Transaction};
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use storage::log_store::Store;
use task_executor::{ShutdownReason, TaskExecutor};
use tokio::sync::broadcast;
use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::RwLock;
const RETRY_WAIT_MS: u64 = 500;
const BROADCAST_CHANNEL_CAPACITY: usize = 16;
#[derive(Clone, Debug)]
pub enum LogSyncEvent {
/// Chain reorg detected without any operation yet.
ReorgDetected { tx_seq: u64 },
/// Transaction reverted in storage.
Reverted { tx_seq: u64 },
/// Synced a transaction from blockchain
TxSynced { tx: Transaction },
}
pub struct LogSyncManager {
config: LogSyncConfig,
log_fetcher: LogEntryFetcher,
store: Arc<RwLock<dyn Store>>,
data_cache: DataCache,
next_tx_seq: u64,
/// To broadcast events to handle in advance.
event_send: broadcast::Sender<LogSyncEvent>,
}
impl LogSyncManager {
pub async fn spawn(
config: LogSyncConfig,
executor: TaskExecutor,
store: Arc<RwLock<dyn Store>>,
) -> Result<broadcast::Sender<LogSyncEvent>> {
let next_tx_seq = store.read().await.next_tx_seq();
let executor_clone = executor.clone();
let mut shutdown_sender = executor.shutdown_sender();
let (event_send, _) = broadcast::channel(BROADCAST_CHANNEL_CAPACITY);
let event_send_cloned = event_send.clone();
// Spawn the task to sync log entries from the blockchain.
executor.spawn(
run_and_log(
move || {
shutdown_sender
.try_send(ShutdownReason::Failure("log sync failure"))
.expect("shutdown send error")
},
async move {
let log_fetcher = LogEntryFetcher::new(
&config.rpc_endpoint_url,
config.contract_address,
config.log_page_size,
config.confirmation_block_count,
config.rate_limit_retries,
config.timeout_retries,
config.initial_backoff,
)
.await?;
let data_cache = DataCache::new(config.cache_config.clone());
let mut log_sync_manager = Self {
config,
log_fetcher,
next_tx_seq,
store,
data_cache,
event_send,
};
// Load previous progress from db and check if chain reorg happens after restart.
// TODO(zz): Handle reorg instead of return.
let start_block_number =
match log_sync_manager.store.read().await.get_sync_progress()? {
// No previous progress, so just use config.
None => log_sync_manager.config.start_block_number,
Some((block_number, block_hash)) => {
match log_sync_manager
.log_fetcher
.provider()
.get_block(block_number)
.await
{
Ok(Some(b)) => {
if b.hash == Some(block_hash) {
block_number
} else {
warn!(
"log sync progress check hash fails, \
block_number={:?} expect={:?} get={:?}",
block_number, block_hash, b.hash
);
// Assume the blocks before this are not reverted.
block_number.saturating_sub(
log_sync_manager.config.confirmation_block_count,
)
}
}
e => {
error!("log sync progress check rpc fails, e={:?}", e);
bail!("log sync start error");
}
}
}
};
let latest_block_number = log_sync_manager
.log_fetcher
.provider()
.get_block_number()
.await?
.as_u64();
// Start watching before recovery to ensure that no log is skipped.
// TODO(zz): Rate limit to avoid OOM during recovery.
let watch_rx = log_sync_manager
.log_fetcher
.start_watch(latest_block_number, &executor_clone);
let recover_rx = log_sync_manager.log_fetcher.start_recover(
start_block_number,
// -1 so the recover and watch ranges do not overlap.
latest_block_number.wrapping_sub(1),
&executor_clone,
Duration::from_millis(log_sync_manager.config.recover_query_delay),
);
log_sync_manager.handle_data(recover_rx).await?;
// Syncing `watch_rx` is supposed to block forever.
log_sync_manager.handle_data(watch_rx).await?;
Ok(())
},
)
.map(|_| ()),
"log_sync",
);
Ok(event_send_cloned)
}
async fn put_tx(&mut self, tx: Transaction) -> bool {
// We call this after process chain reorg, so the sequence number should match.
match tx.seq.cmp(&self.next_tx_seq) {
std::cmp::Ordering::Less => true,
std::cmp::Ordering::Equal => {
debug!("log entry sync get entry: {:?}", tx);
self.put_tx_inner(tx).await
}
std::cmp::Ordering::Greater => {
error!(
"Unexpected transaction seq: next={} get={}",
self.next_tx_seq, tx.seq
);
false
}
}
}
/// `tx_seq` is the first reverted tx seq.
async fn process_reverted(&mut self, tx_seq: u64) {
warn!("revert for chain reorg: seq={}", tx_seq);
{
let store = self.store.read().await;
for seq in tx_seq..self.next_tx_seq {
if matches!(store.check_tx_completed(seq), Ok(true)) {
if let Ok(Some(tx)) = store.get_tx_by_seq_number(seq) {
// TODO(zz): Skip reading the rear padding data?
if let Ok(Some(data)) =
store.get_chunks_by_tx_and_index_range(seq, 0, tx.num_entries())
{
if !self
.data_cache
.add_data(tx.data_merkle_root, seq, data.data)
{
// TODO(zz): Data too large. Save to disk?
warn!("large reverted data dropped for tx={:?}", tx);
}
}
}
}
}
}
let _ = self.event_send.send(LogSyncEvent::ReorgDetected { tx_seq });
// TODO(zz): `wrapping_sub` here is a hack to handle the case of tx_seq=0.
if let Err(e) = self.store.write().await.revert_to(tx_seq.wrapping_sub(1)) {
error!("revert_to fails: e={:?}", e);
return;
}
self.next_tx_seq = tx_seq;
let _ = self.event_send.send(LogSyncEvent::Reverted { tx_seq });
}
async fn handle_data(&mut self, mut rx: UnboundedReceiver<LogFetchProgress>) -> Result<()> {
while let Some(data) = rx.recv().await {
trace!("handle_data: data={:?}", data);
match data {
LogFetchProgress::SyncedBlock(progress) => {
match self
.log_fetcher
.provider()
.get_block(
progress
.0
.saturating_sub(self.config.confirmation_block_count),
)
.await
{
Ok(Some(b)) => {
if let (Some(block_number), Some(block_hash)) = (b.number, b.hash) {
self.store
.write()
.await
.put_sync_progress((block_number.as_u64(), block_hash))?;
}
}
e => {
error!("log put progress check rpc fails, e={:?}", e);
}
}
}
LogFetchProgress::Transaction(tx) => {
if !self.put_tx(tx.clone()).await {
// Unexpected error.
error!("log sync write error");
break;
}
if let Err(e) = self.event_send.send(LogSyncEvent::TxSynced { tx }) {
error!("log sync broadcast error, error={:?}", e);
break;
}
}
LogFetchProgress::Reverted(reverted) => {
self.process_reverted(reverted).await;
}
}
}
Ok(())
}
async fn put_tx_inner(&mut self, tx: Transaction) -> bool {
if let Err(e) = self.store.write().await.put_tx(tx.clone()) {
error!("put_tx error: e={:?}", e);
false
} else {
if let Some(data) = self.data_cache.pop_data(&tx.data_merkle_root) {
let mut store = self.store.write().await;
// We are holding a mutable reference of LogSyncManager, so no chain reorg is
// possible after put_tx.
if let Err(e) = store
.put_chunks_with_tx_hash(
tx.seq,
tx.hash(),
ChunkArray {
data,
start_index: 0,
},
)
.and_then(|_| store.finalize_tx_with_hash(tx.seq, tx.hash()))
{
error!("put_tx data error: e={:?}", e);
return false;
}
}
self.data_cache.garbage_collect(self.next_tx_seq);
self.next_tx_seq += 1;
true
}
}
}
async fn run_and_log<R, E>(
mut on_error: impl FnMut(),
f: impl Future<Output = std::result::Result<R, E>> + Send,
) -> Option<R>
where
E: Debug,
{
match f.await {
Err(e) => {
error!("log sync failure: e={:?}", e);
on_error();
None
}
Ok(r) => Some(r),
}
}
async fn repeat_run_and_log<R, E, F>(f: impl Fn() -> F) -> R
where
E: Debug,
F: Future<Output = std::result::Result<R, E>> + Send,
{
loop {
if let Some(r) = run_and_log(|| {}, f()).await {
break r;
}
tokio::time::sleep(Duration::from_millis(RETRY_WAIT_MS)).await;
}
}
pub(crate) mod config;
mod data_cache;
mod log_entry_fetcher;
mod log_query;

22
node/miner/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
[package]
name = "miner"
version = "0.1.0"
edition = "2021"
[dependencies]
network = { path = "../network" }
storage = { path = "../storage" }
zgs_spec = { path = "../../common/spec" }
zgs_seal = { path = "../../common/zgs_seal" }
task_executor = { path = "../../common/task_executor" }
contract-interface = { path = "../../common/contract-interface" }
ethereum-types = "0.14"
tokio = { version = "1.19.2", features = ["full"] }
tracing = "0.1.35"
blake2 = "0.10"
tiny-keccak = {version="2.0",features = ["keccak"]}
rand = "^0.8"
ethers = "^2"
lazy_static = "1.4"
async-trait = "0.1.56"
shared_types = { path = "../shared_types" }

53
node/miner/src/config.rs Normal file
View File

@ -0,0 +1,53 @@
use ethereum_types::{Address, H256};
use ethers::core::k256::SecretKey;
use ethers::middleware::SignerMiddleware;
use ethers::providers::Http;
use ethers::providers::Middleware;
use ethers::providers::Provider;
use ethers::signers::LocalWallet;
use ethers::signers::Signer;
pub struct MinerConfig {
pub(crate) miner_id: H256,
pub(crate) miner_key: H256,
pub(crate) rpc_endpoint_url: String,
pub(crate) mine_address: Address,
pub(crate) flow_address: Address,
}
pub type MineServiceMiddleware = SignerMiddleware<Provider<Http>, LocalWallet>;
impl MinerConfig {
pub fn new(
miner_id: Option<H256>,
miner_key: Option<H256>,
rpc_endpoint_url: String,
mine_address: Address,
flow_address: Address,
) -> Option<MinerConfig> {
match (miner_id, miner_key) {
(Some(miner_id), Some(miner_key)) => Some(MinerConfig {
miner_id,
miner_key,
rpc_endpoint_url,
mine_address,
flow_address,
}),
_ => None,
}
}
pub(crate) async fn make_provider(&self) -> Result<MineServiceMiddleware, String> {
let provider = Provider::<Http>::try_from(&self.rpc_endpoint_url)
.map_err(|e| format!("Can not parse blockchain endpoint: {:?}", e))?;
let chain_id = provider
.get_chainid()
.await
.map_err(|e| format!("Unable to get chain_id: {:?}", e))?;
let secret_key = SecretKey::from_bytes(self.miner_key.as_ref().into())
.map_err(|e| format!("Cannot parse private key: {:?}", e))?;
let signer = LocalWallet::from(secret_key).with_chain_id(chain_id.as_u64());
let middleware = SignerMiddleware::new(provider, signer);
Ok(middleware)
}
}

19
node/miner/src/lib.rs Normal file
View File

@ -0,0 +1,19 @@
#[macro_use]
extern crate tracing;
extern crate contract_interface;
#[macro_use]
extern crate lazy_static;
mod config;
mod loader;
mod mine;
pub mod pora;
mod sealer;
mod service;
mod submitter;
mod watcher;
pub use config::MinerConfig;
pub use loader::PoraLoader;
pub use mine::CustomMineRange;
pub use service::{MineService, MinerMessage};

20
node/miner/src/loader.rs Normal file
View File

@ -0,0 +1,20 @@
use async_trait::async_trait;
use std::sync::Arc;
use storage::log_store::{MineLoadChunk, Store};
use tokio::sync::RwLock;
#[async_trait]
pub trait PoraLoader: Send + Sync {
async fn load_sealed_data(&self, index: u64) -> Option<MineLoadChunk>;
}
#[async_trait]
impl PoraLoader for Arc<RwLock<dyn Store>> {
async fn load_sealed_data(&self, chunk_index: u64) -> Option<MineLoadChunk> {
let store = &*self.read().await;
match store.flow().load_sealed_data(chunk_index) {
Ok(Some(chunk)) => Some(chunk),
_ => None,
}
}
}

173
node/miner/src/mine.rs Normal file
View File

@ -0,0 +1,173 @@
use contract_interface::zgs_flow::MineContext;
use ethereum_types::{H256, U256};
use rand::{self, Rng};
use task_executor::TaskExecutor;
use tokio::sync::{broadcast, mpsc};
use zgs_spec::{SECTORS_PER_LOAD, SECTORS_PER_MAX_MINING_RANGE, SECTORS_PER_PRICING};
use crate::{
pora::{AnswerWithoutProof, Miner},
watcher::MineContextMessage,
MinerConfig, MinerMessage, PoraLoader,
};
use std::sync::Arc;
pub struct PoraService {
mine_context_receiver: mpsc::UnboundedReceiver<MineContextMessage>,
mine_answer_sender: mpsc::UnboundedSender<AnswerWithoutProof>,
msg_recv: broadcast::Receiver<MinerMessage>,
loader: Arc<dyn PoraLoader>,
puzzle: Option<PoraPuzzle>,
mine_range: CustomMineRange,
miner_id: H256,
}
struct PoraPuzzle {
context: MineContext,
target_quality: U256,
}
#[derive(Clone, Copy, Debug, Default)]
pub struct CustomMineRange {
start_position: Option<u64>,
end_position: Option<u64>,
}
impl CustomMineRange {
#[inline]
fn to_valid_range(self, context: &MineContext) -> Option<(u64, u64)> {
let self_start_position = self.start_position?;
let self_end_position = self.end_position?;
if self_start_position >= self_end_position {
return None;
}
let minable_length =
(context.flow_length.as_u64() / SECTORS_PER_LOAD as u64) * SECTORS_PER_LOAD as u64;
let mining_length = std::cmp::min(minable_length, SECTORS_PER_MAX_MINING_RANGE as u64);
let start_position = std::cmp::min(self_start_position, minable_length - mining_length);
let start_position =
(start_position / SECTORS_PER_PRICING as u64) * SECTORS_PER_PRICING as u64;
Some((start_position, mining_length))
}
#[inline]
pub(crate) fn is_covered(&self, recall_position: u64) -> Option<bool> {
let self_start_position = self.start_position?;
let self_end_position = self.end_position?;
if self.start_position >= self.end_position {
return Some(false);
}
Some(
self_start_position <= recall_position + SECTORS_PER_LOAD as u64
|| self_end_position > recall_position,
)
}
}
impl PoraService {
pub fn spawn(
executor: TaskExecutor,
msg_recv: broadcast::Receiver<MinerMessage>,
mine_context_receiver: mpsc::UnboundedReceiver<MineContextMessage>,
loader: Arc<dyn PoraLoader>,
config: &MinerConfig,
) -> mpsc::UnboundedReceiver<AnswerWithoutProof> {
let (mine_answer_sender, mine_answer_receiver) =
mpsc::unbounded_channel::<AnswerWithoutProof>();
let mine_range = CustomMineRange {
start_position: Some(0),
end_position: Some(u64::MAX),
};
let pora = PoraService {
mine_context_receiver,
mine_answer_sender,
msg_recv,
puzzle: None,
mine_range,
miner_id: config.miner_id,
loader,
};
executor.spawn(async move { Box::pin(pora.start()).await }, "pora_master");
mine_answer_receiver
}
async fn start(mut self) {
let mut mining_enabled = true;
let mut channel_opened = true;
loop {
tokio::select! {
biased;
v = self.msg_recv.recv(), if channel_opened => {
trace!("PoraService receives msg={:?}", v);
match v {
Ok(MinerMessage::ToggleMining(enable)) => {
info!("Toggle mining: {}", if enable { "on" } else { "off" });
mining_enabled = enable;
}
Ok(MinerMessage::SetStartPosition(pos)) => {
info!("Change start position to: {:?}", pos);
self.mine_range.start_position = pos;
}
Ok(MinerMessage::SetEndPosition(pos)) => {
info!("Change end position to: {:?}", pos);
self.mine_range.end_position = pos;
}
Err(broadcast::error::RecvError::Closed)=>{
warn!("Unexpected: Mine service config channel closed.");
channel_opened = false;
}
Err(_)=>{
}
}
}
maybe_msg = self.mine_context_receiver.recv() => {
trace!("PoraService receives context={:?}", maybe_msg);
if let Some(msg) = maybe_msg {
debug!("Update mine service: {:?}", msg);
self.puzzle = msg.map(|(context, target_quality)| PoraPuzzle {
context, target_quality
});
}
}
_ = async {}, if mining_enabled && self.as_miner().is_some() => {
let nonce = H256(rand::thread_rng().gen());
let miner = self.as_miner().unwrap();
if let Some(answer) = miner.iteration(nonce).await{
debug!("Hit Pora answer {:?}", answer);
if self.mine_answer_sender.send(answer).is_err() {
warn!("Mine submitter channel closed");
}
}
}
}
}
}
#[inline]
fn as_miner(&self) -> Option<Miner> {
match self.puzzle.as_ref() {
Some(puzzle) => self.mine_range.to_valid_range(&puzzle.context).map(
|(start_position, mining_length)| Miner {
start_position,
mining_length,
miner_id: &self.miner_id,
custom_mine_range: &self.mine_range,
context: &puzzle.context,
target_quality: &puzzle.target_quality,
loader: &*self.loader,
},
),
_ => None,
}
}
}

165
node/miner/src/pora.rs Normal file
View File

@ -0,0 +1,165 @@
use crate::{CustomMineRange, PoraLoader};
use blake2::{Blake2b512, Digest};
use contract_interface::zgs_flow::MineContext;
use ethereum_types::{H256, U256};
use zgs_spec::{BYTES_PER_SCRATCHPAD, BYTES_PER_SEAL, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
use storage::log_store::MineLoadChunk;
use tiny_keccak::{Hasher, Keccak};
pub const BLAKE2B_OUTPUT_BYTES: usize = 64;
pub const KECCAK256_OUTPUT_BYTES: usize = 32;
fn keccak(input: impl AsRef<[u8]>) -> [u8; KECCAK256_OUTPUT_BYTES] {
let mut hasher = Keccak::v256();
let mut output = [0u8; 32];
hasher.update(input.as_ref());
hasher.finalize(&mut output);
output
}
pub(crate) struct Miner<'a> {
pub start_position: u64,
pub mining_length: u64,
pub miner_id: &'a H256,
pub context: &'a MineContext,
pub target_quality: &'a U256,
pub loader: &'a dyn PoraLoader,
pub custom_mine_range: &'a CustomMineRange,
}
#[derive(Debug)]
pub struct AnswerWithoutProof {
pub context_digest: H256,
pub context_flow_root: H256,
pub nonce: H256,
pub miner_id: H256,
pub start_position: u64,
pub mining_length: u64,
pub recall_position: u64,
pub seal_offset: usize,
pub sealed_data: [u8; BYTES_PER_SEAL],
}
impl<'a> Miner<'a> {
pub async fn iteration(&self, nonce: H256) -> Option<AnswerWithoutProof> {
let (scratch_pad, recall_seed) = self.make_scratch_pad(&nonce);
let (_, recall_offset) = U256::from_big_endian(&recall_seed)
.div_mod(U256::from((self.mining_length as usize) / SECTORS_PER_LOAD));
let recall_offset = recall_offset.as_u64();
if !self
.custom_mine_range
.is_covered(self.start_position + recall_offset * SECTORS_PER_LOAD as u64)
.unwrap()
{
trace!(
"recall offset not in range: recall_offset={}, range={:?}",
recall_offset,
self.custom_mine_range
);
return None;
}
let MineLoadChunk {
loaded_chunk,
avalibilities,
} = self
.loader
.load_sealed_data(self.start_position / SECTORS_PER_LOAD as u64 + recall_offset)
.await?;
let scratch_pad: [[u8; BYTES_PER_SEAL]; BYTES_PER_SCRATCHPAD / BYTES_PER_SEAL] =
unsafe { std::mem::transmute(scratch_pad) };
for ((idx, mut sealed_data), scratch_pad) in loaded_chunk
.into_iter()
.enumerate()
.zip(scratch_pad.iter().cycle())
.zip(avalibilities.into_iter())
.filter_map(|(data, avaliable)| avaliable.then_some(data))
{
// Rust can optimize this loop well.
for (x, y) in sealed_data.iter_mut().zip(scratch_pad.iter()) {
*x ^= y;
}
let quality = self.pora(idx, &nonce, &sealed_data);
if &quality <= self.target_quality {
debug!("Find a PoRA valid answer, quality: {}", quality);
// Undo mix data when find a valid solition
for (x, y) in sealed_data.iter_mut().zip(scratch_pad.iter()) {
*x ^= y;
}
return Some(AnswerWithoutProof {
context_digest: H256::from(self.context.digest),
context_flow_root: self.context.flow_root.into(),
nonce,
miner_id: *self.miner_id,
start_position: self.start_position,
mining_length: self.mining_length,
recall_position: self.start_position
+ recall_offset * SECTORS_PER_LOAD as u64
+ idx as u64 * SECTORS_PER_SEAL as u64,
seal_offset: idx,
sealed_data,
});
}
}
None
}
fn make_scratch_pad(
&self,
nonce: &H256,
) -> ([u8; BYTES_PER_SCRATCHPAD], [u8; KECCAK256_OUTPUT_BYTES]) {
let mut digest: [u8; BLAKE2B_OUTPUT_BYTES] = {
let mut hasher = Blake2b512::new();
hasher.update(self.miner_id);
hasher.update(nonce);
hasher.update(self.context.digest);
hasher.update([0u8; 24]);
hasher.update(self.start_position.to_be_bytes());
hasher.update([0u8; 24]);
hasher.update(self.mining_length.to_be_bytes());
hasher.finalize().into()
};
let mut scratch_pad =
[[0u8; BLAKE2B_OUTPUT_BYTES]; BYTES_PER_SCRATCHPAD / BLAKE2B_OUTPUT_BYTES];
for scratch_pad_cell in scratch_pad.iter_mut() {
digest = Blake2b512::new().chain_update(digest).finalize().into();
*scratch_pad_cell = digest;
}
let scratch_pad: [u8; BYTES_PER_SCRATCHPAD] = unsafe { std::mem::transmute(scratch_pad) };
let recall_seed: [u8; KECCAK256_OUTPUT_BYTES] = keccak(digest);
(scratch_pad, recall_seed)
}
#[inline]
fn pora(&self, seal_index: usize, nonce: &H256, mixed_data: &[u8; BYTES_PER_SEAL]) -> U256 {
let mut hasher = Blake2b512::new();
hasher.update([0u8; 24]);
hasher.update(seal_index.to_be_bytes());
hasher.update(self.miner_id);
hasher.update(nonce);
hasher.update(self.context.digest);
hasher.update([0u8; 24]);
hasher.update(self.start_position.to_be_bytes());
hasher.update([0u8; 24]);
hasher.update(self.mining_length.to_be_bytes());
hasher.update([0u8; 64]);
hasher.update(mixed_data);
let digest = hasher.finalize();
U256::from_big_endian(&digest[0..32])
}
}

213
node/miner/src/sealer.rs Normal file
View File

@ -0,0 +1,213 @@
use std::{collections::BTreeMap, sync::Arc};
use ethereum_types::H256;
use tokio::{
sync::RwLock,
time::{sleep, Duration, Instant},
};
use contract_interface::{EpochRangeWithContextDigest, ZgsFlow};
use zgs_spec::SECTORS_PER_SEAL;
use storage::{
error::Result,
log_store::{SealAnswer, SealTask, Store},
};
use task_executor::TaskExecutor;
use crate::config::{MineServiceMiddleware, MinerConfig};
const DB_QUERY_PERIOD_ON_NO_TASK: u64 = 1;
const DB_QUERY_PERIOD_ON_ERROR: u64 = 5;
const CHAIN_STATUS_QUERY_PERIOD: u64 = 5;
pub struct Sealer {
flow_contract: ZgsFlow<MineServiceMiddleware>,
store: Arc<RwLock<dyn Store>>,
context_cache: BTreeMap<u128, EpochRangeWithContextDigest>,
last_context_flow_length: u64,
miner_id: H256,
}
impl Sealer {
pub fn spawn(
executor: TaskExecutor,
provider: Arc<MineServiceMiddleware>,
store: Arc<RwLock<dyn Store>>,
config: &MinerConfig,
) {
let flow_contract = ZgsFlow::new(config.flow_address, provider);
let sealer = Sealer {
flow_contract,
store,
context_cache: Default::default(),
last_context_flow_length: 0,
miner_id: config.miner_id,
};
executor.spawn(async move { Box::pin(sealer.start()).await }, "data_sealer");
}
async fn start(mut self) {
let db_checker_throttle = sleep(Duration::from_secs(0));
tokio::pin!(db_checker_throttle);
let contract_checker_throttle = sleep(Duration::from_secs(0));
tokio::pin!(contract_checker_throttle);
loop {
tokio::select! {
biased;
() = &mut contract_checker_throttle, if !contract_checker_throttle.is_elapsed() => {
}
() = &mut db_checker_throttle, if !db_checker_throttle.is_elapsed() => {
}
_ = async {}, if contract_checker_throttle.is_elapsed() => {
if let Err(err) = self.update_flow_length().await{
warn!("Fetch onchain context failed {:?}", err);
}
contract_checker_throttle.as_mut().reset(Instant::now() + Duration::from_secs(CHAIN_STATUS_QUERY_PERIOD));
}
_ = async {}, if db_checker_throttle.is_elapsed() => {
match self.seal_iteration().await {
Ok(true) => {},
Ok(false) => {db_checker_throttle.as_mut().reset(Instant::now() + Duration::from_secs(DB_QUERY_PERIOD_ON_NO_TASK));}
Err(err) => {
warn!("Seal iteration failed {:?}", err);
db_checker_throttle.as_mut().reset(Instant::now() + Duration::from_secs(DB_QUERY_PERIOD_ON_ERROR));
}
}
}
}
}
}
async fn update_flow_length(&mut self) -> Result<()> {
let recent_context = self.flow_contract.make_context_with_result().call().await?;
debug!(target: "seal", "Recent context is {:?}", recent_context);
let recent_flow_length = recent_context.flow_length.as_u64();
if self.last_context_flow_length < recent_flow_length {
let epoch_range = self
.flow_contract
.get_epoch_range(recent_context.digest)
.call()
.await?;
self.context_cache.insert(
epoch_range.start,
EpochRangeWithContextDigest {
start: epoch_range.start,
end: epoch_range.end,
digest: recent_context.digest,
},
);
self.last_context_flow_length = recent_flow_length;
info!(target: "seal", "Update sealable flow length: {}", recent_flow_length)
}
Ok(())
}
async fn fetch_context(&mut self, seal_index: u64) -> Result<Option<(H256, u64)>> {
let last_entry = ((seal_index as usize + 1) * SECTORS_PER_SEAL - 1) as u128;
if self.last_context_flow_length <= last_entry as u64 {
return Ok(None);
}
if let Some((_, context)) = self.context_cache.range(..=last_entry).next_back() {
if context.start <= last_entry && context.end > last_entry {
return Ok(Some((
H256(context.digest),
context.end as u64 / SECTORS_PER_SEAL as u64,
)));
}
}
let context = match self
.flow_contract
.query_context_at_position(last_entry)
.call()
.await
{
Ok(context) => context,
Err(err) => {
info!("Error when fetch entries {:?}", err);
return Ok(None);
}
};
info!(
target:"seal", "Fetch new context: range {} -> {}",
context.start, context.end
);
self.context_cache.insert(context.start, context.clone());
Ok(Some((
H256(context.digest),
context.end as u64 / SECTORS_PER_SEAL as u64,
)))
}
async fn fetch_task(&self) -> Result<Option<Vec<SealTask>>> {
let seal_index_max = self.last_context_flow_length as usize / SECTORS_PER_SEAL;
self.store
.read()
.await
.flow()
.pull_seal_chunk(seal_index_max)
}
async fn submit_answer(&self, answers: Vec<SealAnswer>) -> Result<()> {
self.store
.write()
.await
.flow_mut()
.submit_seal_result(answers)
}
async fn seal_iteration(&mut self) -> Result<bool> {
let tasks = match self.fetch_task().await? {
Some(tasks) if !tasks.is_empty() => tasks,
_ => {
return Ok(false);
}
};
debug!(
"Get seal tasks at seal index {:?}",
tasks.iter().map(|x| x.seal_index).collect::<Vec<u64>>()
);
let mut answers = Vec::with_capacity(tasks.len());
for task in tasks {
let (context_digest, end_seal) =
if let Some(context) = self.fetch_context(task.seal_index).await? {
context
} else {
trace!(target: "seal", "Index {} is not ready for seal", task.seal_index);
continue;
};
let mut data = task.non_sealed_data;
zgs_seal::seal(
&mut data,
&self.miner_id,
&context_digest,
task.seal_index * SECTORS_PER_SEAL as u64,
);
answers.push(SealAnswer {
seal_index: task.seal_index,
version: task.version,
sealed_data: data,
miner_id: self.miner_id,
seal_context: context_digest,
context_end_seal: end_seal,
});
}
self.submit_answer(answers).await?;
Ok(true)
}
}

62
node/miner/src/service.rs Normal file
View File

@ -0,0 +1,62 @@
use crate::sealer::Sealer;
use crate::submitter::Submitter;
use crate::{config::MinerConfig, mine::PoraService, watcher::MineContextWatcher};
use network::NetworkMessage;
use std::sync::Arc;
use storage::log_store::Store;
use tokio::sync::mpsc;
use tokio::sync::{broadcast, RwLock};
#[derive(Clone, Debug)]
pub enum MinerMessage {
/// Enable / Disable Mining
ToggleMining(bool),
/// Change mining range
SetStartPosition(Option<u64>),
SetEndPosition(Option<u64>),
}
pub struct MineService;
impl MineService {
pub async fn spawn(
executor: task_executor::TaskExecutor,
_network_send: mpsc::UnboundedSender<NetworkMessage>,
config: MinerConfig,
store: Arc<RwLock<dyn Store>>,
) -> Result<broadcast::Sender<MinerMessage>, String> {
let provider = Arc::new(config.make_provider().await?);
let (msg_send, msg_recv) = broadcast::channel(1024);
let mine_context_receiver = MineContextWatcher::spawn(
executor.clone(),
msg_recv.resubscribe(),
provider.clone(),
&config,
);
let mine_answer_receiver = PoraService::spawn(
executor.clone(),
msg_recv.resubscribe(),
mine_context_receiver,
Arc::new(store.clone()),
&config,
);
Submitter::spawn(
executor.clone(),
mine_answer_receiver,
provider.clone(),
store.clone(),
&config,
);
Sealer::spawn(executor, provider, store, &config);
debug!("Starting miner service");
Ok(msg_send)
}
}

128
node/miner/src/submitter.rs Normal file
View File

@ -0,0 +1,128 @@
use contract_interface::PoraAnswer;
use contract_interface::{ZgsFlow, PoraMine};
use ethers::providers::PendingTransaction;
use shared_types::FlowRangeProof;
use std::sync::Arc;
use storage::log_store::Store;
use task_executor::TaskExecutor;
use tokio::sync::{mpsc, RwLock};
use crate::config::{MineServiceMiddleware, MinerConfig};
use crate::pora::AnswerWithoutProof;
use zgs_spec::SECTORS_PER_SEAL;
const SUBMISSION_RETIES: usize = 3;
pub struct Submitter {
mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>,
mine_contract: PoraMine<MineServiceMiddleware>,
flow_contract: ZgsFlow<MineServiceMiddleware>,
store: Arc<RwLock<dyn Store>>,
}
impl Submitter {
pub fn spawn(
executor: TaskExecutor,
mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>,
provider: Arc<MineServiceMiddleware>,
store: Arc<RwLock<dyn Store>>,
config: &MinerConfig,
) {
let mine_contract = PoraMine::new(config.mine_address, provider.clone());
let flow_contract = ZgsFlow::new(config.flow_address, provider);
let submitter = Submitter {
mine_answer_receiver,
mine_contract,
flow_contract,
store,
};
executor.spawn(
async move { Box::pin(submitter.start()).await },
"mine_answer_submitter",
);
}
async fn start(mut self) {
loop {
match self.mine_answer_receiver.recv().await {
Some(answer) => {
if let Err(e) = self.submit_answer(answer).await {
warn!(e)
}
}
None => {
warn!("Mine submitter stopped because mine answer channel is closed.");
break;
}
};
}
}
async fn submit_answer(&mut self, mine_answer: AnswerWithoutProof) -> Result<(), String> {
debug!("submit answer: {:?}", mine_answer);
let sealed_context_digest = self
.flow_contract
.query_context_at_position(
(mine_answer.recall_position + SECTORS_PER_SEAL as u64 - 1) as u128,
)
.call()
.await
.map_err(|e| format!("Failed to fetch sealed contest digest: {:?}", e))?;
debug!("Fetch sealed context: {:?}", sealed_context_digest);
let flow_proof = self
.store
.read()
.await
.get_proof_at_root(
&mine_answer.context_flow_root,
mine_answer.recall_position,
SECTORS_PER_SEAL as u64,
)
.map_err(|e| e.to_string())?;
let answer = PoraAnswer {
context_digest: mine_answer.context_digest.0,
nonce: mine_answer.nonce.0,
miner_id: mine_answer.miner_id.0,
start_position: mine_answer.start_position.into(),
mine_length: mine_answer.mining_length.into(),
recall_position: mine_answer.recall_position.into(),
seal_offset: mine_answer.seal_offset.into(),
sealed_context_digest: sealed_context_digest.digest, // TODO(kevin): wait for implementation of data sealing.
sealed_data: unsafe { std::mem::transmute(mine_answer.sealed_data) },
merkle_proof: flow_proof_to_pora_merkle_proof(flow_proof),
};
trace!("submit_answer: answer={:?}", answer);
let submission_call = self.mine_contract.submit(answer).legacy();
let pending_transaction: PendingTransaction<'_, _> = submission_call
.send()
.await
.map_err(|e| format!("Fail to send mine answer transaction: {:?}", e))?;
let receipt = pending_transaction
.retries(SUBMISSION_RETIES)
.await
.map_err(|e| format!("Fail to execute mine answer transaction: {:?}", e))?
.ok_or(format!(
"Mine answer transaction dropped after {} retires",
SUBMISSION_RETIES
))?;
info!("Submit PoRA sucess");
debug!("Receipt: {:?}", receipt);
Ok(())
}
}
// TODO: The conversion will be simpler if we optimize range proof structure.
fn flow_proof_to_pora_merkle_proof(flow_proof: FlowRangeProof) -> Vec<[u8; 32]> {
let depth_in_sealed_data = SECTORS_PER_SEAL.trailing_zeros() as usize;
let full_proof: Vec<[u8; 32]> = flow_proof.left_proof.lemma().iter().map(|h| h.0).collect();
// Exclude `item`, the nodes in the sealed data subtree, and `root`.
full_proof[depth_in_sealed_data + 1..full_proof.len() - 1].to_vec()
}

142
node/miner/src/watcher.rs Normal file
View File

@ -0,0 +1,142 @@
#![allow(unused)]
use contract_interface::{zgs_flow::MineContext, ZgsFlow, PoraMine};
use ethereum_types::{Address, H256, U256};
use ethers::{
contract::Contract,
providers::{JsonRpcClient, Middleware, Provider, StreamExt},
types::BlockId,
};
use task_executor::TaskExecutor;
use tokio::{
sync::{broadcast, mpsc},
time::{sleep, Instant, Sleep},
try_join,
};
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use std::{ops::DerefMut, str::FromStr};
use crate::{config::MineServiceMiddleware, MinerConfig, MinerMessage};
pub type MineContextMessage = Option<(MineContext, U256)>;
lazy_static! {
pub static ref EMPTY_HASH: H256 =
H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap();
}
pub struct MineContextWatcher {
provider: Arc<MineServiceMiddleware>,
flow_contract: ZgsFlow<MineServiceMiddleware>,
mine_contract: PoraMine<MineServiceMiddleware>,
mine_context_sender: mpsc::UnboundedSender<MineContextMessage>,
last_report: MineContextMessage,
msg_recv: broadcast::Receiver<MinerMessage>,
}
impl MineContextWatcher {
pub fn spawn(
executor: TaskExecutor,
msg_recv: broadcast::Receiver<MinerMessage>,
provider: Arc<MineServiceMiddleware>,
config: &MinerConfig,
) -> mpsc::UnboundedReceiver<MineContextMessage> {
let provider = provider;
let mine_contract = PoraMine::new(config.mine_address, provider.clone());
let flow_contract = ZgsFlow::new(config.flow_address, provider.clone());
let (mine_context_sender, mine_context_receiver) =
mpsc::unbounded_channel::<MineContextMessage>();
let watcher = MineContextWatcher {
provider,
flow_contract,
mine_contract,
mine_context_sender,
msg_recv,
last_report: None,
};
executor.spawn(
async move { Box::pin(watcher.start()).await },
"mine_context_watcher",
);
mine_context_receiver
}
async fn start(mut self) {
let mut mining_enabled = true;
let mut channel_opened = true;
let mut mining_throttle = sleep(Duration::from_secs(0));
tokio::pin!(mining_throttle);
loop {
tokio::select! {
biased;
v = self.msg_recv.recv(), if channel_opened => {
match v {
Ok(MinerMessage::ToggleMining(enable)) => {
mining_enabled = enable;
}
Err(broadcast::error::RecvError::Closed) => {
channel_opened = false;
}
_ => {}
}
}
() = &mut mining_throttle, if !mining_throttle.is_elapsed() => {
}
_ = async {}, if mining_enabled && mining_throttle.is_elapsed() => {
mining_throttle.as_mut().reset(Instant::now() + Duration::from_secs(1));
if let Err(err) = self.query_recent_context().await {
warn!(err);
}
}
}
}
}
async fn query_recent_context(&mut self) -> Result<(), String> {
// let mut watcher = self
// .provider
// .watch_blocks()
// .await
// .expect("should success")
// .stream();
// watcher.next().await
let context_call = self.flow_contract.make_context_with_result();
let epoch_call = self.mine_contract.last_mined_epoch();
let quality_call = self.mine_contract.target_quality();
let (context, epoch, quality) =
try_join!(context_call.call(), epoch_call.call(), quality_call.call())
.map_err(|e| format!("Failed to query mining context: {:?}", e))?;
let report = if context.epoch > epoch && context.digest != EMPTY_HASH.0 {
if context.block_digest == [0; 32] {
warn!("Mine Context is not updated on time.");
None
} else {
Some((context, quality))
}
} else {
None
};
if report != self.last_report {
self.mine_context_sender
.send(report.clone())
.map_err(|e| format!("Failed to send out the most recent mine context: {:?}", e))?;
}
self.last_report = report;
Ok(())
}
}

55
node/network/Cargo.toml Normal file
View File

@ -0,0 +1,55 @@
[package]
name = "network"
version = "0.2.0"
edition = "2021"
[dependencies]
directory = { path = "../../common/directory" }
dirs = "4.0.0"
discv5 = { version = "0.1.0-beta.13", features = ["libp2p"] }
error-chain = "0.12.4"
eth2_ssz = "0.4.0"
eth2_ssz_derive = "0.3.0"
eth2_ssz_types = "0.2.1"
ethereum-types = "0.14"
fnv = "1.0.7"
futures = "0.3.21"
hashset_delay = { path = "../../common/hashset_delay" }
hex = "0.4.3"
zgs_version = { path = "../../common/zgs_version" }
lazy_static = "1.4.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
lru = "0.7.7"
parking_lot = "0.12.1"
prometheus-client = "0.16.0"
rand = "0.8.5"
regex = "1.5.6"
serde = { version = "1.0.137", features = ["derive"] }
serde_derive = "1.0.137"
sha2 = "0.10.2"
shared_types = { path = "../shared_types" }
smallvec = "1.8.0"
snap = "1.0.5"
strum = { version = "0.24.1", features = ["derive"] }
task_executor = { path = "../../common/task_executor" }
tiny-keccak = "2.0.2"
tokio = { version = "1.19.2", features = ["time", "macros"] }
tokio-io-timeout = "1.2.0"
tokio-util = { version = "0.6.10", features = ["codec", "compat", "time"] }
tracing = "0.1.35"
unsigned-varint = { version = "0.7.1", features = ["codec"] }
if-addrs = "0.10.1"
slog = "2.7.0"
igd = "0.12.1"
[dependencies.libp2p]
version = "0.45.1"
default-features = false
features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns-tokio", "tcp-tokio", "plaintext", "secp256k1"]
[dev-dependencies]
exit-future = "0.2.0"
tempfile = "3.3.0"
tracing-test = "0.2.2"
unused_port = { path = "../../common/unused_port" }
void = "1.0.2"

View File

@ -0,0 +1,183 @@
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use crate::types::GossipKind;
use crate::GossipTopic;
use tokio_util::time::delay_queue::{DelayQueue, Key};
/// Store of gossip messages that we failed to publish and will try again later. By default, all
/// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout`
/// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and
/// will overwrite the default_timeout if present.
pub struct GossipCache {
/// Expire timeouts for each topic-msg pair.
expirations: DelayQueue<(GossipTopic, Vec<u8>)>,
/// Messages cached for each topic.
topic_msgs: HashMap<GossipTopic, HashMap<Vec<u8>, Key>>,
/// Timeout for Example messages.
example: Option<Duration>,
/// Timeout for FindFile messages.
find_file: Option<Duration>,
/// Timeout for AnnounceFile.
announce_file: Option<Duration>,
}
#[derive(Default)]
pub struct GossipCacheBuilder {
default_timeout: Option<Duration>,
/// Timeout for Example messages.
example: Option<Duration>,
/// Timeout for blocks FindFile messages.
find_file: Option<Duration>,
/// Timeout for AnnounceFile messages.
announce_file: Option<Duration>,
}
#[allow(dead_code)]
impl GossipCacheBuilder {
/// By default, all timeouts all disabled. Setting a default timeout will enable all timeout
/// that are not already set.
pub fn default_timeout(mut self, timeout: Duration) -> Self {
self.default_timeout = Some(timeout);
self
}
/// Timeout for Example messages.
pub fn example_timeout(mut self, timeout: Duration) -> Self {
self.example = Some(timeout);
self
}
/// Timeout for FindFile messages.
pub fn find_file_timeout(mut self, timeout: Duration) -> Self {
self.find_file = Some(timeout);
self
}
/// Timeout for AnnounceFile messages.
pub fn announce_file_timeout(mut self, timeout: Duration) -> Self {
self.announce_file = Some(timeout);
self
}
pub fn build(self) -> GossipCache {
let GossipCacheBuilder {
default_timeout,
example,
find_file,
announce_file,
} = self;
GossipCache {
expirations: DelayQueue::default(),
topic_msgs: HashMap::default(),
example: example.or(default_timeout),
find_file: find_file.or(default_timeout),
announce_file: announce_file.or(default_timeout),
}
}
}
impl GossipCache {
/// Get a builder of a `GossipCache`. Topic kinds for which no timeout is defined will be
/// ignored if added in `insert`.
pub fn builder() -> GossipCacheBuilder {
GossipCacheBuilder::default()
}
// Insert a message to be sent later.
pub fn insert(&mut self, topic: GossipTopic, data: Vec<u8>) {
let expire_timeout = match topic.kind() {
GossipKind::Example => self.example,
GossipKind::FindFile => self.find_file,
GossipKind::AnnounceFile => self.announce_file,
};
let expire_timeout = match expire_timeout {
Some(expire_timeout) => expire_timeout,
None => return,
};
match self
.topic_msgs
.entry(topic.clone())
.or_default()
.entry(data.clone())
{
Entry::Occupied(key) => self.expirations.reset(key.get(), expire_timeout),
Entry::Vacant(entry) => {
let key = self.expirations.insert((topic, data), expire_timeout);
entry.insert(key);
}
}
}
// Get the registered messages for this topic.
pub fn retrieve(&mut self, topic: &GossipTopic) -> Option<impl Iterator<Item = Vec<u8>> + '_> {
if let Some(msgs) = self.topic_msgs.remove(topic) {
for (_, key) in msgs.iter() {
self.expirations.remove(key);
}
Some(msgs.into_keys())
} else {
None
}
}
}
impl futures::stream::Stream for GossipCache {
type Item = Result<GossipTopic, String>; // We don't care to retrieve the expired data.
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match self.expirations.poll_expired(cx) {
Poll::Ready(Some(Ok(expired))) => {
let expected_key = expired.key();
let (topic, data) = expired.into_inner();
match self.topic_msgs.get_mut(&topic) {
Some(msgs) => {
let key = msgs.remove(&data);
debug_assert_eq!(key, Some(expected_key));
if msgs.is_empty() {
// no more messages for this topic.
self.topic_msgs.remove(&topic);
}
}
None => {
#[cfg(debug_assertions)]
panic!("Topic for registered message is not present.")
}
}
Poll::Ready(Some(Ok(topic)))
}
Poll::Ready(Some(Err(x))) => Poll::Ready(Some(Err(x.to_string()))),
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
}
#[cfg(test)]
mod tests {
use crate::types::GossipKind;
use super::*;
use futures::stream::StreamExt;
#[tokio::test]
async fn test_stream() {
let mut cache = GossipCache::builder()
.default_timeout(Duration::from_millis(300))
.build();
let test_topic =
GossipTopic::new(GossipKind::Example, crate::types::GossipEncoding::SSZSnappy);
cache.insert(test_topic, vec![]);
tokio::time::sleep(Duration::from_millis(300)).await;
while cache.next().await.is_some() {}
assert!(cache.expirations.is_empty());
assert!(cache.topic_msgs.is_empty());
}
}

View File

@ -0,0 +1,354 @@
// use crate::error;
use libp2p::gossipsub::PeerScoreThresholds;
// use std::cmp::max;
// use std::collections::HashMap;
// use std::time::Duration;
// const MAX_IN_MESH_SCORE: f64 = 10.0;
// const MAX_FIRST_MESSAGE_DELIVERIES_SCORE: f64 = 40.0;
// const BEACON_BLOCK_WEIGHT: f64 = 0.5;
// const BEACON_AGGREGATE_PROOF_WEIGHT: f64 = 0.5;
// const VOLUNTARY_EXIT_WEIGHT: f64 = 0.05;
// const PROPOSER_SLASHING_WEIGHT: f64 = 0.05;
// const ATTESTER_SLASHING_WEIGHT: f64 = 0.05;
/// The time window (seconds) that we expect messages to be forwarded to us in the mesh.
// const MESH_MESSAGE_DELIVERIES_WINDOW: u64 = 2;
// Const as this is used in the peer manager to prevent gossip from disconnecting peers.
pub const GREYLIST_THRESHOLD: f64 = -16000.0;
/// Builds the peer score thresholds.
pub fn lighthouse_gossip_thresholds() -> PeerScoreThresholds {
PeerScoreThresholds {
gossip_threshold: -4000.0,
publish_threshold: -8000.0,
graylist_threshold: GREYLIST_THRESHOLD,
accept_px_threshold: 100.0,
opportunistic_graft_threshold: 5.0,
}
}
// pub struct PeerScoreSettings {
// // slot: Duration,
// epoch: Duration,
// // beacon_attestation_subnet_weight: f64,
// max_positive_score: f64,
// decay_interval: Duration,
// decay_to_zero: f64,
// // mesh_n: usize,
// // max_committees_per_slot: usize,
// // target_committee_size: usize,
// // target_aggregators_per_committee: usize,
// attestation_subnet_count: u64,
// }
// impl PeerScoreSettings {
// pub fn new(chain_spec: &ChainSpec) -> PeerScoreSettings {
// let slot = Duration::from_secs(chain_spec.seconds_per_slot);
// let beacon_attestation_subnet_weight = 1.0 / chain_spec.attestation_subnet_count as f64;
// let max_positive_score = (MAX_IN_MESH_SCORE + MAX_FIRST_MESSAGE_DELIVERIES_SCORE)
// * (BEACON_BLOCK_WEIGHT
// + BEACON_AGGREGATE_PROOF_WEIGHT
// + beacon_attestation_subnet_weight * chain_spec.attestation_subnet_count as f64
// + VOLUNTARY_EXIT_WEIGHT
// + PROPOSER_SLASHING_WEIGHT
// + ATTESTER_SLASHING_WEIGHT);
// PeerScoreSettings {
// // slot,
// // epoch: slot * TSpec::slots_per_epoch() as u32,
// epoch: slot * 32 as u32,
// // beacon_attestation_subnet_weight,
// max_positive_score,
// decay_interval: max(Duration::from_secs(1), slot),
// decay_to_zero: 0.01,
// // mesh_n: gs_config.mesh_n(),
// // max_committees_per_slot: chain_spec.max_committees_per_slot,
// // target_committee_size: chain_spec.target_committee_size,
// // target_aggregators_per_committee: chain_spec.target_aggregators_per_committee as usize,
// attestation_subnet_count: chain_spec.attestation_subnet_count,
// }
// }
// pub fn get_peer_score_params(
// &self,
// thresholds: &PeerScoreThresholds,
// ) -> error::Result<PeerScoreParams> {
// let mut params = PeerScoreParams {
// decay_interval: self.decay_interval,
// decay_to_zero: self.decay_to_zero,
// retain_score: self.epoch * 100,
// app_specific_weight: 1.0,
// ip_colocation_factor_threshold: 8.0, // Allow up to 8 nodes per IP
// behaviour_penalty_threshold: 6.0,
// behaviour_penalty_decay: self.score_parameter_decay(self.epoch * 10),
// ..Default::default()
// };
// let target_value = Self::decay_convergence(
// params.behaviour_penalty_decay,
// // 10.0 / TSpec::slots_per_epoch() as f64,
// 10.0 / 32 as f64,
// ) - params.behaviour_penalty_threshold;
// params.behaviour_penalty_weight = thresholds.gossip_threshold / target_value.powi(2);
// params.topic_score_cap = self.max_positive_score * 0.5;
// params.ip_colocation_factor_weight = -params.topic_score_cap;
// params.topics = HashMap::new();
// // let get_hash = |kind: GossipKind| -> TopicHash {
// // let topic: Topic =
// // GossipTopic::new(kind, GossipEncoding::default()).into();
// // topic.hash()
// // };
// // //first all fixed topics
// // params.topics.insert(
// // get_hash(GossipKind::VoluntaryExit),
// // Self::get_topic_params(
// // self,
// // VOLUNTARY_EXIT_WEIGHT,
// // // 4.0 / TSpec::slots_per_epoch() as f64,
// // 4.0 / 32 as f64,
// // self.epoch * 100,
// // None,
// // ),
// // );
// // params.topics.insert(
// // get_hash(GossipKind::AttesterSlashing),
// // Self::get_topic_params(
// // self,
// // ATTESTER_SLASHING_WEIGHT,
// // // 1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
// // 1.0 / 5.0 / 32 as f64,
// // self.epoch * 100,
// // None,
// // ),
// // );
// // params.topics.insert(
// // get_hash(GossipKind::ProposerSlashing),
// // Self::get_topic_params(
// // self,
// // PROPOSER_SLASHING_WEIGHT,
// // // 1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
// // 1.0 / 5.0 / 32 as f64,
// // self.epoch * 100,
// // None,
// // ),
// // );
// //dynamic topics
// // let (beacon_block_params, beacon_aggregate_proof_params, beacon_attestation_subnet_params) =
// // self.get_dynamic_topic_params(active_validators, current_slot)?;
// // params
// // .topics
// // .insert(get_hash(GossipKind::BeaconBlock), beacon_block_params);
// // params.topics.insert(
// // get_hash(GossipKind::BeaconAggregateAndProof),
// // beacon_aggregate_proof_params,
// // );
// // for i in 0..self.attestation_subnet_count {
// // params.topics.insert(
// // get_hash(GossipKind::Attestation(SubnetId::new(i))),
// // beacon_attestation_subnet_params.clone(),
// // );
// // }
// Ok(params)
// }
// // pub fn get_dynamic_topic_params(
// // &self,
// // active_validators: usize,
// // current_slot: Slot,
// // ) -> error::Result<(TopicScoreParams, TopicScoreParams, TopicScoreParams)> {
// // let (aggregators_per_slot, committees_per_slot) =
// // self.expected_aggregator_count_per_slot(active_validators)?;
// // let multiple_bursts_per_subnet_per_epoch = committees_per_slot as u64
// // >= 2 * self.attestation_subnet_count / TSpec::slots_per_epoch();
// // let beacon_block_params = Self::get_topic_params(
// // self,
// // BEACON_BLOCK_WEIGHT,
// // 1.0,
// // self.epoch * 20,
// // Some((TSpec::slots_per_epoch() * 5, 3.0, self.epoch, current_slot)),
// // );
// // let beacon_aggregate_proof_params = Self::get_topic_params(
// // self,
// // BEACON_AGGREGATE_PROOF_WEIGHT,
// // aggregators_per_slot,
// // self.epoch,
// // Some((TSpec::slots_per_epoch() * 2, 4.0, self.epoch, current_slot)),
// // );
// // let beacon_attestation_subnet_params = Self::get_topic_params(
// // self,
// // self.beacon_attestation_subnet_weight,
// // active_validators as f64
// // / self.attestation_subnet_count as f64
// // / TSpec::slots_per_epoch() as f64,
// // self.epoch
// // * (if multiple_bursts_per_subnet_per_epoch {
// // 1
// // } else {
// // 4
// // }),
// // Some((
// // TSpec::slots_per_epoch()
// // * (if multiple_bursts_per_subnet_per_epoch {
// // 4
// // } else {
// // 16
// // }),
// // 16.0,
// // if multiple_bursts_per_subnet_per_epoch {
// // self.slot * (TSpec::slots_per_epoch() as u32 / 2 + 1)
// // } else {
// // self.epoch * 3
// // },
// // current_slot,
// // )),
// // );
// // Ok((
// // beacon_block_params,
// // beacon_aggregate_proof_params,
// // beacon_attestation_subnet_params,
// // ))
// // }
// pub fn attestation_subnet_count(&self) -> u64 {
// self.attestation_subnet_count
// }
// fn score_parameter_decay_with_base(
// decay_time: Duration,
// decay_interval: Duration,
// decay_to_zero: f64,
// ) -> f64 {
// let ticks = decay_time.as_secs_f64() / decay_interval.as_secs_f64();
// decay_to_zero.powf(1.0 / ticks)
// }
// fn decay_convergence(decay: f64, rate: f64) -> f64 {
// rate / (1.0 - decay)
// }
// // fn threshold(decay: f64, rate: f64) -> f64 {
// // Self::decay_convergence(decay, rate) * decay
// // }
// // fn expected_aggregator_count_per_slot(
// // &self,
// // active_validators: usize,
// // ) -> error::Result<(f64, usize)> {
// // let committees_per_slot = TSpec::get_committee_count_per_slot_with(
// // active_validators,
// // self.max_committees_per_slot,
// // self.target_committee_size,
// // )
// // .map_err(|e| format!("Could not get committee count from spec: {:?}", e))?;
// // let committees = committees_per_slot * TSpec::slots_per_epoch() as usize;
// // let smaller_committee_size = active_validators / committees;
// // let num_larger_committees = active_validators - smaller_committee_size * committees;
// // let modulo_smaller = max(
// // 1,
// // smaller_committee_size / self.target_aggregators_per_committee as usize,
// // );
// // let modulo_larger = max(
// // 1,
// // (smaller_committee_size + 1) / self.target_aggregators_per_committee as usize,
// // );
// // Ok((
// // (((committees - num_larger_committees) * smaller_committee_size) as f64
// // / modulo_smaller as f64
// // + (num_larger_committees * (smaller_committee_size + 1)) as f64
// // / modulo_larger as f64)
// // / TSpec::slots_per_epoch() as f64,
// // committees_per_slot,
// // ))
// // }
// fn score_parameter_decay(&self, decay_time: Duration) -> f64 {
// Self::score_parameter_decay_with_base(decay_time, self.decay_interval, self.decay_to_zero)
// }
// // fn get_topic_params(
// // &self,
// // topic_weight: f64,
// // expected_message_rate: f64,
// // first_message_decay_time: Duration,
// // // decay slots (decay time in slots), cap factor, activation window, current slot
// // mesh_message_info: Option<(u64, f64, Duration, Slot)>,
// // ) -> TopicScoreParams {
// // let mut t_params = TopicScoreParams::default();
// // t_params.topic_weight = topic_weight;
// // t_params.time_in_mesh_quantum = self.slot;
// // t_params.time_in_mesh_cap = 3600.0 / t_params.time_in_mesh_quantum.as_secs_f64();
// // t_params.time_in_mesh_weight = 10.0 / t_params.time_in_mesh_cap;
// // t_params.first_message_deliveries_decay =
// // self.score_parameter_decay(first_message_decay_time);
// // t_params.first_message_deliveries_cap = Self::decay_convergence(
// // t_params.first_message_deliveries_decay,
// // 2.0 * expected_message_rate / self.mesh_n as f64,
// // );
// // t_params.first_message_deliveries_weight = 40.0 / t_params.first_message_deliveries_cap;
// // if let Some((decay_slots, cap_factor, activation_window, current_slot)) = mesh_message_info
// // {
// // let decay_time = self.slot * decay_slots as u32;
// // t_params.mesh_message_deliveries_decay = self.score_parameter_decay(decay_time);
// // t_params.mesh_message_deliveries_threshold = Self::threshold(
// // t_params.mesh_message_deliveries_decay,
// // expected_message_rate / 50.0,
// // );
// // t_params.mesh_message_deliveries_cap =
// // if cap_factor * t_params.mesh_message_deliveries_threshold < 2.0 {
// // 2.0
// // } else {
// // cap_factor * t_params.mesh_message_deliveries_threshold
// // };
// // t_params.mesh_message_deliveries_activation = activation_window;
// // t_params.mesh_message_deliveries_window =
// // Duration::from_secs(MESH_MESSAGE_DELIVERIES_WINDOW);
// // t_params.mesh_failure_penalty_decay = t_params.mesh_message_deliveries_decay;
// // t_params.mesh_message_deliveries_weight = -t_params.topic_weight;
// // t_params.mesh_failure_penalty_weight = t_params.mesh_message_deliveries_weight;
// // if decay_slots >= current_slot.as_u64() {
// // t_params.mesh_message_deliveries_threshold = 0.0;
// // t_params.mesh_message_deliveries_weight = 0.0;
// // }
// // } else {
// // t_params.mesh_message_deliveries_weight = 0.0;
// // t_params.mesh_message_deliveries_threshold = 0.0;
// // t_params.mesh_message_deliveries_decay = 0.0;
// // t_params.mesh_message_deliveries_cap = 0.0;
// // t_params.mesh_message_deliveries_window = Duration::from_secs(0);
// // t_params.mesh_message_deliveries_activation = Duration::from_secs(0);
// // t_params.mesh_failure_penalty_decay = 0.0;
// // t_params.mesh_failure_penalty_weight = 0.0;
// // }
// // t_params.invalid_message_deliveries_weight =
// // -self.max_positive_score / t_params.topic_weight;
// // t_params.invalid_message_deliveries_decay = self.score_parameter_decay(self.epoch * 50);
// // t_params
// // }
// }

Some files were not shown because too many files have changed in this diff Show More