Compare commits

..

7 Commits

Author SHA1 Message Date
0g-peterzhb
28f190a178
Merge 93885bb25c into 1434b94495 2024-09-14 19:07:11 +08:00
bruno-valante
1434b94495
Resolve dependency issue from atty (#204)
Some checks failed
abi-consistent-check / build-and-compare (push) Has been cancelled
code-coverage / unittest-cov (push) Has been cancelled
rust / check (push) Has been cancelled
rust / test (push) Has been cancelled
rust / lints (push) Has been cancelled
functional-test / test (push) Has been cancelled
2024-09-14 19:06:52 +08:00
bruno-valante
da0e2c7031
Fix issues found in auditions (#203) 2024-09-14 18:19:16 +08:00
peilun-conflux
10bd71046b
Add some input validation for append_merkle. (#202)
* Add some input validation for `append_merkle`.

* Fix clippy.
2024-09-14 17:39:07 +08:00
Peter Zhang
93885bb25c range root should also add back missing padded data 2024-09-14 17:38:29 +08:00
Peter Zhang
e58f6ca101 add missing padded subtree leaves to root map when recover the merkle 2024-09-14 17:08:30 +08:00
Peter Zhang
f29af9c872 only async full empty segment 2024-09-14 14:29:14 +08:00
21 changed files with 332 additions and 265 deletions

386
Cargo.lock generated
View File

@ -157,6 +157,55 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
[[package]]
name = "anstyle-parse"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
dependencies = [
"anstyle",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.86" version = "1.0.86"
@ -460,17 +509,6 @@ dependencies = [
"wildmatch", "wildmatch",
] ]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi 0.1.19",
"libc",
"winapi",
]
[[package]] [[package]]
name = "auto_impl" name = "auto_impl"
version = "1.2.0" version = "1.2.0"
@ -601,6 +639,9 @@ name = "bitflags"
version = "2.6.0" version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "bitmaps" name = "bitmaps"
@ -690,9 +731,9 @@ dependencies = [
[[package]] [[package]]
name = "bstr" name = "bstr"
version = "1.9.1" version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
dependencies = [ dependencies = [
"memchr", "memchr",
"serde", "serde",
@ -718,9 +759,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.6.0" version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
dependencies = [ dependencies = [
"serde", "serde",
] ]
@ -859,7 +900,7 @@ dependencies = [
"js-sys", "js-sys",
"num-traits", "num-traits",
"wasm-bindgen", "wasm-bindgen",
"windows-targets 0.52.5", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@ -937,28 +978,30 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "3.2.25" version = "4.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac"
dependencies = [ dependencies = [
"atty", "clap_builder",
"bitflags 1.3.2", ]
[[package]]
name = "clap_builder"
version = "4.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73"
dependencies = [
"anstream",
"anstyle",
"clap_lex", "clap_lex",
"indexmap 1.9.3", "strsim 0.11.1",
"once_cell",
"strsim 0.10.0",
"termcolor",
"textwrap",
] ]
[[package]] [[package]]
name = "clap_lex" name = "clap_lex"
version = "0.2.4" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
dependencies = [
"os_str_bytes",
]
[[package]] [[package]]
name = "cmake" name = "cmake"
@ -1021,6 +1064,12 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "colorchoice"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
[[package]] [[package]]
name = "concurrent-queue" name = "concurrent-queue"
version = "2.5.0" version = "2.5.0"
@ -1032,11 +1081,12 @@ dependencies = [
[[package]] [[package]]
name = "config" name = "config"
version = "0.13.4" version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"convert_case",
"json5", "json5",
"lazy_static", "lazy_static",
"nom", "nom",
@ -1045,7 +1095,7 @@ dependencies = [
"rust-ini", "rust-ini",
"serde", "serde",
"serde_json", "serde_json",
"toml 0.5.11", "toml 0.8.14",
"yaml-rust", "yaml-rust",
] ]
@ -1068,6 +1118,26 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "const-random"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
dependencies = [
"const-random-macro",
]
[[package]]
name = "const-random-macro"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
"getrandom 0.2.15",
"once_cell",
"tiny-keccak",
]
[[package]] [[package]]
name = "constant_time_eq" name = "constant_time_eq"
version = "0.1.5" version = "0.1.5"
@ -1082,6 +1152,15 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "convert_case"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
dependencies = [
"unicode-segmentation",
]
[[package]] [[package]]
name = "core-foundation" name = "core-foundation"
version = "0.9.4" version = "0.9.4"
@ -1136,19 +1215,19 @@ dependencies = [
[[package]] [[package]]
name = "criterion" name = "criterion"
version = "0.4.0" version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [ dependencies = [
"anes", "anes",
"atty",
"cast", "cast",
"ciborium", "ciborium",
"clap", "clap",
"criterion-plot", "criterion-plot",
"is-terminal",
"itertools 0.10.5", "itertools 0.10.5",
"lazy_static",
"num-traits", "num-traits",
"once_cell",
"oorandom", "oorandom",
"plotters", "plotters",
"rayon", "rayon",
@ -1611,9 +1690,12 @@ dependencies = [
[[package]] [[package]]
name = "dlv-list" name = "dlv-list"
version = "0.3.0" version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f"
dependencies = [
"const-random",
]
[[package]] [[package]]
name = "dns-lookup" name = "dns-lookup"
@ -2466,12 +2548,6 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]] [[package]]
name = "funty" name = "funty"
version = "1.1.0" version = "1.1.0"
@ -2729,9 +2805,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]] [[package]]
name = "globset" name = "globset"
version = "0.4.14" version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"bstr", "bstr",
@ -2854,6 +2930,12 @@ dependencies = [
"ahash 0.7.8", "ahash 0.7.8",
] ]
[[package]]
name = "hashbrown"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.5" version = "0.14.5"
@ -2921,15 +3003,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.3.9" version = "0.3.9"
@ -3383,6 +3456,23 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "is-terminal"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
dependencies = [
"hermit-abi 0.4.0",
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]] [[package]]
name = "itertools" name = "itertools"
version = "0.10.5" version = "0.10.5"
@ -3767,7 +3857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"windows-targets 0.52.5", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@ -4929,7 +5019,6 @@ dependencies = [
"lighthouse_metrics", "lighthouse_metrics",
"lru", "lru",
"parking_lot 0.12.3", "parking_lot 0.12.3",
"prometheus-client",
"rand 0.8.5", "rand 0.8.5",
"regex", "regex",
"serde", "serde",
@ -5197,20 +5286,14 @@ dependencies = [
[[package]] [[package]]
name = "ordered-multimap" name = "ordered-multimap"
version = "0.4.3" version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e"
dependencies = [ dependencies = [
"dlv-list", "dlv-list",
"hashbrown 0.12.3", "hashbrown 0.13.2",
] ]
[[package]]
name = "os_str_bytes"
version = "6.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
[[package]] [[package]]
name = "overload" name = "overload"
version = "0.1.1" version = "0.1.1"
@ -5335,7 +5418,7 @@ dependencies = [
"libc", "libc",
"redox_syscall 0.5.2", "redox_syscall 0.5.2",
"smallvec", "smallvec",
"windows-targets 0.52.5", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@ -6100,19 +6183,6 @@ dependencies = [
"nibble_vec", "nibble_vec",
] ]
[[package]]
name = "rand"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
dependencies = [
"fuchsia-cprng",
"libc",
"rand_core 0.3.1",
"rdrand",
"winapi",
]
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.7.3" version = "0.7.3"
@ -6157,21 +6227,6 @@ dependencies = [
"rand_core 0.6.4", "rand_core 0.6.4",
] ]
[[package]]
name = "rand_core"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
dependencies = [
"rand_core 0.4.2",
]
[[package]]
name = "rand_core"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.5.1" version = "0.5.1"
@ -6228,15 +6283,6 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.1",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.16" version = "0.2.16"
@ -6310,15 +6356,6 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.11.27" version = "0.11.27"
@ -6467,13 +6504,14 @@ dependencies = [
[[package]] [[package]]
name = "ron" name = "ron"
version = "0.7.1" version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [ dependencies = [
"base64 0.13.1", "base64 0.21.7",
"bitflags 1.3.2", "bitflags 2.6.0",
"serde", "serde",
"serde_derive",
] ]
[[package]] [[package]]
@ -6548,9 +6586,9 @@ dependencies = [
[[package]] [[package]]
name = "rust-ini" name = "rust-ini"
version = "0.18.0" version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"ordered-multimap", "ordered-multimap",
@ -7241,7 +7279,6 @@ dependencies = [
"serde_json", "serde_json",
"shared_types", "shared_types",
"static_assertions", "static_assertions",
"tempdir",
"tiny-keccak", "tiny-keccak",
"tokio", "tokio",
"tracing", "tracing",
@ -7288,6 +7325,12 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]] [[package]]
name = "strum" name = "strum"
version = "0.24.1" version = "0.24.1"
@ -7469,26 +7512,17 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "tempdir"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
dependencies = [
"rand 0.4.6",
"remove_dir_all",
]
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.10.1" version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"fastrand 2.1.0", "fastrand 2.1.0",
"once_cell",
"rustix 0.38.34", "rustix 0.38.34",
"windows-sys 0.52.0", "windows-sys 0.59.0",
] ]
[[package]] [[package]]
@ -7502,21 +7536,6 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "termcolor"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
dependencies = [
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.61" version = "1.0.61"
@ -8236,6 +8255,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]] [[package]]
name = "uuid" name = "uuid"
version = "0.8.2" version = "0.8.2"
@ -8509,7 +8534,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [ dependencies = [
"windows-targets 0.52.5", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@ -8527,7 +8552,16 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [ dependencies = [
"windows-targets 0.52.5", "windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
] ]
[[package]] [[package]]
@ -8547,18 +8581,18 @@ dependencies = [
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [ dependencies = [
"windows_aarch64_gnullvm 0.52.5", "windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.5", "windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.5", "windows_i686_gnu 0.52.6",
"windows_i686_gnullvm", "windows_i686_gnullvm",
"windows_i686_msvc 0.52.5", "windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.5", "windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.5", "windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.5", "windows_x86_64_msvc 0.52.6",
] ]
[[package]] [[package]]
@ -8569,9 +8603,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]] [[package]]
name = "windows_aarch64_gnullvm" name = "windows_aarch64_gnullvm"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
@ -8587,9 +8621,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
@ -8605,15 +8639,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]] [[package]]
name = "windows_i686_gnullvm" name = "windows_i686_gnullvm"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
@ -8629,9 +8663,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
@ -8647,9 +8681,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
@ -8659,9 +8693,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
@ -8677,9 +8711,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.52.5" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]] [[package]]
name = "winnow" name = "winnow"

View File

@ -137,13 +137,20 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
} }
} }
/// Return the new merkle root.
pub fn append(&mut self, new_leaf: E) { pub fn append(&mut self, new_leaf: E) {
if new_leaf == E::null() {
// appending null is not allowed.
return;
}
self.layers[0].push(new_leaf); self.layers[0].push(new_leaf);
self.recompute_after_append_leaves(self.leaves() - 1); self.recompute_after_append_leaves(self.leaves() - 1);
} }
pub fn append_list(&mut self, mut leaf_list: Vec<E>) { pub fn append_list(&mut self, mut leaf_list: Vec<E>) {
if leaf_list.contains(&E::null()) {
// appending null is not allowed.
return;
}
let start_index = self.leaves(); let start_index = self.leaves();
self.layers[0].append(&mut leaf_list); self.layers[0].append(&mut leaf_list);
self.recompute_after_append_leaves(start_index); self.recompute_after_append_leaves(start_index);
@ -155,6 +162,10 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
/// Other nodes in the subtree will be set to `null` nodes. /// Other nodes in the subtree will be set to `null` nodes.
/// TODO: Optimize to avoid storing the `null` nodes? /// TODO: Optimize to avoid storing the `null` nodes?
pub fn append_subtree(&mut self, subtree_depth: usize, subtree_root: E) -> Result<()> { pub fn append_subtree(&mut self, subtree_depth: usize, subtree_root: E) -> Result<()> {
if subtree_root == E::null() {
// appending null is not allowed.
bail!("subtree_root is null");
}
let start_index = self.leaves(); let start_index = self.leaves();
self.append_subtree_inner(subtree_depth, subtree_root)?; self.append_subtree_inner(subtree_depth, subtree_root)?;
self.recompute_after_append_subtree(start_index, subtree_depth - 1); self.recompute_after_append_subtree(start_index, subtree_depth - 1);
@ -162,6 +173,10 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
} }
pub fn append_subtree_list(&mut self, subtree_list: Vec<(usize, E)>) -> Result<()> { pub fn append_subtree_list(&mut self, subtree_list: Vec<(usize, E)>) -> Result<()> {
if subtree_list.iter().any(|(_, root)| root == &E::null()) {
// appending null is not allowed.
bail!("subtree_list contains null");
}
for (subtree_depth, subtree_root) in subtree_list { for (subtree_depth, subtree_root) in subtree_list {
let start_index = self.leaves(); let start_index = self.leaves();
self.append_subtree_inner(subtree_depth, subtree_root)?; self.append_subtree_inner(subtree_depth, subtree_root)?;
@ -173,6 +188,10 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
/// Change the value of the last leaf and return the new merkle root. /// Change the value of the last leaf and return the new merkle root.
/// This is needed if our merkle-tree in memory only keeps intermediate nodes instead of real leaves. /// This is needed if our merkle-tree in memory only keeps intermediate nodes instead of real leaves.
pub fn update_last(&mut self, updated_leaf: E) { pub fn update_last(&mut self, updated_leaf: E) {
if updated_leaf == E::null() {
// updating to null is not allowed.
return;
}
if self.layers[0].is_empty() { if self.layers[0].is_empty() {
// Special case for the first data. // Special case for the first data.
self.layers[0].push(updated_leaf); self.layers[0].push(updated_leaf);
@ -183,10 +202,12 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
} }
/// Fill an unknown `null` leaf with its real value. /// Fill an unknown `null` leaf with its real value.
/// Panics if the leaf changes the merkle root or the index is out of range. /// Panics if the leaf is already set and different or the index is out of range.
/// TODO: Batch computing intermediate nodes. /// TODO: Batch computing intermediate nodes.
pub fn fill_leaf(&mut self, index: usize, leaf: E) { pub fn fill_leaf(&mut self, index: usize, leaf: E) {
if self.layers[0][index] == E::null() { if leaf == E::null() {
// fill leaf with null is not allowed.
} else if self.layers[0][index] == E::null() {
self.layers[0][index] = leaf; self.layers[0][index] = leaf;
self.recompute_after_fill_leaves(index, index + 1); self.recompute_after_fill_leaves(index, index + 1);
} else if self.layers[0][index] != leaf { } else if self.layers[0][index] != leaf {
@ -349,7 +370,6 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
right_most_nodes.push((layer.len() - 1, layer.last().unwrap().clone())); right_most_nodes.push((layer.len() - 1, layer.last().unwrap().clone()));
} }
let root = self.root().clone(); let root = self.root().clone();
assert_eq!(root, right_most_nodes.last().unwrap().1);
self.delta_nodes_map self.delta_nodes_map
.insert(tx_seq, DeltaNodes::new(right_most_nodes)); .insert(tx_seq, DeltaNodes::new(right_most_nodes));
self.root_to_tx_seq_map.insert(root, tx_seq); self.root_to_tx_seq_map.insert(root, tx_seq);

View File

@ -74,10 +74,7 @@ pub trait MerkleTreeRead {
bail!("Not ready to generate proof for leaf_index={}", leaf_index); bail!("Not ready to generate proof for leaf_index={}", leaf_index);
} }
if self.height() == 1 { if self.height() == 1 {
return Ok(Proof::new( return Proof::new(vec![self.root().clone(), self.root().clone()], vec![]);
vec![self.root().clone(), self.root().clone()],
vec![],
));
} }
let mut lemma: Vec<Self::E> = Vec::with_capacity(self.height()); // path + root let mut lemma: Vec<Self::E> = Vec::with_capacity(self.height()); // path + root
let mut path: Vec<bool> = Vec::with_capacity(self.height() - 2); // path - 1 let mut path: Vec<bool> = Vec::with_capacity(self.height() - 2); // path - 1
@ -112,7 +109,7 @@ pub trait MerkleTreeRead {
path path
); );
} }
Ok(Proof::new(lemma, path)) Proof::new(lemma, path)
} }
fn gen_range_proof(&self, start_index: usize, end_index: usize) -> Result<RangeProof<Self::E>> { fn gen_range_proof(&self, start_index: usize, end_index: usize) -> Result<RangeProof<Self::E>> {

View File

@ -11,9 +11,11 @@ pub struct Proof<T: HashElement> {
impl<T: HashElement> Proof<T> { impl<T: HashElement> Proof<T> {
/// Creates new MT inclusion proof /// Creates new MT inclusion proof
pub fn new(hash: Vec<T>, path: Vec<bool>) -> Proof<T> { pub fn new(hash: Vec<T>, path: Vec<bool>) -> Result<Proof<T>> {
assert_eq!(hash.len() - 2, path.len()); if hash.len() != path.len() + 2 {
Proof { lemma: hash, path } bail!("hash and path length mismatch");
}
Ok(Proof { lemma: hash, path })
} }
pub fn new_empty() -> Proof<T> { pub fn new_empty() -> Proof<T> {
@ -58,10 +60,10 @@ impl<T: HashElement> Proof<T> {
bail!("Invalid proof"); bail!("Invalid proof");
} }
if *item != self.item() { if *item != self.item() {
bail!("Proof item unmatch"); bail!("Proof item mismatch");
} }
if position != self.position() { if position != self.position() {
bail!("Proof position unmatch"); bail!("Proof position mismatch");
} }
Ok(()) Ok(())
} }
@ -88,7 +90,7 @@ impl<T: HashElement> Proof<T> {
/// Return `Vec<(index_in_layer, data)>`. /// Return `Vec<(index_in_layer, data)>`.
pub fn proof_nodes_in_tree(&self) -> Vec<(usize, T)> { pub fn proof_nodes_in_tree(&self) -> Vec<(usize, T)> {
let mut r = Vec::with_capacity(self.lemma.len()); let mut r = Vec::with_capacity(self.lemma.len() - 1);
let mut pos = 0; let mut pos = 0;
r.push((0, self.root())); r.push((0, self.root()));
for (i, is_left) in self.path.iter().rev().enumerate() { for (i, is_left) in self.path.iter().rev().enumerate() {
@ -108,7 +110,7 @@ impl<T: HashElement> Proof<T> {
tx_merkle_nodes: Vec<(usize, T)>, tx_merkle_nodes: Vec<(usize, T)>,
tx_merkle_nodes_size: usize, tx_merkle_nodes_size: usize,
) -> Vec<(usize, T)> { ) -> Vec<(usize, T)> {
let mut r = Vec::with_capacity(self.lemma.len()); let mut r = Vec::with_capacity(self.path.len());
let mut subtree_pos = 0; let mut subtree_pos = 0;
let mut root_pos = 0; let mut root_pos = 0;
let mut in_subtree = tx_merkle_nodes_size == 1; let mut in_subtree = tx_merkle_nodes_size == 1;
@ -222,7 +224,7 @@ impl<E: HashElement> RangeProof<E> {
} }
children_layer = parent_layer; children_layer = parent_layer;
} }
assert_eq!(children_layer.len(), 1); ensure_eq!(children_layer.len(), 1);
let computed_root = children_layer.pop().unwrap(); let computed_root = children_layer.pop().unwrap();
ensure_eq!(computed_root, self.root()); ensure_eq!(computed_root, self.root());

View File

@ -5,7 +5,7 @@ edition = "2021"
[dependencies] [dependencies]
anyhow = { version = "1.0.58", features = ["backtrace"] } anyhow = { version = "1.0.58", features = ["backtrace"] }
clap = { version = "3.2.5", features = ["cargo"] } clap = { version = "4.5.17", features = ["cargo"] }
ctrlc = "3.2.2" ctrlc = "3.2.2"
error-chain = "0.12.4" error-chain = "0.12.4"
ethereum-types = "0.14" ethereum-types = "0.14"
@ -35,7 +35,7 @@ chunk_pool = { path = "./chunk_pool" }
itertools = "0.10.5" itertools = "0.10.5"
serde = { version = "1.0.137", features = ["derive"] } serde = { version = "1.0.137", features = ["derive"] }
duration-str = "0.5.1" duration-str = "0.5.1"
config = "0.13.1" config = "0.14"
public-ip = "0.2" public-ip = "0.2"
ethers = "2.0.14" ethers = "2.0.14"
metrics = { workspace = true } metrics = { workspace = true }

View File

@ -11,7 +11,7 @@ append_merkle = { path = "../../common/append_merkle" }
async-trait = "0.1.56" async-trait = "0.1.56"
ethereum-types = "0.14" ethereum-types = "0.14"
futures = "0.3.21" futures = "0.3.21"
jsonrpsee = { version = "0.14.0", features = ["full"] } jsonrpsee = { version = "0.14", features = ["full"] }
shared_types = { path = "../shared_types" } shared_types = { path = "../shared_types" }
task_executor = { path = "../../common/task_executor" } task_executor = { path = "../../common/task_executor" }
tokio = "1.19.2" tokio = "1.19.2"

View File

@ -98,7 +98,7 @@ impl MineRangeConfig {
let self_start_position = self.start_position?; let self_start_position = self.start_position?;
let self_end_position = self.end_position?; let self_end_position = self.end_position?;
if self.start_position >= self.end_position { if self_start_position >= self_end_position {
return Some(false); return Some(false);
} }
Some( Some(

View File

@ -79,7 +79,7 @@ impl<'a> Miner<'a> {
inc_counter(&LOADING_COUNT); inc_counter(&LOADING_COUNT);
let MineLoadChunk { let MineLoadChunk {
loaded_chunk, loaded_chunk,
avalibilities, availabilities,
} = self } = self
.loader .loader
.load_sealed_data(recall_position / SECTORS_PER_LOAD as u64) .load_sealed_data(recall_position / SECTORS_PER_LOAD as u64)
@ -92,8 +92,8 @@ impl<'a> Miner<'a> {
.into_iter() .into_iter()
.enumerate() .enumerate()
.zip(scratch_pad.iter().cycle()) .zip(scratch_pad.iter().cycle())
.zip(avalibilities.into_iter()) .zip(availabilities.into_iter())
.filter_map(|(data, avaliable)| avaliable.then_some(data)) .filter_map(|(data, availiable)| availiable.then_some(data))
{ {
inc_counter(&PAD_MIX_COUNT); inc_counter(&PAD_MIX_COUNT);
// Rust can optimize this loop well. // Rust can optimize this loop well.
@ -114,7 +114,7 @@ impl<'a> Miner<'a> {
difficulty_scale_x64.as_u128() as f64 / u64::MAX as f64 difficulty_scale_x64.as_u128() as f64 / u64::MAX as f64
); );
inc_counter(&HIT_COUNT); inc_counter(&HIT_COUNT);
// Undo mix data when find a valid solition // Undo mix data when find a valid solution
for (x, y) in sealed_data.iter_mut().zip(scratch_pad.iter()) { for (x, y) in sealed_data.iter_mut().zip(scratch_pad.iter()) {
*x ^= y; *x ^= y;
} }
@ -171,7 +171,7 @@ impl<'a> Miner<'a> {
) -> U256 { ) -> U256 {
let mut hasher = Blake2b512::new(); let mut hasher = Blake2b512::new();
hasher.update([0u8; 24]); hasher.update([0u8; 24]);
hasher.update(seal_index.to_be_bytes()); hasher.update((seal_index as u64).to_be_bytes());
hasher.update(pad_seed); hasher.update(pad_seed);
hasher.update([0u8; 32]); hasher.update([0u8; 32]);

View File

@ -31,12 +31,13 @@ impl RecallRange {
} }
pub fn load_position(&self, seed: [u8; 32]) -> Option<u64> { pub fn load_position(&self, seed: [u8; 32]) -> Option<u64> {
let (_, origin_recall_offset) = U256::from_big_endian(&seed) let origin_recall_offset = U256::from_big_endian(&seed)
.div_mod(U256::from((self.mining_length as usize) / SECTORS_PER_LOAD)); .checked_rem(U256::from((self.mining_length as usize) / SECTORS_PER_LOAD))?;
let origin_recall_offset = origin_recall_offset.as_u64(); let origin_recall_offset = origin_recall_offset.as_u64();
let recall_offset = (origin_recall_offset & self.shard_mask) | self.shard_id; let recall_offset = (origin_recall_offset & self.shard_mask) | self.shard_id;
Some(self.start_position + recall_offset * SECTORS_PER_LOAD as u64) self.start_position
.checked_add(recall_offset * SECTORS_PER_LOAD as u64)
} }
pub fn difficulty_scale_x64(&self, flow_length: u64) -> U256 { pub fn difficulty_scale_x64(&self, flow_length: u64) -> U256 {

View File

@ -18,7 +18,7 @@ use crate::watcher::MineContextMessage;
use zgs_spec::{BYTES_PER_SEAL, SECTORS_PER_SEAL}; use zgs_spec::{BYTES_PER_SEAL, SECTORS_PER_SEAL};
const SUBMISSION_RETIES: usize = 15; const SUBMISSION_RETRIES: usize = 15;
pub struct Submitter { pub struct Submitter {
mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>, mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>,
@ -154,7 +154,7 @@ impl Submitter {
let pending_transaction: PendingTransaction<'_, _> = submission_call let pending_transaction: PendingTransaction<'_, _> = submission_call
.send() .send()
.await .await
.map_err(|e| format!("Fail to send mine answer transaction: {:?}", e))?; .map_err(|e| format!("Fail to send PoRA submission transaction: {:?}", e))?;
debug!( debug!(
"Signed submission transaction hash: {:?}", "Signed submission transaction hash: {:?}",
@ -162,13 +162,13 @@ impl Submitter {
); );
let receipt = pending_transaction let receipt = pending_transaction
.retries(SUBMISSION_RETIES) .retries(SUBMISSION_RETRIES)
.interval(Duration::from_secs(2)) .interval(Duration::from_secs(2))
.await .await
.map_err(|e| format!("Fail to execute mine answer transaction: {:?}", e))? .map_err(|e| format!("Fail to execute PoRA submission transaction: {:?}", e))?
.ok_or(format!( .ok_or(format!(
"Mine answer transaction dropped after {} retries", "PoRA submission transaction dropped after {} retries",
SUBMISSION_RETIES SUBMISSION_RETRIES
))?; ))?;
info!("Submit PoRA success, receipt: {:?}", receipt); info!("Submit PoRA success, receipt: {:?}", receipt);

View File

@ -47,8 +47,6 @@ impl MineContextWatcher {
provider: Arc<MineServiceMiddleware>, provider: Arc<MineServiceMiddleware>,
config: &MinerConfig, config: &MinerConfig,
) -> broadcast::Receiver<MineContextMessage> { ) -> broadcast::Receiver<MineContextMessage> {
let provider = provider;
let mine_contract = PoraMine::new(config.mine_address, provider.clone()); let mine_contract = PoraMine::new(config.mine_address, provider.clone());
let flow_contract = ZgsFlow::new(config.flow_address, provider.clone()); let flow_contract = ZgsFlow::new(config.flow_address, provider.clone());

View File

@ -21,7 +21,6 @@ lazy_static = "1.4.0"
lighthouse_metrics = { path = "../../common/lighthouse_metrics" } lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
lru = "0.7.7" lru = "0.7.7"
parking_lot = "0.12.1" parking_lot = "0.12.1"
prometheus-client = "0.16.0"
rand = "0.8.5" rand = "0.8.5"
regex = "1.5.6" regex = "1.5.6"
serde = { version = "1.0.137", features = ["derive"] } serde = { version = "1.0.137", features = ["derive"] }
@ -49,7 +48,7 @@ features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dn
[dev-dependencies] [dev-dependencies]
exit-future = "0.2.0" exit-future = "0.2.0"
tempfile = "3.3.0" tempfile = "3.12.0"
tracing-test = "0.2.2" tracing-test = "0.2.2"
unused_port = { path = "../../common/unused_port" } unused_port = { path = "../../common/unused_port" }
void = "1.0.2" void = "1.0.2"

View File

@ -74,8 +74,6 @@ impl<'de> Deserialize<'de> for PeerIdSerialized {
pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage}; pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
pub use prometheus_client;
pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response}; pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response};
pub use config::Config as NetworkConfig; pub use config::Config as NetworkConfig;
pub use discovery::{CombinedKeyExt, EnrExt}; pub use discovery::{CombinedKeyExt, EnrExt};

View File

@ -364,7 +364,7 @@ impl TryFrom<FileProof> for FlowProof {
if lemma.len() != value.path.len() + 2 { if lemma.len() != value.path.len() + 2 {
Err(anyhow!("invalid file proof")) Err(anyhow!("invalid file proof"))
} else { } else {
Ok(Self::new(lemma, value.path)) Self::new(lemma, value.path)
} }
} }
} }

View File

@ -1,6 +1,6 @@
use clap::{arg, command, Command}; use clap::{arg, command, Command};
pub fn cli_app<'a>() -> Command<'a> { pub fn cli_app() -> Command {
command!() command!()
.arg(arg!(-c --config <FILE> "Sets a custom config file")) .arg(arg!(-c --config <FILE> "Sets a custom config file"))
.arg(arg!(--"log-config-file" [FILE] "Sets log configuration file (Default: log_config)")) .arg(arg!(--"log-config-file" [FILE] "Sets log configuration file (Default: log_config)"))

View File

@ -48,7 +48,7 @@ macro_rules! build_config{
let mut config = RawConfiguration::default(); let mut config = RawConfiguration::default();
// read from config file // read from config file
if let Some(config_file) = matches.value_of("config") { if let Some(config_file) = matches.get_one::<String>("config") {
let config_value = std::fs::read_to_string(config_file) let config_value = std::fs::read_to_string(config_file)
.map_err(|e| format!("failed to read configuration file: {:?}", e))? .map_err(|e| format!("failed to read configuration file: {:?}", e))?
.parse::<toml::Value>() .parse::<toml::Value>()
@ -67,7 +67,7 @@ macro_rules! build_config{
// read from command line // read from command line
$( $(
#[allow(unused_variables)] #[allow(unused_variables)]
if let Some(value) = matches.value_of(underscore_to_hyphen!(stringify!($name))) { if let Some(value) = matches.get_one::<String>(&underscore_to_hyphen!(stringify!($name))) {
if_not_vector!($($type)+, THEN { if_not_vector!($($type)+, THEN {
config.$name = if_option!($($type)+, config.$name = if_option!($($type)+,
THEN{ Some(value.parse().map_err(|_| concat!("Invalid ", stringify!($name)).to_owned())?) } THEN{ Some(value.parse().map_err(|_| concat!("Invalid ", stringify!($name)).to_owned())?) }

View File

@ -110,8 +110,7 @@ impl Store {
} }
pub async fn get_num_entries(&self) -> Result<u64> { pub async fn get_num_entries(&self) -> Result<u64> {
self.spawn(move |store| store.get_num_entries()) self.spawn(move |store| store.get_num_entries()).await
.await
} }
pub async fn remove_chunks_batch(&self, batch_list: &[u64]) -> Result<()> { pub async fn remove_chunks_batch(&self, batch_list: &[u64]) -> Result<()> {

View File

@ -32,10 +32,9 @@ serde_json = "1.0.127"
tokio = { version = "1.10.0", features = ["sync"] } tokio = { version = "1.10.0", features = ["sync"] }
[dev-dependencies] [dev-dependencies]
tempdir = "0.3.7"
rand = "0.8.5" rand = "0.8.5"
hex-literal = "0.3.4" hex-literal = "0.3.4"
criterion = "0.4" criterion = "0.5"
[[bench]] [[bench]]
name = "benchmark" name = "benchmark"

View File

@ -3,7 +3,8 @@ use super::{MineLoadChunk, SealAnswer, SealTask};
use crate::config::ShardConfig; use crate::config::ShardConfig;
use crate::error::Error; use crate::error::Error;
use crate::log_store::log_manager::{ use crate::log_store::log_manager::{
bytes_to_entries, COL_ENTRY_BATCH, COL_ENTRY_BATCH_ROOT, COL_FLOW_MPT_NODES, PORA_CHUNK_SIZE, bytes_to_entries, data_to_merkle_leaves, COL_ENTRY_BATCH, COL_ENTRY_BATCH_ROOT,
COL_FLOW_MPT_NODES, ENTRY_SIZE, PORA_CHUNK_SIZE,
}; };
use crate::log_store::{FlowRead, FlowSeal, FlowWrite}; use crate::log_store::{FlowRead, FlowSeal, FlowWrite};
use crate::{try_option, ZgsKeyValueDB}; use crate::{try_option, ZgsKeyValueDB};
@ -11,7 +12,7 @@ use anyhow::{anyhow, bail, Result};
use append_merkle::{MerkleTreeInitialData, MerkleTreeRead}; use append_merkle::{MerkleTreeInitialData, MerkleTreeRead};
use itertools::Itertools; use itertools::Itertools;
use parking_lot::RwLock; use parking_lot::RwLock;
use shared_types::{ChunkArray, DataRoot, FlowProof}; use shared_types::{ChunkArray, DataRoot, FlowProof, Merkle};
use ssz::{Decode, Encode}; use ssz::{Decode, Encode};
use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode}; use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
use std::cmp::Ordering; use std::cmp::Ordering;
@ -203,7 +204,7 @@ impl FlowRead for FlowStore {
for (seal_index, (sealed, validity)) in mine_chunk for (seal_index, (sealed, validity)) in mine_chunk
.loaded_chunk .loaded_chunk
.iter_mut() .iter_mut()
.zip(mine_chunk.avalibilities.iter_mut()) .zip(mine_chunk.availabilities.iter_mut())
.enumerate() .enumerate()
{ {
if let Some(data) = batch.get_sealed_data(seal_index as u16) { if let Some(data) = batch.get_sealed_data(seal_index as u16) {
@ -441,6 +442,10 @@ impl FlowDBStore {
// and they will be updated in the merkle tree with `fill_leaf` by the caller. // and they will be updated in the merkle tree with `fill_leaf` by the caller.
let mut leaf_list = Vec::new(); let mut leaf_list = Vec::new();
let mut expected_index = 0; let mut expected_index = 0;
let empty_data = vec![0; PORA_CHUNK_SIZE * ENTRY_SIZE];
let empty_root = *Merkle::new(data_to_merkle_leaves(&empty_data)?, 0, None).root();
for r in self.kvdb.iter(COL_ENTRY_BATCH_ROOT) { for r in self.kvdb.iter(COL_ENTRY_BATCH_ROOT) {
let (index_bytes, root_bytes) = r?; let (index_bytes, root_bytes) = r?;
let (batch_index, subtree_depth) = decode_batch_root_key(index_bytes.as_ref())?; let (batch_index, subtree_depth) = decode_batch_root_key(index_bytes.as_ref())?;
@ -475,25 +480,26 @@ impl FlowDBStore {
expected_index += 1; expected_index += 1;
} }
Ordering::Greater => { Ordering::Greater => {
bail!( while batch_index > expected_index {
"unexpected chunk leaf in range, expected={}, get={}, range={:?}", // Fill the gap with empty leaves.
expected_index, root_list.push((1, empty_root));
batch_index, expected_index += 1;
range_root, }
); range_root = None;
root_list.push((1, root));
expected_index += 1;
} }
} }
} }
} else if expected_index == batch_index { } else {
while batch_index > expected_index {
// Fill the gap with empty leaves.
root_list.push((1, empty_root));
expected_index += 1;
}
range_root = Some(BatchRoot::Multiple((subtree_depth, root))); range_root = Some(BatchRoot::Multiple((subtree_depth, root)));
root_list.push((subtree_depth, root)); root_list.push((subtree_depth, root));
expected_index += 1 << (subtree_depth - 1); expected_index += 1 << (subtree_depth - 1);
} else {
bail!(
"unexpected range root: expected={} get={}",
expected_index,
batch_index
);
} }
} }
let extra_node_list = self.get_mpt_node_list()?; let extra_node_list = self.get_mpt_node_list()?;

View File

@ -21,9 +21,9 @@ use shared_types::{
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::path::Path; use std::path::Path;
use std::sync::mpsc;
use std::sync::Arc; use std::sync::Arc;
use std::thread; use std::thread;
use std::sync::mpsc;
use tracing::{debug, error, info, instrument, trace, warn}; use tracing::{debug, error, info, instrument, trace, warn};
use super::tx_store::BlockHashAndSubmissionIndex; use super::tx_store::BlockHashAndSubmissionIndex;
@ -50,7 +50,7 @@ const PAD_MAX_SIZE: usize = 1 << 20;
pub struct UpdateFlowMessage { pub struct UpdateFlowMessage {
pub root_map: BTreeMap<usize, (H256, usize)>, pub root_map: BTreeMap<usize, (H256, usize)>,
pub pad_data: Vec<u8>, pub pad_data: usize,
pub tx_start_flow_index: u64, pub tx_start_flow_index: u64,
} }
@ -252,7 +252,7 @@ impl LogStoreWrite for LogManager {
debug!("recovery with tx_seq={}", tx.seq); debug!("recovery with tx_seq={}", tx.seq);
} else { } else {
// This is not supposed to happen since we have checked the tx seq in log entry sync. // This is not supposed to happen since we have checked the tx seq in log entry sync.
error!("tx unmatch, expected={} get={:?}", expected_seq, tx); error!("tx mismatch, expected={} get={:?}", expected_seq, tx);
bail!("unexpected tx!"); bail!("unexpected tx!");
} }
} }
@ -732,7 +732,7 @@ impl LogManager {
tx_store, tx_store,
flow_store, flow_store,
merkle, merkle,
sender sender,
}; };
log_manager.start_receiver(receiver); log_manager.start_receiver(receiver);
@ -768,10 +768,12 @@ impl LogManager {
// Update the flow database. // Update the flow database.
// This should be called before `complete_last_chunk_merkle` so that we do not save // This should be called before `complete_last_chunk_merkle` so that we do not save
// subtrees with data known. // subtrees with data known.
flow_store.append_entries(ChunkArray { flow_store
data: data.pad_data, .append_entries(ChunkArray {
start_index: data.tx_start_flow_index, data: vec![0; data.pad_data],
}).unwrap(); start_index: data.tx_start_flow_index,
})
.unwrap();
} }
std::result::Result::Err(_) => { std::result::Result::Err(_) => {
bail!("Receiver error"); bail!("Receiver error");
@ -917,6 +919,7 @@ impl LogManager {
); );
if extra != 0 { if extra != 0 {
for pad_data in Self::padding((first_subtree_size - extra) as usize) { for pad_data in Self::padding((first_subtree_size - extra) as usize) {
let mut is_full_empty = true;
let mut root_map = BTreeMap::new(); let mut root_map = BTreeMap::new();
// Update the in-memory merkle tree. // Update the in-memory merkle tree.
@ -928,6 +931,7 @@ impl LogManager {
let mut completed_chunk_index = None; let mut completed_chunk_index = None;
if pad_data.len() < last_chunk_pad { if pad_data.len() < last_chunk_pad {
is_full_empty = false;
merkle merkle
.last_chunk_merkle .last_chunk_merkle
.append_list(data_to_merkle_leaves(&pad_data)?); .append_list(data_to_merkle_leaves(&pad_data)?);
@ -936,6 +940,7 @@ impl LogManager {
.update_last(*merkle.last_chunk_merkle.root()); .update_last(*merkle.last_chunk_merkle.root());
} else { } else {
if last_chunk_pad != 0 { if last_chunk_pad != 0 {
is_full_empty = false;
// Pad the last chunk. // Pad the last chunk.
merkle merkle
.last_chunk_merkle .last_chunk_merkle
@ -964,13 +969,26 @@ impl LogManager {
assert_eq!(pad_data.len(), start_index * ENTRY_SIZE); assert_eq!(pad_data.len(), start_index * ENTRY_SIZE);
} }
let data_size = pad_data.len() / ENTRY_SIZE; let data_size = pad_data.len() / ENTRY_SIZE;
self.sender.send(UpdateFlowMessage { if is_full_empty {
root_map, self.sender.send(UpdateFlowMessage {
pad_data: pad_data.to_vec(), root_map,
tx_start_flow_index, pad_data: pad_data.len(),
})?; tx_start_flow_index,
})?;
} else {
self.flow_store.put_batch_root_list(root_map).unwrap();
// Update the flow database.
// This should be called before `complete_last_chunk_merkle` so that we do not save
// subtrees with data known.
self.flow_store
.append_entries(ChunkArray {
data: pad_data.to_vec(),
start_index: tx_start_flow_index,
})
.unwrap();
}
tx_start_flow_index += data_size as u64; tx_start_flow_index += data_size as u64;
if let Some(index) = completed_chunk_index { if let Some(index) = completed_chunk_index {
self.complete_last_chunk_merkle(index, &mut *merkle)?; self.complete_last_chunk_merkle(index, &mut *merkle)?;
@ -1224,7 +1242,7 @@ pub fn sub_merkle_tree(leaf_data: &[u8]) -> Result<FileMerkleTree> {
pub fn data_to_merkle_leaves(leaf_data: &[u8]) -> Result<Vec<H256>> { pub fn data_to_merkle_leaves(leaf_data: &[u8]) -> Result<Vec<H256>> {
if leaf_data.len() % ENTRY_SIZE != 0 { if leaf_data.len() % ENTRY_SIZE != 0 {
bail!("merkle_tree: unmatch data size"); bail!("merkle_tree: mismatched data size");
} }
// If the data size is small, using `rayon` would introduce more overhead. // If the data size is small, using `rayon` would introduce more overhead.
let r = if leaf_data.len() >= ENTRY_SIZE * 8 { let r = if leaf_data.len() >= ENTRY_SIZE * 8 {
@ -1262,7 +1280,7 @@ fn entry_proof(top_proof: &FlowProof, sub_proof: &FlowProof) -> Result<FlowProof
assert!(lemma.pop().is_some()); assert!(lemma.pop().is_some());
lemma.extend_from_slice(&top_proof.lemma()[1..]); lemma.extend_from_slice(&top_proof.lemma()[1..]);
path.extend_from_slice(top_proof.path()); path.extend_from_slice(top_proof.path());
Ok(FlowProof::new(lemma, path)) FlowProof::new(lemma, path)
} }
pub fn split_nodes(data_size: usize) -> Vec<usize> { pub fn split_nodes(data_size: usize) -> Vec<usize> {

View File

@ -183,23 +183,19 @@ pub trait Store:
LogStoreRead + LogStoreWrite + config::Configurable + Send + Sync + 'static LogStoreRead + LogStoreWrite + config::Configurable + Send + Sync + 'static
{ {
} }
impl< impl<T: LogStoreRead + LogStoreWrite + config::Configurable + Send + Sync + 'static> Store for T {}
T: LogStoreRead + LogStoreWrite + config::Configurable + Send + Sync + 'static,
> Store for T
{
}
pub struct MineLoadChunk { pub struct MineLoadChunk {
// Use `Vec` instead of array to avoid thread stack overflow. // Use `Vec` instead of array to avoid thread stack overflow.
pub loaded_chunk: Vec<[u8; BYTES_PER_SEAL]>, pub loaded_chunk: Vec<[u8; BYTES_PER_SEAL]>,
pub avalibilities: [bool; SEALS_PER_LOAD], pub availabilities: [bool; SEALS_PER_LOAD],
} }
impl Default for MineLoadChunk { impl Default for MineLoadChunk {
fn default() -> Self { fn default() -> Self {
Self { Self {
loaded_chunk: vec![[0u8; BYTES_PER_SEAL]; SEALS_PER_LOAD], loaded_chunk: vec![[0u8; BYTES_PER_SEAL]; SEALS_PER_LOAD],
avalibilities: [false; SEALS_PER_LOAD], availabilities: [false; SEALS_PER_LOAD],
} }
} }
} }