Change miner id logic & request miner id automatically (#60)

* Change miner id logic & request miner id automatically

* Not enable all features in the test workflow.

* Auto configurable mining period

* Adjust test params for ci
This commit is contained in:
Chenxing Li 2024-04-27 11:15:57 +08:00 committed by GitHub
parent 5bcd3602b0
commit 193e154361
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 202 additions and 32 deletions

View File

@ -37,7 +37,7 @@ jobs:
uses: ./.github/actions/setup-rust uses: ./.github/actions/setup-rust
- name: Run unittest - name: Run unittest
run: cargo test --all-features --no-fail-fast run: cargo test --no-fail-fast
env: env:
CARGO_INCREMENTAL: '0' CARGO_INCREMENTAL: '0'
RUSTC_BOOTSTRAP: '1' RUSTC_BOOTSTRAP: '1'

1
.gitignore vendored
View File

@ -5,3 +5,4 @@
tests/**/__pycache__ tests/**/__pycache__
tests/tmp/** tests/tmp/**
.vscode/*.json .vscode/*.json
/0g-storage-contracts-dev

@ -1 +1 @@
Subproject commit d466311abb6f629a6489450f2e684b2c6e7b1089 Subproject commit 6a9f52e8c10ff9b5cd7a5844c543c0951b97d395

View File

@ -11,4 +11,4 @@ ethers = "^2"
serde_json = "1.0.82" serde_json = "1.0.82"
[features] [features]
compile-contracts = [] dev = []

View File

@ -1,3 +1,7 @@
fn main() { fn main() {
if cfg!(not(feature = "dev")) {
println!("cargo:rerun-if-changed=../../0g-storage-contracts/artifacts/"); println!("cargo:rerun-if-changed=../../0g-storage-contracts/artifacts/");
} else {
println!("cargo:rerun-if-changed=../../0g-storage-contracts-dev/artifacts/");
}
} }

View File

@ -2,12 +2,26 @@ use ethers::prelude::abigen;
// run `cargo doc -p contract-interface --open` to read struct definition // run `cargo doc -p contract-interface --open` to read struct definition
#[cfg(not(feature = "dev"))]
abigen!( abigen!(
ZgsFlow, ZgsFlow,
"../../0g-storage-contracts/artifacts/contracts/dataFlow/Flow.sol/Flow.json" "../../0g-storage-contracts/artifacts/contracts/dataFlow/Flow.sol/Flow.json"
); );
#[cfg(not(feature = "dev"))]
abigen!( abigen!(
PoraMine, PoraMine,
"../../0g-storage-contracts/artifacts/contracts/test/PoraMineTest.sol/PoraMineTest.json" "../../0g-storage-contracts/artifacts/contracts/miner/Mine.sol/PoraMine.json"
);
#[cfg(feature = "dev")]
abigen!(
ZgsFlow,
"../../0g-storage-contracts-dev/artifacts/contracts/dataFlow/Flow.sol/Flow.json"
);
#[cfg(feature = "dev")]
abigen!(
PoraMine,
"../../0g-storage-contracts-dev/artifacts/contracts/miner/Mine.sol/PoraMine.json"
); );

View File

@ -8,7 +8,7 @@ use ethers::signers::LocalWallet;
use ethers::signers::Signer; use ethers::signers::Signer;
pub struct MinerConfig { pub struct MinerConfig {
pub(crate) miner_id: H256, pub(crate) miner_id: Option<H256>,
pub(crate) miner_key: H256, pub(crate) miner_key: H256,
pub(crate) rpc_endpoint_url: String, pub(crate) rpc_endpoint_url: String,
pub(crate) mine_address: Address, pub(crate) mine_address: Address,
@ -32,8 +32,7 @@ impl MinerConfig {
cpu_percentage: u64, cpu_percentage: u64,
iter_batch: usize, iter_batch: usize,
) -> Option<MinerConfig> { ) -> Option<MinerConfig> {
match (miner_id, miner_key) { miner_key.map(|miner_key| MinerConfig {
(Some(miner_id), Some(miner_key)) => Some(MinerConfig {
miner_id, miner_id,
miner_key, miner_key,
rpc_endpoint_url, rpc_endpoint_url,
@ -42,9 +41,7 @@ impl MinerConfig {
submission_gas, submission_gas,
cpu_percentage, cpu_percentage,
iter_batch, iter_batch,
}), })
_ => None,
}
} }
pub(crate) async fn make_provider(&self) -> Result<MineServiceMiddleware, String> { pub(crate) async fn make_provider(&self) -> Result<MineServiceMiddleware, String> {

View File

@ -7,6 +7,7 @@ extern crate lazy_static;
mod config; mod config;
mod loader; mod loader;
mod mine; mod mine;
mod miner_id;
pub mod pora; pub mod pora;
mod sealer; mod sealer;
mod service; mod service;
@ -16,4 +17,5 @@ mod watcher;
pub use config::MinerConfig; pub use config::MinerConfig;
pub use loader::PoraLoader; pub use loader::PoraLoader;
pub use mine::CustomMineRange; pub use mine::CustomMineRange;
pub use miner_id::load_miner_id;
pub use service::{MineService, MinerMessage}; pub use service::{MineService, MinerMessage};

View File

@ -82,6 +82,7 @@ impl PoraService {
mine_context_receiver: mpsc::UnboundedReceiver<MineContextMessage>, mine_context_receiver: mpsc::UnboundedReceiver<MineContextMessage>,
loader: Arc<dyn PoraLoader>, loader: Arc<dyn PoraLoader>,
config: &MinerConfig, config: &MinerConfig,
miner_id: H256,
) -> mpsc::UnboundedReceiver<AnswerWithoutProof> { ) -> mpsc::UnboundedReceiver<AnswerWithoutProof> {
let (mine_answer_sender, mine_answer_receiver) = let (mine_answer_sender, mine_answer_receiver) =
mpsc::unbounded_channel::<AnswerWithoutProof>(); mpsc::unbounded_channel::<AnswerWithoutProof>();
@ -95,7 +96,7 @@ impl PoraService {
msg_recv, msg_recv,
puzzle: None, puzzle: None,
mine_range, mine_range,
miner_id: config.miner_id, miner_id,
loader, loader,
cpu_percentage: config.cpu_percentage, cpu_percentage: config.cpu_percentage,
iter_batch: config.iter_batch, iter_batch: config.iter_batch,

111
node/miner/src/miner_id.rs Normal file
View File

@ -0,0 +1,111 @@
use crate::config::MineServiceMiddleware;
use crate::config::MinerConfig;
use contract_interface::{NewMinerIdFilter, PoraMine};
use ethereum_types::Address;
use ethers::contract::ContractCall;
use ethers::contract::EthEvent;
use std::sync::Arc;
use storage::log_store::{config::ConfigurableExt, Store};
use storage::H256;
use tokio::sync::RwLock;
const MINER_ID: &str = "mine.miner_id";
pub fn load_miner_id(store: &dyn Store) -> storage::error::Result<Option<H256>> {
store.get_config_decoded(&MINER_ID)
}
fn set_miner_id(store: &dyn Store, miner_id: &H256) -> storage::error::Result<()> {
store.set_config_encoded(&MINER_ID, miner_id)
}
pub(crate) async fn check_and_request_miner_id(
config: &MinerConfig,
store: &RwLock<dyn Store>,
provider: &Arc<MineServiceMiddleware>,
) -> Result<H256, String> {
let db_miner_id = load_miner_id(&*store.read().await)
.map_err(|e| format!("miner_id on db corrupt: {:?}", e))?;
let mine_contract = PoraMine::new(config.mine_address, provider.clone());
match (db_miner_id, config.miner_id) {
(Some(d_id), Some(c_id)) => {
if d_id != c_id {
Err(format!(
"database miner id {} != configuration miner id {}",
d_id, c_id
))
} else {
Ok(d_id)
}
}
(None, Some(c_id)) => {
check_miner_id(&mine_contract, c_id).await?;
set_miner_id(&*store.write().await, &c_id)
.map_err(|e| format!("set miner id on db corrupt: {:?}", e))?;
Ok(c_id)
}
(Some(d_id), None) => {
check_miner_id(&mine_contract, d_id).await?;
Ok(d_id)
}
(None, None) => {
let beneficiary = provider.address();
let id = request_miner_id(&mine_contract, beneficiary).await?;
set_miner_id(&*store.write().await, &id)
.map_err(|e| format!("set miner id on db corrupt: {:?}", e))?;
Ok(id)
}
}
}
async fn check_miner_id(
mine_contract: &PoraMine<MineServiceMiddleware>,
miner_id: H256,
) -> Result<Address, String> {
debug!("Checking miner id on chain...");
let beneficiary = mine_contract
.beneficiaries(miner_id.0)
.call()
.await
.map_err(|e| format!("Fail to query miner id information: {:?}", e))?;
if beneficiary == Address::zero() {
Err("candidate miner id is not registered".into())
} else {
Ok(beneficiary)
}
}
async fn request_miner_id(
mine_contract: &PoraMine<MineServiceMiddleware>,
beneficiary: Address,
) -> Result<H256, String> {
debug!("Requesting miner id on chain...");
let submission_call: ContractCall<_, _> =
mine_contract.request_miner_id(beneficiary, 0).legacy();
let pending_tx = submission_call
.send()
.await
.map_err(|e| format!("Fail to request miner id: {:?}", e))?;
let receipt = pending_tx
.retries(3)
.await
.map_err(|e| format!("Fail to execute mine answer transaction: {:?}", e))?
.ok_or("Request miner id transaction dropped after 3 retires")?;
let first_log = receipt
.logs
.first()
.ok_or("Fail to find minerId in receipt")?;
let new_id_event = NewMinerIdFilter::decode_log(&first_log.clone().into())
.map_err(|e| format!("Fail to decode NewMinerId event: {:?}", e))?;
Ok(H256(new_id_event.miner_id))
}

View File

@ -34,6 +34,7 @@ impl Sealer {
provider: Arc<MineServiceMiddleware>, provider: Arc<MineServiceMiddleware>,
store: Arc<RwLock<dyn Store>>, store: Arc<RwLock<dyn Store>>,
config: &MinerConfig, config: &MinerConfig,
miner_id: H256,
) { ) {
let flow_contract = ZgsFlow::new(config.flow_address, provider); let flow_contract = ZgsFlow::new(config.flow_address, provider);
let sealer = Sealer { let sealer = Sealer {
@ -41,7 +42,7 @@ impl Sealer {
store, store,
context_cache: Default::default(), context_cache: Default::default(),
last_context_flow_length: 0, last_context_flow_length: 0,
miner_id: config.miner_id, miner_id,
}; };
executor.spawn(async move { Box::pin(sealer.start()).await }, "data_sealer"); executor.spawn(async move { Box::pin(sealer.start()).await }, "data_sealer");

View File

@ -1,3 +1,4 @@
use crate::miner_id::check_and_request_miner_id;
use crate::sealer::Sealer; use crate::sealer::Sealer;
use crate::submitter::Submitter; use crate::submitter::Submitter;
use crate::{config::MinerConfig, mine::PoraService, watcher::MineContextWatcher}; use crate::{config::MinerConfig, mine::PoraService, watcher::MineContextWatcher};
@ -30,6 +31,9 @@ impl MineService {
let (msg_send, msg_recv) = broadcast::channel(1024); let (msg_send, msg_recv) = broadcast::channel(1024);
let miner_id = check_and_request_miner_id(&config, &store, &provider).await?;
debug!("miner id setting complete.");
let mine_context_receiver = MineContextWatcher::spawn( let mine_context_receiver = MineContextWatcher::spawn(
executor.clone(), executor.clone(),
msg_recv.resubscribe(), msg_recv.resubscribe(),
@ -43,6 +47,7 @@ impl MineService {
mine_context_receiver, mine_context_receiver,
Arc::new(store.clone()), Arc::new(store.clone()),
&config, &config,
miner_id,
); );
Submitter::spawn( Submitter::spawn(
@ -53,7 +58,7 @@ impl MineService {
&config, &config,
); );
Sealer::spawn(executor, provider, store, &config); Sealer::spawn(executor, provider, store, &config, miner_id);
debug!("Starting miner service"); debug!("Starting miner service");

View File

@ -97,7 +97,7 @@ impl Submitter {
mine_length: mine_answer.mining_length.into(), mine_length: mine_answer.mining_length.into(),
recall_position: mine_answer.recall_position.into(), recall_position: mine_answer.recall_position.into(),
seal_offset: mine_answer.seal_offset.into(), seal_offset: mine_answer.seal_offset.into(),
sealed_context_digest: sealed_context_digest.digest, // TODO(kevin): wait for implementation of data sealing. sealed_context_digest: sealed_context_digest.digest,
sealed_data: unsafe { std::mem::transmute(mine_answer.sealed_data) }, sealed_data: unsafe { std::mem::transmute(mine_answer.sealed_data) },
merkle_proof: flow_proof_to_pora_merkle_proof(flow_proof), merkle_proof: flow_proof_to_pora_merkle_proof(flow_proof),
}; };

View File

@ -16,6 +16,8 @@ class LongTimeMineTest(TestFramework):
"mine_iter_batch_size": 50, "mine_iter_batch_size": 50,
} }
self.mine_period = 15 self.mine_period = 15
self.launch_wait_seconds = 15
def submit_data(self, item, size): def submit_data(self, item, size):
submissions_before = self.contract.num_submissions() submissions_before = self.contract.num_submissions()

View File

@ -3,6 +3,7 @@ from test_framework.test_framework import TestFramework
from config.node_config import MINER_ID, GENESIS_PRIV_KEY from config.node_config import MINER_ID, GENESIS_PRIV_KEY
from utility.submission import create_submission, submit_data from utility.submission import create_submission, submit_data
from utility.utils import wait_until from utility.utils import wait_until
from test_framework.blockchain_node import BlockChainNodeType
class MineTest(TestFramework): class MineTest(TestFramework):
@ -10,10 +11,10 @@ class MineTest(TestFramework):
self.num_blockchain_nodes = 1 self.num_blockchain_nodes = 1
self.num_nodes = 1 self.num_nodes = 1
self.zgs_node_configs[0] = { self.zgs_node_configs[0] = {
"miner_id": MINER_ID,
"miner_key": GENESIS_PRIV_KEY, "miner_key": GENESIS_PRIV_KEY,
} }
self.mine_period = 15 self.mine_period = int(45 / self.block_time)
self.launch_wait_seconds = 15
def submit_data(self, item, size): def submit_data(self, item, size):
submissions_before = self.contract.num_submissions() submissions_before = self.contract.num_submissions()

View File

@ -3,6 +3,8 @@ from test_framework.test_framework import TestFramework
from config.node_config import MINER_ID, GENESIS_PRIV_KEY from config.node_config import MINER_ID, GENESIS_PRIV_KEY
from utility.submission import create_submission, submit_data from utility.submission import create_submission, submit_data
from utility.utils import wait_until, assert_equal, assert_greater_than from utility.utils import wait_until, assert_equal, assert_greater_than
from test_framework.blockchain_node import BlockChainNodeType
import math import math
@ -13,11 +15,12 @@ class MineTest(TestFramework):
self.num_blockchain_nodes = 1 self.num_blockchain_nodes = 1
self.num_nodes = 1 self.num_nodes = 1
self.zgs_node_configs[0] = { self.zgs_node_configs[0] = {
"miner_id": MINER_ID,
"miner_key": GENESIS_PRIV_KEY, "miner_key": GENESIS_PRIV_KEY,
} }
self.enable_market = True self.enable_market = True
self.mine_period = 20 self.mine_period = int(60 / self.block_time)
self.launch_wait_seconds = 15
def submit_data(self, item, size, no_submit = False): def submit_data(self, item, size, no_submit = False):
submissions_before = self.contract.num_submissions() submissions_before = self.contract.num_submissions()

View File

@ -28,6 +28,13 @@ class BlockChainNodeType(Enum):
BSC = 1 BSC = 1
Evmos = 2 Evmos = 2
def block_time(self):
if self == BlockChainNodeType.Conflux:
return 0.5
elif self == BlockChainNodeType.BSC:
return 0.25
else:
return 3.0
@unique @unique
class NodeType(Enum): class NodeType(Enum):
@ -299,8 +306,8 @@ class BlockchainNode(TestNode):
self.log.debug("Mine deployed") self.log.debug("Mine deployed")
self.log.info("All contracts deployed") self.log.info("All contracts deployed")
tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS) # tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS)
self.wait_for_transaction_receipt(w3, tx_hash) # self.wait_for_transaction_receipt(w3, tx_hash)
dummy_reward_contract = w3.eth.contract( dummy_reward_contract = w3.eth.contract(
address = book.functions.reward().call(), address = book.functions.reward().call(),
@ -328,8 +335,8 @@ class BlockchainNode(TestNode):
self.log.info("All contracts deployed") self.log.info("All contracts deployed")
tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS) # tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS)
self.wait_for_transaction_receipt(w3, tx_hash) # self.wait_for_transaction_receipt(w3, tx_hash)
return flow_contract, flow_contract_hash, mine_contract, reward_contract return flow_contract, flow_contract_hash, mine_contract, reward_contract

View File

@ -47,8 +47,10 @@ class TestFramework:
self.blockchain_node_configs = {} self.blockchain_node_configs = {}
self.zgs_node_configs = {} self.zgs_node_configs = {}
self.blockchain_node_type = blockchain_node_type self.blockchain_node_type = blockchain_node_type
self.block_time = blockchain_node_type.block_time()
self.enable_market = False self.enable_market = False
self.mine_period = 100 self.mine_period = 100
self.launch_wait_seconds = 1
# Set default binary path # Set default binary path
binary_ext = ".exe" if is_windows_platform() else "" binary_ext = ".exe" if is_windows_platform() else ""
@ -203,7 +205,9 @@ class TestFramework:
time.sleep(1) time.sleep(1)
node.start() node.start()
time.sleep(1) self.log.info("Wait the zgs_node launch for %d seconds", self.launch_wait_seconds)
time.sleep(self.launch_wait_seconds)
for node in self.nodes: for node in self.nodes:
node.wait_for_rpc_connection() node.wait_for_rpc_connection()
@ -268,6 +272,10 @@ class TestFramework:
"--tmpdir", dest="tmpdir", help="Root directory for datadirs" "--tmpdir", dest="tmpdir", help="Root directory for datadirs"
) )
parser.add_argument(
"--devdir", dest="devdir", help="A softlink point to the last run"
)
parser.add_argument( parser.add_argument(
"--randomseed", dest="random_seed", type=int, help="Set a random seed" "--randomseed", dest="random_seed", type=int, help="Set a random seed"
) )
@ -433,6 +441,19 @@ class TestFramework:
self.__start_logging() self.__start_logging()
self.log.info("Root dir: %s", self.root_dir) self.log.info("Root dir: %s", self.root_dir)
if self.options.devdir:
dst = self.options.devdir
if os.path.islink(dst):
os.remove(dst)
elif os.path.isdir(dst):
shutil.rmtree(dst)
elif os.path.exists(dst):
os.remove(dst)
os.symlink(self.options.tmpdir, dst)
self.log.info("Symlink: %s", Path(dst).absolute())
if self.blockchain_node_type == BlockChainNodeType.Conflux: if self.blockchain_node_type == BlockChainNodeType.Conflux:
self.blockchain_binary = os.path.abspath(self.options.conflux) self.blockchain_binary = os.path.abspath(self.options.conflux)
elif self.blockchain_node_type == BlockChainNodeType.BSC: elif self.blockchain_node_type == BlockChainNodeType.BSC: