smeb_y/0g-storage-node
1
0
Fork 0
mirror of https://github.com/0glabs/0g-storage-node.git synced 2024-11-15 04:25:19 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

add more metrics for serial file sync

Browse Source
This commit is contained in:
boqiu 2024-08-20 18:50:17 +08:00
parent 914fc785de
commit 4aae302df5
2 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
node/sync/src/controllers/metrics.rs
View File

@ -1,8 +1,10 @@
use std::sync::Arc; use std::sync::Arc;
use metrics::{register_timer, Timer}; use metrics::{register_timer, Counter, CounterUsize, Histogram, Sample, Timer};
lazy_static::lazy_static! { lazy_static::lazy_static! {
pub static ref SERIAL_SYNC_FILE_COMPLETED: Arc<dyn Timer> = register_timer("sync_controllers_serial_sync_file_completed"); pub static ref SERIAL_SYNC_FILE_COMPLETED: Arc<dyn Timer> = register_timer("sync_controllers_serial_sync_file_completed");
pub static ref SERIAL_SYNC_CHUNKS_COMPLETED: Arc<dyn Timer> = register_timer("sync_controllers_serial_sync_chunks_completed"); pub static ref SERIAL_SYNC_CHUNKS_COMPLETED: Arc<dyn Timer> = register_timer("sync_controllers_serial_sync_chunks_completed");
pub static ref SERIAL_SYNC_SEGMENT_LATENCY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_controllers_serial_sync_segment_latency", 1024);
pub static ref SERIAL_SYNC_UNEXPECTED_ERRORS: Arc<dyn Counter<usize>> = CounterUsize::register("sync_controllers_serial_sync_unexpected_errors");
} }

8
node/sync/src/controllers/serial.rs
View File

@ -428,6 +428,7 @@ impl SerialSyncController {
let data_len = response.chunks.data.len(); let data_len = response.chunks.data.len();
if data_len == 0 || data_len % CHUNK_SIZE > 0 { if data_len == 0 || data_len % CHUNK_SIZE > 0 {
warn!(%from_peer_id, %self.tx_seq, %data_len, "Invalid chunk response data length"); warn!(%from_peer_id, %self.tx_seq, %data_len, "Invalid chunk response data length");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.ban_peer(from_peer_id, "Invalid chunk response data length"); self.ban_peer(from_peer_id, "Invalid chunk response data length");
self.state = SyncState::Idle; self.state = SyncState::Idle;
return; return;
@ -465,6 +466,7 @@ impl SerialSyncController {
} }
Err(err) => { Err(err) => {
warn!(%err, %self.tx_seq, "Failed to validate chunks response"); warn!(%err, %self.tx_seq, "Failed to validate chunks response");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.ban_peer(from_peer_id, "Chunk array validation failed"); self.ban_peer(from_peer_id, "Chunk array validation failed");
self.state = SyncState::Idle; self.state = SyncState::Idle;
return; return;
@ -473,6 +475,8 @@ impl SerialSyncController {
self.failures = 0; self.failures = 0;
metrics::SERIAL_SYNC_SEGMENT_LATENCY.update_since(since.0);
let shard_config = self.store.get_store().flow().get_shard_config(); let shard_config = self.store.get_store().flow().get_shard_config();
let next_chunk = shard_config.next_segment_index( let next_chunk = shard_config.next_segment_index(
(from_chunk / PORA_CHUNK_SIZE as u64) as usize, (from_chunk / PORA_CHUNK_SIZE as u64) as usize,
@ -487,6 +491,7 @@ impl SerialSyncController {
Ok(true) => self.next_chunk = next_chunk as u64, Ok(true) => self.next_chunk = next_chunk as u64,
Ok(false) => { Ok(false) => {
warn!(%self.tx_seq, ?self.tx_id, "Transaction reverted while storing chunks"); warn!(%self.tx_seq, ?self.tx_id, "Transaction reverted while storing chunks");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.state = SyncState::Failed { self.state = SyncState::Failed {
reason: FailureReason::TxReverted(self.tx_id), reason: FailureReason::TxReverted(self.tx_id),
}; };
@ -494,6 +499,7 @@ impl SerialSyncController {
} }
Err(err) => { Err(err) => {
error!(%err, %self.tx_seq, "Unexpected DB error while storing chunks"); error!(%err, %self.tx_seq, "Unexpected DB error while storing chunks");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.state = SyncState::Failed { self.state = SyncState::Failed {
reason: FailureReason::DBError(err.to_string()), reason: FailureReason::DBError(err.to_string()),
}; };
@ -527,12 +533,14 @@ impl SerialSyncController {
} }
Ok(false) => { Ok(false) => {
warn!(?self.tx_id, %self.tx_seq, "Transaction reverted during finalize_tx"); warn!(?self.tx_id, %self.tx_seq, "Transaction reverted during finalize_tx");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.state = SyncState::Failed { self.state = SyncState::Failed {
reason: FailureReason::TxReverted(self.tx_id), reason: FailureReason::TxReverted(self.tx_id),
}; };
} }
Err(err) => { Err(err) => {
error!(%err, %self.tx_seq, "Unexpected error during finalize_tx"); error!(%err, %self.tx_seq, "Unexpected error during finalize_tx");
metrics::SERIAL_SYNC_UNEXPECTED_ERRORS.inc(1);
self.state = SyncState::Failed { self.state = SyncState::Failed {
reason: FailureReason::DBError(err.to_string()), reason: FailureReason::DBError(err.to_string()),
}; };