Merge f456773b72 into 03286ebd78

* Add debug log config

* Add config file for ssd and hdd

* Set most default value for ssd and hdd config file

node/src/cli/mod.rs

@@ -3,6 +3,7 @@ use clap::{arg, command, Command};
 pub fn cli_app<'a>() -> Command<'a> {
     command!()
         .arg(arg!(-c --config <FILE> "Sets a custom config file"))
+        .arg(arg!(--"log-config-file" [FILE] "Sets log configuration file (Default: log_config)"))
         .arg(arg!(--"miner-key" [KEY] "Sets miner private key (Default: None)"))
         .arg(
             arg!(--"blockchain-rpc-endpoint" [URL] "Sets blockchain RPC endpoint (Default: http://127.0.0.1:8545)")

run/config-testnet-standard.toml

@@ -0,0 +1,268 @@
+# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+#######################################################################
+###                     Network Config Options                      ###
+#######################################################################
+
+# Data directory where node's keyfile is stored.
+# network_dir = "network"
+
+# IP address to listen on.
+# network_listen_address = "0.0.0.0"
+
+# The address to broadcast to peers about which address we are listening on. Generally,
+# configure public IP address for UDP discovery. If not specified, program will try to
+# detect public IP address automatically.
+# network_enr_address = ""
+
+# The tcp port to broadcast to peers in order to reach back for libp2p services.
+# network_enr_tcp_port = 1234
+
+# The udp port to broadcast to peers in order to reach back for discovery.
+# network_enr_udp_port = 1234
+
+# The TCP port that libp2p listens on.
+# network_libp2p_port = 1234
+
+# UDP port that discovery listens on.
+# network_discovery_port = 1234
+
+# Target number of connected peers.
+# network_target_peers = 50
+
+# List of nodes to bootstrap UDP discovery. Note, `network_enr_address` should be
+# configured as well to enable UDP discovery.
+network_boot_nodes = ["/ip4/35.95.5.134/udp/1234/p2p/16Uiu2HAmFGrDV8wKToa1dd8uh6bz8bSY28n33iRP3pvfeBU6ysCw","/ip4/35.84.189.77/udp/1234/p2p/16Uiu2HAmF7t5iuRoWLMvQVfHbbJr5TFgHo2oU1CDxJm56eLdxRAY"]
+
+# List of libp2p nodes to initially connect to.
+# network_libp2p_nodes = []
+
+# Indicates if the user has set the network to be in private mode. Currently this
+# prevents sending client identifying information over identify.
+# network_private = false
+
+# Disables the discovery protocol from starting.
+# network_disable_discovery = false
+
+#######################################################################
+###                   UDP Discovery Config Options                  ###
+#######################################################################
+# The request timeout for each UDP request.
+# discv5_request_timeout_secs = 5
+
+# The timeout after which a `QueryPeer` in an ongoing query is marked unresponsive.
+# Unresponsive peers don't count towards the parallelism limits for a query.
+# Hence, we may potentially end up making more requests to good peers.
+# discv5_query_peer_timeout_secs = 2
+
+# The number of retries for each UDP request.
+# discv5_request_retries = 1
+
+# The number of peers to request in parallel in a single query.
+# discv5_query_parallelism = 5
+
+# Reports all discovered ENR's when traversing the DHT to the event stream.
+# discv5_report_discovered_peers = false
+
+# Disables the incoming packet filter.
+# discv5_disable_packet_filter = false
+
+# Disable to limit the number of IP addresses from the same
+# /24 subnet in the kbuckets table. This is to mitigate eclipse attacks.
+# discv5_disable_ip_limit = false
+
+#######################################################################
+###                     Log Sync Config Options                     ###
+#######################################################################
+
+# RPC endpoint to sync event logs on EVM compatible blockchain.
+# blockchain_rpc_endpoint = "http://127.0.0.1:8545"
+
+# Flow contract address to sync event logs.
+log_contract_address = "0x0460aA47b41a66694c0a73f667a1b795A5ED3556"
+
+# Block number to sync event logs from blockchain. Generally, this is
+# the block number when flow contract deployed.
+log_sync_start_block_number = 595059
+
+# Number of blocks to confirm a transaction.
+confirmation_block_count = 6
+
+# Maximum number of event logs to poll at a time.
+# log_page_size = 999
+
+# Maximum data size to cache in memory (by default, 100MB).
+# max_cache_data_size = 104857600
+
+# TTL to cache data in memory.
+# cache_tx_seq_ttl = 500
+
+# The number of retries after a RPC request times out.
+# rate_limit_retries = 100
+
+# The nubmer of retries for rate limited responses.
+# timeout_retries = 100
+
+# The duration to wait before retry, in ms.
+# initial_backoff = 500
+
+# The duration between each paginated getLogs RPC call, in ms.
+# This is set to avoid triggering the throttling mechanism in the RPC server.
+# recover_query_delay = 50
+
+# The counter assumed the finalized block behind the latest block.
+# default_finalized_block_count = 100
+
+# Remove finalized block trigger interval.
+# remove_finalized_block_interval_minutes = 30
+
+# Watch_loop (eth_getLogs) trigger interval.
+# watch_loop_wait_time_ms = 500
+
+#######################################################################
+###                     RPC Config Options                          ###
+#######################################################################
+
+# Whether to provide RPC service.
+# rpc_enabled = true
+
+# HTTP server address to bind for public RPC.
+# rpc_listen_address = "0.0.0.0:5678"
+
+# HTTP server address to bind for admin and debug RPC.
+# rpc_listen_address_admin = "127.0.0.1:5679"
+
+# Maximum data size of RPC request body (by default, 100MB).
+# max_request_body_size = 104857600
+
+# Number of chunks for a single segment.
+# rpc_chunks_per_segment = 1024
+
+# Maximum file size that allowed to cache in memory (by default, 10MB).
+# rpc_max_cache_file_size = 10485760
+
+#######################################################################
+###                     Chunk Pool Config Options                   ###
+#######################################################################
+
+# Maximum number of threads to upload segments of a single file simultaneously.
+# chunk_pool_write_window_size = 4
+
+# Maximum data size of cached segment in pool (by default, 4MB).
+# chunk_pool_max_cached_chunks_all = 4194304
+
+# Maximum number of threads to upload segments for all files simultaneously.
+# chunk_pool_max_writings = 16
+
+# Expiration time to cache uploaded segments in memory.
+# chunk_pool_expiration_time_secs = 300
+
+#######################################################################
+###                     DB Config Options                           ###
+#######################################################################
+
+# Directory to store data.
+# db_dir = "db"
+
+#######################################################################
+###                     Misc Config Options                         ###
+#######################################################################
+
+# Log configuration file.
+# log_config_file = "log_config"
+
+# Log directory.
+# log_directory = "log"
+
+#######################################################################
+###                     Mine Config Options                         ###
+#######################################################################
+
+# Mine contract address for incentive.
+mine_contract_address = "0x1785c8683b3c527618eFfF78d876d9dCB4b70285"
+
+# Miner key is used to sign blockchain transaction for incentive.
+# The value should be a hex string of length 64 without 0x prefix.
+#
+# Note, the corresponding address should have enough tokens to pay
+# transaction gas fee.
+# miner_key = ""
+
+#######################################################################
+###                   Sharding Config Options                       ###
+#######################################################################
+# The max number of chunk entries to store in db.
+# Each entry is 256B, so the db size is roughly limited to
+# `256 * db_max_num_sectors` Bytes.
+# If this limit is reached, the node will update its `shard_position`
+# and store only half data.
+#
+db_max_num_sectors = 1000000000
+
+# The format is <shard_id>/<shard_number>, where the shard number is 2^n.
+# This only applies if there is no stored shard config in db.
+# shard_position = "0/2"
+
+reward_contract_address = "0x0496D0817BD8519e0de4894Dc379D35c35275609"
+# The time interval to check if we should half `shard_position` to prune data.
+#
+# prune_check_time_s = 60
+
+# The number of chunk entries to delete in a batch when we prune data.
+#
+# prune_batch_size = 1024
+
+# The time interval to wait between each prune batch deletion to avoid
+# IO resource exhaustion.
+#
+# prune_batch_wait_time_ms = 1000
+
+#######################################################################
+###                   File Sync Config Options                      ###
+#######################################################################
+[sync]
+
+# Enable file sync among peers automatically. When enabled, each node will store
+# all files, and sufficient disk space is required.
+auto_sync_enabled = true
+
+#  Maximum number of files in sync from other peers simultaneously.
+max_sync_files = 32
+
+# Timeout to terminate a file sync when automatically sync from other peers.
+# If timeout, terminated file sync will be triggered later.
+# find_peer_timeout = "10s"
+
+# Enable to start a file sync via RPC (e.g. `admin_startSyncFile`).
+# sync_file_by_rpc_enabled = true
+
+# Enable to start a file sync automatically when a file announcement P2P message received.
+# sync_file_on_announcement_enabled = false
+
+# Maximum threads to sync files in sequence.
+max_sequential_workers = 24
+
+# Maximum threads to sync files randomly.
+# max_random_workers = 4
+
+#######################################################################
+###                File Location Cache Options                      ###
+#######################################################################
+
+# [file_location_cache]
+
+# File location cache is a cache that maintains storage positions of files. 
+# Storage location information is represented by the IP address of the storage node and the timestamp indicating when the node declared that it stores the corresponding file.
+# It has both a global capacity limit and a limit on the capacity for location information of each individual file.
+# When the cache is full, the storage position information with oldest timestamp will be replaced.
+
+# Global cache capacity.
+# max_entries_total = 4096
+
+# Location information capacity for each file.
+# max_entries_per_file = 4
+
+# Validity period of location information.
+# If the timestamp in the storage location information exceeds this duration from the current time, it will be removed from the cache.
+# entry_expiration_time_secs = 3600

run/config-testnet-turbo.toml

@@ -33,7 +33,7 @@
 
 # List of nodes to bootstrap UDP discovery. Note, `network_enr_address` should be
 # configured as well to enable UDP discovery.
-network_boot_nodes = ["/ip4/54.219.26.22/udp/1234/p2p/16Uiu2HAmTVDGNhkHD98zDnJxQWu3i1FL1aFYeh9wiQTNu4pDCgps","/ip4/52.52.127.117/udp/1234/p2p/16Uiu2HAkzRjxK2gorngB1Xq84qDrT4hSVznYDHj6BkbaE4SGx9oS","/ip4/18.167.69.68/udp/1234/p2p/16Uiu2HAm2k6ua2mGgvZ8rTMV8GhpW71aVzkQWy7D37TTDuLCpgmX"]
+network_boot_nodes = ["/ip4/54.219.26.22/udp/1234/p2p/16Uiu2HAmTVDGNhkHD98zDnJxQWu3i1FL1aFYeh9wiQTNu4pDCgps","/ip4/52.52.127.117/udp/1234/p2p/16Uiu2HAkzRjxK2gorngB1Xq84qDrT4hSVznYDHj6BkbaE4SGx9oS","/ip4/18.162.65.205/udp/1234/p2p/16Uiu2HAm2k6ua2mGgvZ8rTMV8GhpW71aVzkQWy7D37TTDuLCpgmX"]
 
 # List of libp2p nodes to initially connect to.
 # network_libp2p_nodes = []
@@ -80,14 +80,14 @@ network_boot_nodes = ["/ip4/54.219.26.22/udp/1234/p2p/16Uiu2HAmTVDGNhkHD98zDnJxQ
 # blockchain_rpc_endpoint = "http://127.0.0.1:8545"
 
 # Flow contract address to sync event logs.
-log_contract_address = "0x8873cc79c5b3b5666535C825205C9a128B1D75F1"
+log_contract_address = "0xbD2C3F0E65eDF5582141C35969d66e34629cC768"
 
 # Block number to sync event logs from blockchain. Generally, this is
 # the block number when flow contract deployed.
-log_sync_start_block_number = 802
+log_sync_start_block_number = 595059
 
 # Number of blocks to confirm a transaction.
-# confirmation_block_count = 12
+confirmation_block_count = 6
 
 # Maximum number of event logs to poll at a time.
 # log_page_size = 999
@@ -180,7 +180,7 @@ log_sync_start_block_number = 802
 #######################################################################
 
 # Mine contract address for incentive.
-mine_contract_address = "0x85F6722319538A805ED5733c5F4882d96F1C7384"
+mine_contract_address = "0x6815F41019255e00D6F34aAB8397a6Af5b6D806f"
 
 # Miner key is used to sign blockchain transaction for incentive.
 # The value should be a hex string of length 64 without 0x prefix.
@@ -198,12 +198,13 @@ mine_contract_address = "0x85F6722319538A805ED5733c5F4882d96F1C7384"
 # If this limit is reached, the node will update its `shard_position`
 # and store only half data.
 #
-# db_max_num_sectors = 1000000000
+db_max_num_sectors = 1000000000
 
 # The format is <shard_id>/<shard_number>, where the shard number is 2^n.
 # This only applies if there is no stored shard config in db.
 # shard_position = "0/2"
 
+reward_contract_address = "0x51998C4d486F406a788B766d93510980ae1f9360"
 # The time interval to check if we should half `shard_position` to prune data.
 #
 # prune_check_time_s = 60
@@ -227,7 +228,7 @@ mine_contract_address = "0x85F6722319538A805ED5733c5F4882d96F1C7384"
 auto_sync_enabled = true
 
 #  Maximum number of files in sync from other peers simultaneously.
-# max_sync_files = 16
+max_sync_files = 32
 
 # Timeout to terminate a file sync when automatically sync from other peers.
 # If timeout, terminated file sync will be triggered later.
@@ -240,7 +241,28 @@ auto_sync_enabled = true
 # sync_file_on_announcement_enabled = false
 
 # Maximum threads to sync files in sequence.
-# max_sequential_workers = 8
+max_sequential_workers = 24
 
 # Maximum threads to sync files randomly.
 # max_random_workers = 4
+
+#######################################################################
+###                File Location Cache Options                      ###
+#######################################################################
+
+# [file_location_cache]
+
+# File location cache is a cache that maintains storage positions of files. 
+# Storage location information is represented by the IP address of the storage node and the timestamp indicating when the node declared that it stores the corresponding file.
+# It has both a global capacity limit and a limit on the capacity for location information of each individual file.
+# When the cache is full, the storage position information with oldest timestamp will be replaced.
+
+# Global cache capacity.
+# max_entries_total = 4096
+
+# Location information capacity for each file.
+# max_entries_per_file = 4
+
+# Validity period of location information.
+# If the timestamp in the storage location information exceeds this duration from the current time, it will be removed from the cache.
+# entry_expiration_time_secs = 3600

run/log_config

@@ -1 +1 @@
-debug,hyper=info,h2=info
+info

run/log_config_debug

@@ -0,0 +1 @@
+debug,hyper=info,h2=info,rpc=info,discv5=info,router=info,jsonrpsee_http_server=info

scripts/update_config.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-MINER_KEY=""
-MINE_CONTRACT=""
-BLOCKCHAIN_RPC=""
-FLOW_CONTRACT=""
-BLOCK_NUMBER=0
-PUBLIC_IP=$(curl -s https://ipinfo.io/ip)
-
-FILE=run/config.toml
-
-# enable sync
-sed -in-place='' 's/# \[sync\]/\[sync\]/g' $FILE
-# enable auto_sync
-sed -in-place='' 's/# auto_sync_enabled = false/auto_sync_enabled = true/g' $FILE
-# reduce timeout for finding peers
-sed -in-place='' 's/# find_peer_timeout = .*/find_peer_timeout = "10s"/g' $FILE
-# set public ip
-sed -in-place='' "s/# network_enr_address = .*/network_enr_address = \"$PUBLIC_IP\"/g" $FILE
-# set miner key
-sed -in-place='' "s/miner_key = \"\"/miner_key = \"$MINER_KEY\"/g" $FILE
-# set miner contract address
-sed -in-place='' "s/mine_contract_address = .*/mine_contract_address = \"$MINE_CONTRACT\"/g" $FILE
-# set blockchain rpc endpoint
-sed -in-place='' "s|blockchain_rpc_endpoint = .*|blockchain_rpc_endpoint = \"$BLOCKCHAIN_RPC\"|g" $FILE
-# set flow contract address
-sed -in-place='' "s/log_contract_address = .*/log_contract_address = \"$FLOW_CONTRACT\"/g" $FILE
-# set contract deployed block number
-sed -in-place='' "s/log_sync_start_block_number = .*/log_sync_start_block_number = $BLOCK_NUMBER/g" $FILE
-# update the boot node ids
-sed -in-place='' 's|network_boot_nodes = .*|network_boot_nodes = ["/ip4/54.219.26.22/udp/1234/p2p/16Uiu2HAmTVDGNhkHD98zDnJxQWu3i1FL1aFYeh9wiQTNu4pDCgps","/ip4/52.52.127.117/udp/1234/p2p/16Uiu2HAkzRjxK2gorngB1Xq84qDrT4hSVznYDHj6BkbaE4SGx9oS","/ip4/18.167.69.68/udp/1234/p2p/16Uiu2HAm2k6ua2mGgvZ8rTMV8GhpW71aVzkQWy7D37TTDuLCpgmX"]|g' $FILE