add grpc for uploading and downloading segments

This commit is contained in:
Peter Zhang 2025-07-01 18:00:03 +08:00
parent 28654efde1
commit a4b0bc1dac
12 changed files with 268 additions and 13 deletions

187
Cargo.lock generated
View File

@ -551,6 +551,34 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "axum"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
dependencies = [
"async-trait",
"axum-core 0.3.4",
"bitflags 1.3.2",
"bytes",
"futures-util",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.29",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite 0.2.14",
"rustversion",
"serde",
"sync_wrapper 0.1.2",
"tower",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.7.5"
@ -558,7 +586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
dependencies = [
"async-trait",
"axum-core",
"axum-core 0.4.5",
"bytes",
"futures-util",
"http 1.2.0",
@ -578,6 +606,23 @@ dependencies = [
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http 0.2.12",
"http-body 0.4.6",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.4.5"
@ -682,7 +727,7 @@ dependencies = [
"lazy_static",
"lazycell",
"peeking_take_while",
"prettyplease",
"prettyplease 0.2.20",
"proc-macro2",
"quote",
"regex",
@ -1189,7 +1234,7 @@ dependencies = [
"futures-core",
"prost 0.13.4",
"prost-types 0.13.4",
"tonic",
"tonic 0.12.3",
"tracing-core",
]
@ -1213,7 +1258,7 @@ dependencies = [
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tonic 0.12.3",
"tracing",
"tracing-core",
"tracing-subscriber",
@ -2293,7 +2338,7 @@ dependencies = [
"ethers-core",
"ethers-etherscan",
"eyre",
"prettyplease",
"prettyplease 0.2.20",
"proc-macro2",
"quote",
"regex",
@ -3435,6 +3480,18 @@ dependencies = [
"tracing",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper 0.14.29",
"pin-project-lite 0.2.14",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "hyper-timeout"
version = "0.5.2"
@ -6104,6 +6161,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "prettyplease"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
dependencies = [
"proc-macro2",
"syn 1.0.109",
]
[[package]]
name = "prettyplease"
version = "0.2.20"
@ -6277,6 +6344,16 @@ dependencies = [
"prost-derive 0.10.1",
]
[[package]]
name = "prost"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
dependencies = [
"bytes",
"prost-derive 0.11.9",
]
[[package]]
name = "prost"
version = "0.13.4"
@ -6329,6 +6406,28 @@ dependencies = [
"which",
]
[[package]]
name = "prost-build"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270"
dependencies = [
"bytes",
"heck 0.4.1",
"itertools 0.10.5",
"lazy_static",
"log",
"multimap",
"petgraph",
"prettyplease 0.1.25",
"prost 0.11.9",
"prost-types 0.11.9",
"regex",
"syn 1.0.109",
"tempfile",
"which",
]
[[package]]
name = "prost-codec"
version = "0.1.0"
@ -6368,6 +6467,19 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "prost-derive"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
dependencies = [
"anyhow",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "prost-derive"
version = "0.13.4"
@ -6401,6 +6513,15 @@ dependencies = [
"prost 0.10.4",
]
[[package]]
name = "prost-types"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
dependencies = [
"prost 0.11.9",
]
[[package]]
name = "prost-types"
version = "0.13.4"
@ -6866,7 +6987,7 @@ dependencies = [
[[package]]
name = "rpc"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"append_merkle",
"base64 0.13.1",
@ -6881,6 +7002,9 @@ dependencies = [
"miner",
"network",
"parking_lot 0.12.3",
"prost 0.11.9",
"prost-build 0.11.9",
"prost-types 0.11.9",
"serde",
"serde_json",
"shared_types",
@ -6889,6 +7013,8 @@ dependencies = [
"sync",
"task_executor",
"tokio",
"tonic 0.9.2",
"tonic-build",
"tracing",
]
@ -8188,6 +8314,34 @@ dependencies = [
"winnow 0.6.13",
]
[[package]]
name = "tonic"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
dependencies = [
"async-trait",
"axum 0.6.20",
"base64 0.21.7",
"bytes",
"futures-core",
"futures-util",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.29",
"hyper-timeout 0.4.1",
"percent-encoding",
"pin-project 1.1.5",
"prost 0.11.9",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tonic"
version = "0.12.3"
@ -8196,7 +8350,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
dependencies = [
"async-stream",
"async-trait",
"axum",
"axum 0.7.5",
"base64 0.22.1",
"bytes",
"h2 0.4.7",
@ -8204,7 +8358,7 @@ dependencies = [
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.2",
"hyper-timeout",
"hyper-timeout 0.5.2",
"hyper-util",
"percent-encoding",
"pin-project 1.1.5",
@ -8218,6 +8372,19 @@ dependencies = [
"tracing",
]
[[package]]
name = "tonic-build"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07"
dependencies = [
"prettyplease 0.1.25",
"proc-macro2",
"prost-build 0.11.9",
"quote",
"syn 1.0.109",
]
[[package]]
name = "tower"
version = "0.4.13"
@ -8246,9 +8413,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"

View File

@ -1,7 +1,8 @@
[package]
name = "rpc"
version = "0.1.0"
version = "0.2.0"
edition = "2021"
build = "build.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -28,3 +29,10 @@ merkle_tree = { path = "../../common/merkle_tree"}
futures-channel = "^0.3"
metrics = { workspace = true }
parking_lot = "0.12.3"
tonic = { version = "0.9.2", features = ["transport"] }
prost = "0.11.9"
prost-types = "0.11.9"
[build-dependencies]
tonic-build = "0.9.2"
prost-build = "0.11.9"

5
node/rpc/build.rs Normal file
View File

@ -0,0 +1,5 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
// Compile proto/my_service.proto
tonic_build::compile_protos("proto/zgs_grpc.proto")?;
Ok(())
}

View File

@ -0,0 +1,16 @@
syntax = "proto3";
package zgs_grpc;
// A trivial ping service
service ZgsGrpcService {
rpc Ping (PingRequest) returns (PingReply);
}
message PingRequest {
string message = 1;
}
message PingReply {
string message = 1;
}

View File

@ -8,6 +8,7 @@ pub struct Config {
pub enabled: bool,
pub listen_address: SocketAddr,
pub listen_address_admin: SocketAddr,
pub listen_address_grpc: SocketAddr,
pub chunks_per_segment: usize,
pub max_request_body_size: u32,
pub max_cache_file_size: usize,
@ -19,6 +20,7 @@ impl Default for Config {
enabled: true,
listen_address: SocketAddr::from_str("0.0.0.0:5678").unwrap(),
listen_address_admin: SocketAddr::from_str("127.0.0.1:5679").unwrap(),
listen_address_grpc: SocketAddr::from_str("0.0.0.0:50051").unwrap(),
chunks_per_segment: 1024,
max_request_body_size: 100 * 1024 * 1024, // 100MB
max_cache_file_size: 10 * 1024 * 1024, // 10MB

View File

@ -12,6 +12,8 @@ pub mod types;
mod zgs;
use crate::miner::RpcServer as MinerRpcServer;
use crate::zgs_grpc::zgs_grpc::ZgsGrpcServiceImpl;
use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcServiceServer;
use admin::RpcServer as AdminRpcServer;
use chunk_pool::MemoryChunkPool;
use file_location_cache::FileLocationCache;
@ -33,6 +35,14 @@ pub use config::Config as RPCConfig;
pub use miner::RpcClient as ZgsMinerRpcClient;
pub use zgs::RpcClient as ZgsRPCClient;
pub mod zgs_grpc_proto {
tonic::include_proto!("zgs_grpc");
}
mod zgs_grpc;
use tonic::transport::Server;
/// A wrapper around all the items required to spawn the HTTP server.
///
/// The server will gracefully handle the case where any fields are `None`.
@ -73,7 +83,7 @@ pub async fn run_server(
(run_server_all(ctx).await?, None)
};
info!("Server started");
info!("Rpc Server started");
Ok(handles)
}
@ -133,3 +143,14 @@ async fn run_server_public_private(
Ok((handle_public, Some(handle_private)))
}
pub async fn run_grpc_server(ctx: Context) -> Result<(), Box<dyn Error>> {
let grpc_addr = ctx.config.listen_address_grpc;
let server = ZgsGrpcServiceServer::new(ZgsGrpcServiceImpl { ctx });
Server::builder()
.add_service(server)
.serve(grpc_addr)
.await?;
Ok(())
}

View File

@ -0,0 +1 @@
pub mod zgs_grpc;

View File

@ -0,0 +1,19 @@
use crate::zgs_grpc_proto::{PingRequest, PingReply};
use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcService;
use crate::Context;
pub struct ZgsGrpcServiceImpl {
pub ctx: Context,
}
#[tonic::async_trait]
impl ZgsGrpcService for ZgsGrpcServiceImpl {
async fn ping(
&self,
request: tonic::Request<PingRequest>,
) -> Result<tonic::Response<PingReply>, tonic::Status> {
let msg = request.into_inner().message;
let reply = PingReply { message: format!("Echo: {}", msg) };
Ok(tonic::Response::new(reply))
}
}

View File

@ -298,7 +298,7 @@ impl ClientBuilder {
mine_service_sender: mine_send,
};
let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx)
let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx.clone())
.await
.map_err(|e| format!("Unable to start HTTP RPC server: {:?}", e))?;
@ -307,6 +307,13 @@ impl ClientBuilder {
executor.spawn(admin_rpc_handle, "rpc_admin");
}
executor.spawn(
async move {
rpc::run_grpc_server(ctx.clone()).await.expect("Failed to start gRPC server");
},
"grpc",
);
Ok(self)
}

View File

@ -305,6 +305,9 @@ auto_sync_enabled = true
# HTTP server address to bind for admin and debug RPC.
# listen_address_admin = "127.0.0.1:5679"
## Grpc server address to bind
# listen_address_grpc = "0.0.0.0:50051"
# Number of chunks for a single segment.
# chunks_per_segment = 1024

View File

@ -317,6 +317,9 @@ auto_sync_enabled = true
# HTTP server address to bind for admin and debug RPC.
# listen_address_admin = "127.0.0.1:5679"
## Grpc server address to bind
# listen_address_grpc = "0.0.0.0:50051"
# Number of chunks for a single segment.
# chunks_per_segment = 1024

View File

@ -319,6 +319,9 @@
# HTTP server address to bind for admin and debug RPC.
# listen_address_admin = "127.0.0.1:5679"
## Grpc server address to bind
# listen_address_grpc = "0.0.0.0:50051"
# Number of chunks for a single segment.
# chunks_per_segment = 1024