Remove cli build from server build workflow (#54)

* Remove cli build from server build workflow

* Build cli for a single test
This commit is contained in:
Chenxing Li 2024-04-23 14:35:24 +08:00 committed by GitHub
parent 95485c5c35
commit c85fe38c59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 77 additions and 72 deletions

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "0g-storage-contracts"] [submodule "0g-storage-contracts"]
path = 0g-storage-contracts path = 0g-storage-contracts
url = https://github.com/0glabs/0g-storage-contracts.git url = https://github.com/0glabs/0g-storage-contracts.git
[submodule "0g-storage-client"]
path = 0g-storage-client
url = https://github.com/0glabs/0g-storage-client.git

@ -1 +0,0 @@
Subproject commit 1d09ec4f0b9c27428b2357de46b66e8c231b74df

View File

@ -1,35 +1,3 @@
use std::process::Command;
const INSTALL_ERROR_MESSAGE: &str =
"Install dependencies for contract fail, try to run `yarn` in folder '0g-storage-contracts'";
const COMPILE_ERROR_MESSAGE: &str =
"Compile solidity contracts fail, try to run `yarn compile` in folder '0g-storage-contracts'";
#[allow(dead_code)]
fn compile_contracts() {
println!("cargo:rerun-if-changed=../../0g-storage-contracts/contracts/");
println!("cargo:rerun-if-changed=../../0g-storage-contracts/hardhat.config.ts");
let output = Command::new("yarn")
.arg("--cwd")
.arg("../../0g-storage-contracts")
.status()
.expect(INSTALL_ERROR_MESSAGE);
assert!(output.success(), "{}", INSTALL_ERROR_MESSAGE);
let output = Command::new("yarn")
.arg("--cwd")
.arg("../../0g-storage-contracts")
.arg("compile")
.status()
.expect(COMPILE_ERROR_MESSAGE);
assert!(output.success(), "{}", COMPILE_ERROR_MESSAGE);
}
fn main() { fn main() {
if cfg!(feature = "compile-contracts") {
// compile_contracts();
// return;
}
println!("cargo:rerun-if-changed=../../0g-storage-contracts/artifacts/"); println!("cargo:rerun-if-changed=../../0g-storage-contracts/artifacts/");
} }

View File

@ -2,7 +2,6 @@
name = "zgs_node" name = "zgs_node"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
build = "build.rs"
[dependencies] [dependencies]
anyhow = { version = "1.0.58", features = ["backtrace"] } anyhow = { version = "1.0.58", features = ["backtrace"] }

View File

@ -1,13 +0,0 @@
use std::process::Command;
fn main() {
println!("cargo:rerun-if-changed=../0g-storage-client");
let status = Command::new("go")
.current_dir("../0g-storage-client")
.args(vec!["build", "-o", "../target"])
.status()
.unwrap();
println!("build 0g-storage-client with status {}", status);
}

View File

@ -10,6 +10,7 @@ import sys
import tempfile import tempfile
import time import time
import traceback import traceback
from pathlib import Path
from eth_utils import encode_hex from eth_utils import encode_hex
from test_framework.bsc_node import BSCNode from test_framework.bsc_node import BSCNode
@ -19,6 +20,7 @@ from test_framework.blockchain_node import BlockChainNodeType
from test_framework.conflux_node import ConfluxNode, connect_sample_nodes from test_framework.conflux_node import ConfluxNode, connect_sample_nodes
from test_framework.evmos_node import EvmosNode, evmos_init_genesis from test_framework.evmos_node import EvmosNode, evmos_init_genesis
from utility.utils import PortMin, is_windows_platform, wait_until from utility.utils import PortMin, is_windows_platform, wait_until
from utility.build_binary import build_cli
__file_path__ = os.path.dirname(os.path.realpath(__file__)) __file_path__ = os.path.dirname(os.path.realpath(__file__))
@ -65,7 +67,7 @@ class TestFramework:
root_dir, "target", "release", "zgs_node" + binary_ext root_dir, "target", "release", "zgs_node" + binary_ext
) )
self.__default_zgs_cli_binary__ = os.path.join( self.__default_zgs_cli_binary__ = os.path.join(
root_dir, "target", "0g-storage-client" + binary_ext tests_dir, "tmp", "0g-storage-client" + binary_ext
) )
def __setup_blockchain_node(self): def __setup_blockchain_node(self):
@ -314,6 +316,15 @@ class TestFramework:
self.log.addHandler(fh) self.log.addHandler(fh)
self.log.addHandler(ch) self.log.addHandler(ch)
def _check_cli_binary(self):
if Path(self.cli_binary).absolute() == Path(self.__default_zgs_cli_binary__).absolute() and not os.path.exists(self.cli_binary):
dir = Path(self.cli_binary).parent.absolute()
build_cli(dir)
assert os.path.exists(self.cli_binary), (
"zgs CLI binary not found: %s" % self.cli_binary
)
def _upload_file_use_cli( def _upload_file_use_cli(
self, self,
blockchain_node_rpc_url, blockchain_node_rpc_url,
@ -322,9 +333,8 @@ class TestFramework:
ionion_node_rpc_url, ionion_node_rpc_url,
file_to_upload, file_to_upload,
): ):
assert os.path.exists(self.cli_binary), ( self._check_cli_binary()
"zgs CLI binary not found: %s" % self.cli_binary
)
upload_args = [ upload_args = [
self.cli_binary, self.cli_binary,
"upload", "upload",

View File

@ -7,10 +7,18 @@ import platform
from utility.utils import is_windows_platform, wait_until from utility.utils import is_windows_platform, wait_until
CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client"
EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98"
CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df"
def build_conflux(dir: str) -> bool: def build_conflux(dir: str) -> bool:
return __build_from_github( # Build conflux binary if absent
build_from_github(
dir=dir, dir=dir,
binary_name="conflux.exe" if is_windows_platform() else "conflux", binary_name=CONFLUX_BINARY,
github_url="https://github.com/Conflux-Chain/conflux-rust.git", github_url="https://github.com/Conflux-Chain/conflux-rust.git",
build_cmd="cargo build --release --bin conflux", build_cmd="cargo build --release --bin conflux",
compiled_relative_path=["target", "release"], compiled_relative_path=["target", "release"],
@ -35,24 +43,38 @@ def build_bsc(dir: str) -> bool:
) )
def build_evmos(dir: str) -> bool: def build_evmos(dir: str) -> bool:
return __build_from_github( # Build evmos binary if absent
build_from_github(
dir=dir, dir=dir,
binary_name="evmosd.exe" if is_windows_platform() else "evmosd", binary_name=EVMOS_BINARY,
github_url="-b testnet https://github.com/0glabs/0g-evmos.git", github_url="https://github.com/0glabs/0g-evmos.git",
git_rev=EVMOS_GIT_REV,
build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .", build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
compiled_relative_path=[], compiled_relative_path=[],
) )
def build_cli(dir: str) -> bool:
# Build 0g-storage-client binary if absent
build_from_github(
dir=dir,
binary_name=CLIENT_BINARY,
github_url="https://github.com/0glabs/0g-storage-client.git",
git_rev=CLI_GIT_REV,
build_cmd="go build",
compiled_relative_path=[],
)
def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool: def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
if not os.path.exists(dir): if git_rev is not None:
os.makedirs(dir, exist_ok=True) versioned_binary_name = f"{binary_name}_{git_rev}"
else:
binary_path = os.path.join(dir, binary_name) versioned_binary_name = binary_name
if os.path.exists(binary_path):
return False
print("Begin to build binary from github: %s" % binary_name, flush=True) binary_path = os.path.join(dir, binary_name)
versioned_binary_path = os.path.join(dir, versioned_binary_name)
if os.path.exists(versioned_binary_path):
create_sym_link(versioned_binary_name, binary_name, dir)
return False
start_time = time.time() start_time = time.time()
@ -61,17 +83,19 @@ def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd:
code_tmp_dir = os.path.join(dir, code_tmp_dir_name) code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
if os.path.exists(code_tmp_dir): if os.path.exists(code_tmp_dir):
shutil.rmtree(code_tmp_dir) shutil.rmtree(code_tmp_dir)
clone_command = "git clone " + github_url + " " + code_tmp_dir os.system(f"git clone {github_url} {code_tmp_dir}")
os.system(clone_command)
# build binary # build binary
origin_path = os.getcwd() origin_path = os.getcwd()
os.chdir(code_tmp_dir) os.chdir(code_tmp_dir)
if git_rev is not None:
os.system(f"git checkout {git_rev}")
os.system(build_cmd) os.system(build_cmd)
# copy compiled binary to right place # copy compiled binary to right place
compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name) compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
shutil.copyfile(compiled_binary, binary_path) shutil.copyfile(compiled_binary, versioned_binary_path)
create_sym_link(versioned_binary_name, binary_name, dir)
if not is_windows_platform(): if not is_windows_platform():
st = os.stat(binary_path) st = os.stat(binary_path)
@ -81,10 +105,28 @@ def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd:
shutil.rmtree(code_tmp_dir, ignore_errors=True) shutil.rmtree(code_tmp_dir, ignore_errors=True)
print("Completed to build binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True) print("Completed to build binary " + binary_name + ", Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
return True return True
def create_sym_link(src, dst, path = None):
if src == dst:
return
origin_path = os.getcwd()
if path is not None:
os.chdir(path)
if os.path.exists(dst):
if os.path.isdir(dst):
shutil.rmtree(dst)
else:
os.remove(dst)
os.symlink(src, dst)
os.chdir(origin_path)
def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool: def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool:
if not os.path.exists(dir): if not os.path.exists(dir):
os.makedirs(dir, exist_ok=True) os.makedirs(dir, exist_ok=True)

View File

@ -6,7 +6,7 @@ import sys
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from utility.build_binary import build_conflux, build_bsc, build_evmos from utility.build_binary import build_conflux, build_bsc, build_evmos, build_cli
DEFAULT_PORT_MIN = 11000 DEFAULT_PORT_MIN = 11000
DEFAULT_PORT_MAX = 65535 DEFAULT_PORT_MAX = 65535
@ -59,10 +59,12 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
if not os.path.exists(tmp_dir): if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir, exist_ok=True) os.makedirs(tmp_dir, exist_ok=True)
# Build blockchain binaries if absent # Build blockchain binaries if absent
build_conflux(tmp_dir) build_conflux(tmp_dir)
build_bsc(tmp_dir) build_bsc(tmp_dir)
build_evmos(tmp_dir) build_evmos(tmp_dir)
build_cli(tmp_dir)
start_time = time.time() start_time = time.time()
@ -137,3 +139,4 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
for c in failed: for c in failed:
print(c) print(c)
sys.exit(1) sys.exit(1)