Remove cli build from server build workflow (#54)

* Remove cli build from server build workflow

* Build cli for a single test

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "0g-storage-contracts"]
 	path = 0g-storage-contracts
 	url = https://github.com/0glabs/0g-storage-contracts.git
-[submodule "0g-storage-client"]
-	path = 0g-storage-client
-	url = https://github.com/0glabs/0g-storage-client.git

0g-storage-client

@@ -1 +0,0 @@
-Subproject commit 1d09ec4f0b9c27428b2357de46b66e8c231b74df

common/contract-interface/build.rs

@@ -1,35 +1,3 @@
-use std::process::Command;
-
-const INSTALL_ERROR_MESSAGE: &str =
-    "Install dependencies for contract fail, try to run `yarn` in folder '0g-storage-contracts'";
-const COMPILE_ERROR_MESSAGE: &str =
-    "Compile solidity contracts fail, try to run `yarn compile` in folder '0g-storage-contracts'";
-
-#[allow(dead_code)]
-fn compile_contracts() {
-    println!("cargo:rerun-if-changed=../../0g-storage-contracts/contracts/");
-    println!("cargo:rerun-if-changed=../../0g-storage-contracts/hardhat.config.ts");
-
-    let output = Command::new("yarn")
-        .arg("--cwd")
-        .arg("../../0g-storage-contracts")
-        .status()
-        .expect(INSTALL_ERROR_MESSAGE);
-    assert!(output.success(), "{}", INSTALL_ERROR_MESSAGE);
-
-    let output = Command::new("yarn")
-        .arg("--cwd")
-        .arg("../../0g-storage-contracts")
-        .arg("compile")
-        .status()
-        .expect(COMPILE_ERROR_MESSAGE);
-    assert!(output.success(), "{}", COMPILE_ERROR_MESSAGE);
-}
-
 fn main() {
-    if cfg!(feature = "compile-contracts") {
-        // compile_contracts();
-        // return;
-    }
     println!("cargo:rerun-if-changed=../../0g-storage-contracts/artifacts/");
 }

node/Cargo.toml

@@ -2,7 +2,6 @@
 name = "zgs_node"
 version = "0.1.0"
 edition = "2021"
-build = "build.rs"
 
 [dependencies]
 anyhow = { version = "1.0.58", features = ["backtrace"] }

node/build.rs

@@ -1,13 +0,0 @@
-use std::process::Command;
-
-fn main() {
-    println!("cargo:rerun-if-changed=../0g-storage-client");
-
-    let status = Command::new("go")
-        .current_dir("../0g-storage-client")
-        .args(vec!["build", "-o", "../target"])
-        .status()
-        .unwrap();
-
-    println!("build 0g-storage-client with status {}", status);
-}

tests/test_framework/test_framework.py

@@ -10,6 +10,7 @@ import sys
 import tempfile
 import time
 import traceback
+from pathlib import Path
 
 from eth_utils import encode_hex
 from test_framework.bsc_node import BSCNode
@@ -19,6 +20,7 @@ from test_framework.blockchain_node import BlockChainNodeType
 from test_framework.conflux_node import ConfluxNode, connect_sample_nodes
 from test_framework.evmos_node import EvmosNode, evmos_init_genesis
 from utility.utils import PortMin, is_windows_platform, wait_until
+from utility.build_binary import build_cli
 
 __file_path__ = os.path.dirname(os.path.realpath(__file__))
 
@@ -65,7 +67,7 @@ class TestFramework:
             root_dir, "target", "release", "zgs_node" + binary_ext
         )
         self.__default_zgs_cli_binary__ = os.path.join(
-            root_dir, "target", "0g-storage-client"  + binary_ext
+            tests_dir, "tmp", "0g-storage-client"  + binary_ext
         )
 
     def __setup_blockchain_node(self):
@@ -314,6 +316,15 @@ class TestFramework:
         self.log.addHandler(fh)
         self.log.addHandler(ch)
 
+    def _check_cli_binary(self):
+        if Path(self.cli_binary).absolute() == Path(self.__default_zgs_cli_binary__).absolute() and not os.path.exists(self.cli_binary):
+            dir = Path(self.cli_binary).parent.absolute()
+            build_cli(dir)
+        
+        assert os.path.exists(self.cli_binary), (
+            "zgs CLI binary not found: %s" % self.cli_binary
+        )
+
     def _upload_file_use_cli(
         self,
         blockchain_node_rpc_url,
@@ -322,9 +333,8 @@ class TestFramework:
         ionion_node_rpc_url,
         file_to_upload,
     ):
-        assert os.path.exists(self.cli_binary), (
+        self._check_cli_binary()
-            "zgs CLI binary not found: %s" % self.cli_binary
+        
-        )
         upload_args = [
             self.cli_binary,
             "upload",

tests/utility/build_binary.py

@@ -7,10 +7,18 @@ import platform
 
 from utility.utils import is_windows_platform, wait_until
 
+CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
+EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
+CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client"
+
+EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98"
+CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df"
+
 def build_conflux(dir: str) -> bool:
-    return __build_from_github(
+    # Build conflux binary if absent
+    build_from_github(
         dir=dir,
-        binary_name="conflux.exe" if is_windows_platform() else "conflux",
+        binary_name=CONFLUX_BINARY,
         github_url="https://github.com/Conflux-Chain/conflux-rust.git",
         build_cmd="cargo build --release --bin conflux",
         compiled_relative_path=["target", "release"],
@@ -35,25 +43,39 @@ def build_bsc(dir: str) -> bool:
     )
 
 def build_evmos(dir: str) -> bool:
-    return __build_from_github(
+    # Build evmos binary if absent
+    build_from_github(
         dir=dir,
-        binary_name="evmosd.exe" if is_windows_platform() else "evmosd",
+        binary_name=EVMOS_BINARY,
-        github_url="-b testnet https://github.com/0glabs/0g-evmos.git",
+        github_url="https://github.com/0glabs/0g-evmos.git",
+        git_rev=EVMOS_GIT_REV,
         build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
         compiled_relative_path=[],
     )
 
+def build_cli(dir: str) -> bool:
+    # Build 0g-storage-client binary if absent
+    build_from_github(
+        dir=dir,
+        binary_name=CLIENT_BINARY,
+        github_url="https://github.com/0glabs/0g-storage-client.git",
+        git_rev=CLI_GIT_REV,
+        build_cmd="go build",
+        compiled_relative_path=[],
+    )
 
-def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool:
+def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
-    if not os.path.exists(dir):
+    if git_rev is not None:
-        os.makedirs(dir, exist_ok=True)
+        versioned_binary_name = f"{binary_name}_{git_rev}"
+    else:
+        versioned_binary_name = binary_name
     
     binary_path = os.path.join(dir, binary_name)
-    if os.path.exists(binary_path):
+    versioned_binary_path = os.path.join(dir, versioned_binary_name)
+    if os.path.exists(versioned_binary_path):
+        create_sym_link(versioned_binary_name, binary_name, dir)
         return False
     
-    print("Begin to build binary from github: %s" % binary_name, flush=True)
-    
     start_time = time.time()
     
     # clone code from github to a temp folder
@@ -61,17 +83,19 @@ def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd:
     code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
     if os.path.exists(code_tmp_dir):
         shutil.rmtree(code_tmp_dir)
-    clone_command = "git clone " + github_url + " " + code_tmp_dir
+    os.system(f"git clone {github_url} {code_tmp_dir}")
-    os.system(clone_command)
 
     # build binary
     origin_path = os.getcwd()
     os.chdir(code_tmp_dir)
+    if git_rev is not None:
+        os.system(f"git checkout {git_rev}")
     os.system(build_cmd)
 
     # copy compiled binary to right place
     compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
-    shutil.copyfile(compiled_binary, binary_path)
+    shutil.copyfile(compiled_binary, versioned_binary_path)
+    create_sym_link(versioned_binary_name, binary_name, dir)
 
     if not is_windows_platform():
         st = os.stat(binary_path)
@@ -81,10 +105,28 @@ def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd:
 
     shutil.rmtree(code_tmp_dir, ignore_errors=True)
 
-    print("Completed to build binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
+    print("Completed to build binary " + binary_name + ", Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
 
     return True
 
+def create_sym_link(src, dst, path = None):
+    if src == dst:
+        return
+    
+    origin_path = os.getcwd()
+    if path is not None:
+        os.chdir(path)
+
+    if os.path.exists(dst):
+        if os.path.isdir(dst):
+            shutil.rmtree(dst)
+        else:
+            os.remove(dst)
+
+    os.symlink(src, dst)
+
+    os.chdir(origin_path)
+
 def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool:
     if not os.path.exists(dir):
         os.makedirs(dir, exist_ok=True)

tests/utility/run_all.py

@@ -6,7 +6,7 @@ import sys
 
 from concurrent.futures import ProcessPoolExecutor
 
-from utility.build_binary import build_conflux, build_bsc, build_evmos
+from utility.build_binary import build_conflux, build_bsc, build_evmos, build_cli
 
 DEFAULT_PORT_MIN = 11000
 DEFAULT_PORT_MAX = 65535
@@ -59,10 +59,12 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
     if not os.path.exists(tmp_dir):
         os.makedirs(tmp_dir, exist_ok=True)
 
+
     # Build blockchain binaries if absent
     build_conflux(tmp_dir)
     build_bsc(tmp_dir)
     build_evmos(tmp_dir)
+    build_cli(tmp_dir)
 
     start_time = time.time()
 
@@ -137,3 +139,4 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
         for c in failed:
             print(c)
         sys.exit(1)
+