From 57090464ae15c8746776cd7ae2908f912c434415 Mon Sep 17 00:00:00 2001 From: Bo QIU <35757521+boqiu@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:25:05 +0800 Subject: [PATCH] Supports to download blockchain binaries from github release (#61) * build evmos before init genesis * Supports to download conflux and evmos binary * print unzip info * download conflux windows binary in zip format * unnecessary to install evmos when init node * aaa * bbb * ccc * ddd * eee * 111 * 2222 * aaa * aaa * aaa * aaa --- .github/workflows/tests.yml | 1 + tests/test_framework/evmos_node.py | 8 +- tests/test_framework/test_framework.py | 2 +- tests/utility/build_binary.py | 110 +++++++++++++++++-------- tests/utility/run_all.py | 1 - 5 files changed, 82 insertions(+), 40 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 846698f..3469b88 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,4 +53,5 @@ jobs: - name: Run tests run: | cd tests + uname -a python test_all.py \ No newline at end of file diff --git a/tests/test_framework/evmos_node.py b/tests/test_framework/evmos_node.py index 3701fb9..ebd7340 100644 --- a/tests/test_framework/evmos_node.py +++ b/tests/test_framework/evmos_node.py @@ -11,9 +11,12 @@ EVMOS_PORT_CATEGORY_P2P = 1 EVMOS_PORT_CATEGORY_RPC = 2 EVMOS_PORT_CATEGORY_PPROF = 3 -def evmos_init_genesis(root_dir: str, num_nodes: int): +def evmos_init_genesis(binary: str, root_dir: str, num_nodes: int): assert num_nodes > 0, "Invalid number of blockchain nodes: %s" % num_nodes + if not os.path.exists(binary): + build_evmos(os.path.dirname(binary)) + shell_script = os.path.join( os.path.dirname(os.path.realpath(__file__)), # test_framework folder "..", "config", "evmos-init-genesis.sh" @@ -46,9 +49,6 @@ class EvmosNode(BlockchainNode): log, rpc_timeout=10, ): - if not os.path.exists(binary): - build_evmos(os.path.dirname(binary)) - data_dir = os.path.join(root_dir, "evmosd", "node" + str(index)) rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index) diff --git a/tests/test_framework/test_framework.py b/tests/test_framework/test_framework.py index afcabca..1df6bd0 100644 --- a/tests/test_framework/test_framework.py +++ b/tests/test_framework/test_framework.py @@ -72,7 +72,7 @@ class TestFramework: def __setup_blockchain_node(self): if self.blockchain_node_type == BlockChainNodeType.Evmos: - evmos_init_genesis(self.root_dir, self.num_blockchain_nodes) + evmos_init_genesis(self.blockchain_binary, self.root_dir, self.num_blockchain_nodes) self.log.info("Evmos genesis initialized for %s nodes" % self.num_blockchain_nodes) for i in range(self.num_blockchain_nodes): diff --git a/tests/utility/build_binary.py b/tests/utility/build_binary.py index 39df824..d1614a5 100644 --- a/tests/utility/build_binary.py +++ b/tests/utility/build_binary.py @@ -7,7 +7,11 @@ import platform from utility.utils import is_windows_platform, wait_until +# v1.0.0-ci release +GITHUB_DOWNLOAD_URL="https://api.github.com/repos/0glabs/0g-storage-node/releases/152560136" + CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux" +BSC_BINARY = "geth.exe" if is_windows_platform() else "geth" EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd" CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client" @@ -15,8 +19,16 @@ EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98" CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df" def build_conflux(dir: str) -> bool: - # Build conflux binary if absent - build_from_github( + # Download or build conflux binary if absent + if __download_from_github( + dir=dir, + binary_name=CONFLUX_BINARY, + github_url=GITHUB_DOWNLOAD_URL, + asset_name=__asset_name(CONFLUX_BINARY, zip=True), + ): + return True + + return __build_from_github( dir=dir, binary_name=CONFLUX_BINARY, github_url="https://github.com/Conflux-Chain/conflux-rust.git", @@ -25,26 +37,31 @@ def build_conflux(dir: str) -> bool: ) def build_bsc(dir: str) -> bool: - sys = platform.system().lower() - if sys == "linux": - asset_name = "geth_linux" - elif sys == "windows": - asset_name = "geth_windows.exe" - elif sys == "darwin": - asset_name = "geth_mac" - else: - raise RuntimeError("Unable to recognize platform") - - return __download_from_github( + # Download bsc binary if absent + downloaded = __download_from_github( dir=dir, - binary_name="geth.exe" if is_windows_platform() else "geth", + binary_name=BSC_BINARY, github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895", - asset_name=asset_name, + asset_name=__asset_name(BSC_BINARY), ) + # Requires to download binary successfully, since it is not ready to build + # binary from source code. + assert downloaded, "Cannot download download binary from github [%s]" % BSC_BINARY + + return True + def build_evmos(dir: str) -> bool: - # Build evmos binary if absent - build_from_github( + # Download or build evmos binary if absent + if __download_from_github( + dir=dir, + binary_name=EVMOS_BINARY, + github_url=GITHUB_DOWNLOAD_URL, + asset_name=__asset_name(EVMOS_BINARY, zip=True), + ): + return True + + return __build_from_github( dir=dir, binary_name=EVMOS_BINARY, github_url="https://github.com/0glabs/0g-evmos.git", @@ -55,7 +72,7 @@ def build_evmos(dir: str) -> bool: def build_cli(dir: str) -> bool: # Build 0g-storage-client binary if absent - build_from_github( + return __build_from_github( dir=dir, binary_name=CLIENT_BINARY, github_url="https://github.com/0glabs/0g-storage-client.git", @@ -64,7 +81,19 @@ def build_cli(dir: str) -> bool: compiled_relative_path=[], ) -def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool: +def __asset_name(binary_name: str, zip: bool = False) -> str: + sys = platform.system().lower() + if sys == "linux": + return f"{binary_name}_linux.zip" if zip else f"{binary_name}_linux" + elif sys == "windows": + binary_name = binary_name.removesuffix(".exe") + return f"{binary_name}_windows.zip" if zip else f"{binary_name}_windows.exe" + elif sys == "darwin": + return f"{binary_name}_mac.zip" if zip else f"{binary_name}_mac" + else: + raise RuntimeError("Unable to recognize platform") + +def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool: if git_rev is not None: versioned_binary_name = f"{binary_name}_{git_rev}" else: @@ -73,7 +102,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st binary_path = os.path.join(dir, binary_name) versioned_binary_path = os.path.join(dir, versioned_binary_name) if os.path.exists(versioned_binary_path): - create_sym_link(versioned_binary_name, binary_name, dir) + __create_sym_link(versioned_binary_name, binary_name, dir) return False start_time = time.time() @@ -95,7 +124,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st # copy compiled binary to right place compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name) shutil.copyfile(compiled_binary, versioned_binary_path) - create_sym_link(versioned_binary_name, binary_name, dir) + __create_sym_link(versioned_binary_name, binary_name, dir) if not is_windows_platform(): st = os.stat(binary_path) @@ -109,7 +138,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st return True -def create_sym_link(src, dst, path = None): +def __create_sym_link(src, dst, path = None): if src == dst: return @@ -139,21 +168,34 @@ def __download_from_github(dir: str, binary_name: str, github_url: str, asset_na start_time = time.time() - with open(binary_path, "xb") as f: - req = requests.get(github_url) - assert req.ok, "Failed to request: %s" % github_url - for asset in req.json()["assets"]: - if asset["name"].lower() == asset_name: - download_url = asset["browser_download_url"] - break + req = requests.get(github_url) + assert req.ok, "Failed to request: %s" % github_url + download_url = None + for asset in req.json()["assets"]: + if asset["name"].lower() == asset_name: + download_url = asset["browser_download_url"] + break - assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name + if download_url is None: + print(f"Cannot find asset by name {asset_name}", flush=True) + return False + + content = requests.get(download_url).content - f.write(requests.get(download_url).content) + # Supports to read from zipped binary + if asset_name.endswith(".zip"): + asset_path = os.path.join(dir, asset_name) + with open(asset_path, "xb") as f: + f.write(content) + shutil.unpack_archive(asset_path, dir) + assert os.path.exists(binary_path), f"Cannot find binary after unzip, binary = {binary_name}, asset = {asset_name}" + else: + with open(binary_path, "xb") as f: + f.write(content) - if not is_windows_platform(): - st = os.stat(binary_path) - os.chmod(binary_path, st.st_mode | stat.S_IEXEC) + if not is_windows_platform(): + st = os.stat(binary_path) + os.chmod(binary_path, st.st_mode | stat.S_IEXEC) wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120) diff --git a/tests/utility/run_all.py b/tests/utility/run_all.py index c32e2b7..1dd936c 100644 --- a/tests/utility/run_all.py +++ b/tests/utility/run_all.py @@ -59,7 +59,6 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={}, if not os.path.exists(tmp_dir): os.makedirs(tmp_dir, exist_ok=True) - # Build blockchain binaries if absent build_conflux(tmp_dir) build_bsc(tmp_dir)