Supports to download blockchain binaries from github release (#61)

* build evmos before init genesis

* Supports to download conflux and evmos binary

* print unzip info

* download conflux windows binary in zip format

* unnecessary to install evmos when init node

* aaa

* bbb

* ccc

* ddd

* eee

* 111

* 2222

* aaa

* aaa

* aaa

* aaa
This commit is contained in:
Bo QIU 2024-04-25 11:25:05 +08:00 committed by GitHub
parent c85fe38c59
commit 57090464ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 82 additions and 40 deletions

View File

@ -53,4 +53,5 @@ jobs:
- name: Run tests - name: Run tests
run: | run: |
cd tests cd tests
uname -a
python test_all.py python test_all.py

View File

@ -11,9 +11,12 @@ EVMOS_PORT_CATEGORY_P2P = 1
EVMOS_PORT_CATEGORY_RPC = 2 EVMOS_PORT_CATEGORY_RPC = 2
EVMOS_PORT_CATEGORY_PPROF = 3 EVMOS_PORT_CATEGORY_PPROF = 3
def evmos_init_genesis(root_dir: str, num_nodes: int): def evmos_init_genesis(binary: str, root_dir: str, num_nodes: int):
assert num_nodes > 0, "Invalid number of blockchain nodes: %s" % num_nodes assert num_nodes > 0, "Invalid number of blockchain nodes: %s" % num_nodes
if not os.path.exists(binary):
build_evmos(os.path.dirname(binary))
shell_script = os.path.join( shell_script = os.path.join(
os.path.dirname(os.path.realpath(__file__)), # test_framework folder os.path.dirname(os.path.realpath(__file__)), # test_framework folder
"..", "config", "evmos-init-genesis.sh" "..", "config", "evmos-init-genesis.sh"
@ -46,9 +49,6 @@ class EvmosNode(BlockchainNode):
log, log,
rpc_timeout=10, rpc_timeout=10,
): ):
if not os.path.exists(binary):
build_evmos(os.path.dirname(binary))
data_dir = os.path.join(root_dir, "evmosd", "node" + str(index)) data_dir = os.path.join(root_dir, "evmosd", "node" + str(index))
rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index) rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index)

View File

@ -72,7 +72,7 @@ class TestFramework:
def __setup_blockchain_node(self): def __setup_blockchain_node(self):
if self.blockchain_node_type == BlockChainNodeType.Evmos: if self.blockchain_node_type == BlockChainNodeType.Evmos:
evmos_init_genesis(self.root_dir, self.num_blockchain_nodes) evmos_init_genesis(self.blockchain_binary, self.root_dir, self.num_blockchain_nodes)
self.log.info("Evmos genesis initialized for %s nodes" % self.num_blockchain_nodes) self.log.info("Evmos genesis initialized for %s nodes" % self.num_blockchain_nodes)
for i in range(self.num_blockchain_nodes): for i in range(self.num_blockchain_nodes):

View File

@ -7,7 +7,11 @@ import platform
from utility.utils import is_windows_platform, wait_until from utility.utils import is_windows_platform, wait_until
# v1.0.0-ci release
GITHUB_DOWNLOAD_URL="https://api.github.com/repos/0glabs/0g-storage-node/releases/152560136"
CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux" CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
BSC_BINARY = "geth.exe" if is_windows_platform() else "geth"
EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd" EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client" CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client"
@ -15,8 +19,16 @@ EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98"
CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df" CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df"
def build_conflux(dir: str) -> bool: def build_conflux(dir: str) -> bool:
# Build conflux binary if absent # Download or build conflux binary if absent
build_from_github( if __download_from_github(
dir=dir,
binary_name=CONFLUX_BINARY,
github_url=GITHUB_DOWNLOAD_URL,
asset_name=__asset_name(CONFLUX_BINARY, zip=True),
):
return True
return __build_from_github(
dir=dir, dir=dir,
binary_name=CONFLUX_BINARY, binary_name=CONFLUX_BINARY,
github_url="https://github.com/Conflux-Chain/conflux-rust.git", github_url="https://github.com/Conflux-Chain/conflux-rust.git",
@ -25,26 +37,31 @@ def build_conflux(dir: str) -> bool:
) )
def build_bsc(dir: str) -> bool: def build_bsc(dir: str) -> bool:
sys = platform.system().lower() # Download bsc binary if absent
if sys == "linux": downloaded = __download_from_github(
asset_name = "geth_linux"
elif sys == "windows":
asset_name = "geth_windows.exe"
elif sys == "darwin":
asset_name = "geth_mac"
else:
raise RuntimeError("Unable to recognize platform")
return __download_from_github(
dir=dir, dir=dir,
binary_name="geth.exe" if is_windows_platform() else "geth", binary_name=BSC_BINARY,
github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895", github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895",
asset_name=asset_name, asset_name=__asset_name(BSC_BINARY),
) )
# Requires to download binary successfully, since it is not ready to build
# binary from source code.
assert downloaded, "Cannot download download binary from github [%s]" % BSC_BINARY
return True
def build_evmos(dir: str) -> bool: def build_evmos(dir: str) -> bool:
# Build evmos binary if absent # Download or build evmos binary if absent
build_from_github( if __download_from_github(
dir=dir,
binary_name=EVMOS_BINARY,
github_url=GITHUB_DOWNLOAD_URL,
asset_name=__asset_name(EVMOS_BINARY, zip=True),
):
return True
return __build_from_github(
dir=dir, dir=dir,
binary_name=EVMOS_BINARY, binary_name=EVMOS_BINARY,
github_url="https://github.com/0glabs/0g-evmos.git", github_url="https://github.com/0glabs/0g-evmos.git",
@ -55,7 +72,7 @@ def build_evmos(dir: str) -> bool:
def build_cli(dir: str) -> bool: def build_cli(dir: str) -> bool:
# Build 0g-storage-client binary if absent # Build 0g-storage-client binary if absent
build_from_github( return __build_from_github(
dir=dir, dir=dir,
binary_name=CLIENT_BINARY, binary_name=CLIENT_BINARY,
github_url="https://github.com/0glabs/0g-storage-client.git", github_url="https://github.com/0glabs/0g-storage-client.git",
@ -64,7 +81,19 @@ def build_cli(dir: str) -> bool:
compiled_relative_path=[], compiled_relative_path=[],
) )
def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool: def __asset_name(binary_name: str, zip: bool = False) -> str:
sys = platform.system().lower()
if sys == "linux":
return f"{binary_name}_linux.zip" if zip else f"{binary_name}_linux"
elif sys == "windows":
binary_name = binary_name.removesuffix(".exe")
return f"{binary_name}_windows.zip" if zip else f"{binary_name}_windows.exe"
elif sys == "darwin":
return f"{binary_name}_mac.zip" if zip else f"{binary_name}_mac"
else:
raise RuntimeError("Unable to recognize platform")
def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
if git_rev is not None: if git_rev is not None:
versioned_binary_name = f"{binary_name}_{git_rev}" versioned_binary_name = f"{binary_name}_{git_rev}"
else: else:
@ -73,7 +102,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
binary_path = os.path.join(dir, binary_name) binary_path = os.path.join(dir, binary_name)
versioned_binary_path = os.path.join(dir, versioned_binary_name) versioned_binary_path = os.path.join(dir, versioned_binary_name)
if os.path.exists(versioned_binary_path): if os.path.exists(versioned_binary_path):
create_sym_link(versioned_binary_name, binary_name, dir) __create_sym_link(versioned_binary_name, binary_name, dir)
return False return False
start_time = time.time() start_time = time.time()
@ -95,7 +124,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
# copy compiled binary to right place # copy compiled binary to right place
compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name) compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
shutil.copyfile(compiled_binary, versioned_binary_path) shutil.copyfile(compiled_binary, versioned_binary_path)
create_sym_link(versioned_binary_name, binary_name, dir) __create_sym_link(versioned_binary_name, binary_name, dir)
if not is_windows_platform(): if not is_windows_platform():
st = os.stat(binary_path) st = os.stat(binary_path)
@ -109,7 +138,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
return True return True
def create_sym_link(src, dst, path = None): def __create_sym_link(src, dst, path = None):
if src == dst: if src == dst:
return return
@ -139,21 +168,34 @@ def __download_from_github(dir: str, binary_name: str, github_url: str, asset_na
start_time = time.time() start_time = time.time()
with open(binary_path, "xb") as f: req = requests.get(github_url)
req = requests.get(github_url) assert req.ok, "Failed to request: %s" % github_url
assert req.ok, "Failed to request: %s" % github_url download_url = None
for asset in req.json()["assets"]: for asset in req.json()["assets"]:
if asset["name"].lower() == asset_name: if asset["name"].lower() == asset_name:
download_url = asset["browser_download_url"] download_url = asset["browser_download_url"]
break break
assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name if download_url is None:
print(f"Cannot find asset by name {asset_name}", flush=True)
return False
content = requests.get(download_url).content
f.write(requests.get(download_url).content) # Supports to read from zipped binary
if asset_name.endswith(".zip"):
asset_path = os.path.join(dir, asset_name)
with open(asset_path, "xb") as f:
f.write(content)
shutil.unpack_archive(asset_path, dir)
assert os.path.exists(binary_path), f"Cannot find binary after unzip, binary = {binary_name}, asset = {asset_name}"
else:
with open(binary_path, "xb") as f:
f.write(content)
if not is_windows_platform(): if not is_windows_platform():
st = os.stat(binary_path) st = os.stat(binary_path)
os.chmod(binary_path, st.st_mode | stat.S_IEXEC) os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120) wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120)

View File

@ -59,7 +59,6 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
if not os.path.exists(tmp_dir): if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir, exist_ok=True) os.makedirs(tmp_dir, exist_ok=True)
# Build blockchain binaries if absent # Build blockchain binaries if absent
build_conflux(tmp_dir) build_conflux(tmp_dir)
build_bsc(tmp_dir) build_bsc(tmp_dir)