mirror of
https://github.com/0glabs/0g-storage-node.git
synced 2024-12-25 15:55:17 +00:00
Supports to download blockchain binaries from github release (#61)
* build evmos before init genesis * Supports to download conflux and evmos binary * print unzip info * download conflux windows binary in zip format * unnecessary to install evmos when init node * aaa * bbb * ccc * ddd * eee * 111 * 2222 * aaa * aaa * aaa * aaa
This commit is contained in:
parent
c85fe38c59
commit
57090464ae
1
.github/workflows/tests.yml
vendored
1
.github/workflows/tests.yml
vendored
@ -53,4 +53,5 @@ jobs:
|
||||
- name: Run tests
|
||||
run: |
|
||||
cd tests
|
||||
uname -a
|
||||
python test_all.py
|
@ -11,9 +11,12 @@ EVMOS_PORT_CATEGORY_P2P = 1
|
||||
EVMOS_PORT_CATEGORY_RPC = 2
|
||||
EVMOS_PORT_CATEGORY_PPROF = 3
|
||||
|
||||
def evmos_init_genesis(root_dir: str, num_nodes: int):
|
||||
def evmos_init_genesis(binary: str, root_dir: str, num_nodes: int):
|
||||
assert num_nodes > 0, "Invalid number of blockchain nodes: %s" % num_nodes
|
||||
|
||||
if not os.path.exists(binary):
|
||||
build_evmos(os.path.dirname(binary))
|
||||
|
||||
shell_script = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), # test_framework folder
|
||||
"..", "config", "evmos-init-genesis.sh"
|
||||
@ -46,9 +49,6 @@ class EvmosNode(BlockchainNode):
|
||||
log,
|
||||
rpc_timeout=10,
|
||||
):
|
||||
if not os.path.exists(binary):
|
||||
build_evmos(os.path.dirname(binary))
|
||||
|
||||
data_dir = os.path.join(root_dir, "evmosd", "node" + str(index))
|
||||
rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index)
|
||||
|
||||
|
@ -72,7 +72,7 @@ class TestFramework:
|
||||
|
||||
def __setup_blockchain_node(self):
|
||||
if self.blockchain_node_type == BlockChainNodeType.Evmos:
|
||||
evmos_init_genesis(self.root_dir, self.num_blockchain_nodes)
|
||||
evmos_init_genesis(self.blockchain_binary, self.root_dir, self.num_blockchain_nodes)
|
||||
self.log.info("Evmos genesis initialized for %s nodes" % self.num_blockchain_nodes)
|
||||
|
||||
for i in range(self.num_blockchain_nodes):
|
||||
|
@ -7,7 +7,11 @@ import platform
|
||||
|
||||
from utility.utils import is_windows_platform, wait_until
|
||||
|
||||
# v1.0.0-ci release
|
||||
GITHUB_DOWNLOAD_URL="https://api.github.com/repos/0glabs/0g-storage-node/releases/152560136"
|
||||
|
||||
CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
|
||||
BSC_BINARY = "geth.exe" if is_windows_platform() else "geth"
|
||||
EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
|
||||
CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client"
|
||||
|
||||
@ -15,8 +19,16 @@ EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98"
|
||||
CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df"
|
||||
|
||||
def build_conflux(dir: str) -> bool:
|
||||
# Build conflux binary if absent
|
||||
build_from_github(
|
||||
# Download or build conflux binary if absent
|
||||
if __download_from_github(
|
||||
dir=dir,
|
||||
binary_name=CONFLUX_BINARY,
|
||||
github_url=GITHUB_DOWNLOAD_URL,
|
||||
asset_name=__asset_name(CONFLUX_BINARY, zip=True),
|
||||
):
|
||||
return True
|
||||
|
||||
return __build_from_github(
|
||||
dir=dir,
|
||||
binary_name=CONFLUX_BINARY,
|
||||
github_url="https://github.com/Conflux-Chain/conflux-rust.git",
|
||||
@ -25,26 +37,31 @@ def build_conflux(dir: str) -> bool:
|
||||
)
|
||||
|
||||
def build_bsc(dir: str) -> bool:
|
||||
sys = platform.system().lower()
|
||||
if sys == "linux":
|
||||
asset_name = "geth_linux"
|
||||
elif sys == "windows":
|
||||
asset_name = "geth_windows.exe"
|
||||
elif sys == "darwin":
|
||||
asset_name = "geth_mac"
|
||||
else:
|
||||
raise RuntimeError("Unable to recognize platform")
|
||||
|
||||
return __download_from_github(
|
||||
# Download bsc binary if absent
|
||||
downloaded = __download_from_github(
|
||||
dir=dir,
|
||||
binary_name="geth.exe" if is_windows_platform() else "geth",
|
||||
binary_name=BSC_BINARY,
|
||||
github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895",
|
||||
asset_name=asset_name,
|
||||
asset_name=__asset_name(BSC_BINARY),
|
||||
)
|
||||
|
||||
# Requires to download binary successfully, since it is not ready to build
|
||||
# binary from source code.
|
||||
assert downloaded, "Cannot download download binary from github [%s]" % BSC_BINARY
|
||||
|
||||
return True
|
||||
|
||||
def build_evmos(dir: str) -> bool:
|
||||
# Build evmos binary if absent
|
||||
build_from_github(
|
||||
# Download or build evmos binary if absent
|
||||
if __download_from_github(
|
||||
dir=dir,
|
||||
binary_name=EVMOS_BINARY,
|
||||
github_url=GITHUB_DOWNLOAD_URL,
|
||||
asset_name=__asset_name(EVMOS_BINARY, zip=True),
|
||||
):
|
||||
return True
|
||||
|
||||
return __build_from_github(
|
||||
dir=dir,
|
||||
binary_name=EVMOS_BINARY,
|
||||
github_url="https://github.com/0glabs/0g-evmos.git",
|
||||
@ -55,7 +72,7 @@ def build_evmos(dir: str) -> bool:
|
||||
|
||||
def build_cli(dir: str) -> bool:
|
||||
# Build 0g-storage-client binary if absent
|
||||
build_from_github(
|
||||
return __build_from_github(
|
||||
dir=dir,
|
||||
binary_name=CLIENT_BINARY,
|
||||
github_url="https://github.com/0glabs/0g-storage-client.git",
|
||||
@ -64,7 +81,19 @@ def build_cli(dir: str) -> bool:
|
||||
compiled_relative_path=[],
|
||||
)
|
||||
|
||||
def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
|
||||
def __asset_name(binary_name: str, zip: bool = False) -> str:
|
||||
sys = platform.system().lower()
|
||||
if sys == "linux":
|
||||
return f"{binary_name}_linux.zip" if zip else f"{binary_name}_linux"
|
||||
elif sys == "windows":
|
||||
binary_name = binary_name.removesuffix(".exe")
|
||||
return f"{binary_name}_windows.zip" if zip else f"{binary_name}_windows.exe"
|
||||
elif sys == "darwin":
|
||||
return f"{binary_name}_mac.zip" if zip else f"{binary_name}_mac"
|
||||
else:
|
||||
raise RuntimeError("Unable to recognize platform")
|
||||
|
||||
def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
|
||||
if git_rev is not None:
|
||||
versioned_binary_name = f"{binary_name}_{git_rev}"
|
||||
else:
|
||||
@ -73,7 +102,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
|
||||
binary_path = os.path.join(dir, binary_name)
|
||||
versioned_binary_path = os.path.join(dir, versioned_binary_name)
|
||||
if os.path.exists(versioned_binary_path):
|
||||
create_sym_link(versioned_binary_name, binary_name, dir)
|
||||
__create_sym_link(versioned_binary_name, binary_name, dir)
|
||||
return False
|
||||
|
||||
start_time = time.time()
|
||||
@ -95,7 +124,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
|
||||
# copy compiled binary to right place
|
||||
compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
|
||||
shutil.copyfile(compiled_binary, versioned_binary_path)
|
||||
create_sym_link(versioned_binary_name, binary_name, dir)
|
||||
__create_sym_link(versioned_binary_name, binary_name, dir)
|
||||
|
||||
if not is_windows_platform():
|
||||
st = os.stat(binary_path)
|
||||
@ -109,7 +138,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
|
||||
|
||||
return True
|
||||
|
||||
def create_sym_link(src, dst, path = None):
|
||||
def __create_sym_link(src, dst, path = None):
|
||||
if src == dst:
|
||||
return
|
||||
|
||||
@ -139,21 +168,34 @@ def __download_from_github(dir: str, binary_name: str, github_url: str, asset_na
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
with open(binary_path, "xb") as f:
|
||||
req = requests.get(github_url)
|
||||
assert req.ok, "Failed to request: %s" % github_url
|
||||
for asset in req.json()["assets"]:
|
||||
if asset["name"].lower() == asset_name:
|
||||
download_url = asset["browser_download_url"]
|
||||
break
|
||||
req = requests.get(github_url)
|
||||
assert req.ok, "Failed to request: %s" % github_url
|
||||
download_url = None
|
||||
for asset in req.json()["assets"]:
|
||||
if asset["name"].lower() == asset_name:
|
||||
download_url = asset["browser_download_url"]
|
||||
break
|
||||
|
||||
assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name
|
||||
if download_url is None:
|
||||
print(f"Cannot find asset by name {asset_name}", flush=True)
|
||||
return False
|
||||
|
||||
f.write(requests.get(download_url).content)
|
||||
content = requests.get(download_url).content
|
||||
|
||||
if not is_windows_platform():
|
||||
st = os.stat(binary_path)
|
||||
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
|
||||
# Supports to read from zipped binary
|
||||
if asset_name.endswith(".zip"):
|
||||
asset_path = os.path.join(dir, asset_name)
|
||||
with open(asset_path, "xb") as f:
|
||||
f.write(content)
|
||||
shutil.unpack_archive(asset_path, dir)
|
||||
assert os.path.exists(binary_path), f"Cannot find binary after unzip, binary = {binary_name}, asset = {asset_name}"
|
||||
else:
|
||||
with open(binary_path, "xb") as f:
|
||||
f.write(content)
|
||||
|
||||
if not is_windows_platform():
|
||||
st = os.stat(binary_path)
|
||||
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
|
||||
|
||||
wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120)
|
||||
|
||||
|
@ -59,7 +59,6 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
|
||||
if not os.path.exists(tmp_dir):
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
|
||||
|
||||
# Build blockchain binaries if absent
|
||||
build_conflux(tmp_dir)
|
||||
build_bsc(tmp_dir)
|
||||
|
Loading…
Reference in New Issue
Block a user