0g-storage-node/tests/crash_test.py
Bo QIU 8b225fde43
Fix python test (#8)
* Fix stdouterr issue when shutdown storage node

* Fix random failure for submission test

* add more info for failed rpc

* use single file instead of subprocess pipe to launch CLI

* Opt test_all for windows

* wait for log entry before file sync by admin
2024-01-24 20:42:25 +08:00

158 lines
6.0 KiB
Python

#!/usr/bin/env python3
from test_framework.test_framework import TestFramework
from utility.submission import create_submission, submit_data
from utility.utils import wait_until
class CrashTest(TestFramework):
def setup_params(self):
self.num_blockchain_nodes = 1
self.num_nodes = 4
def run_test(self):
# 1. all nodes are running
chunk_data = b"\x01" * 256
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 1)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
segment = submit_data(self.nodes[0], chunk_data)
self.log.info("segment: %s", segment)
for i in range(self.num_nodes):
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(0)
self.log.info("wait for node: %s", i)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 2: first node runnging, other nodes killed
self.log.info("kill node")
# kill node to simulate node crash
for i in range(1, self.num_nodes):
self.nodes[i].stop(kill=True)
chunk_data = b"\x02" * 256 * 1024
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 2)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
segment = submit_data(self.nodes[0], chunk_data)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
self.start_storage_node(1)
self.nodes[1].wait_for_rpc_connection()
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root) is not None)
self.nodes[1].admin_start_sync_file(1)
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root)["finalized"])
for i in range(2, self.num_nodes):
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(1)
self.nodes[i].stop(kill=True)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(1)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 4: node[1..] synced contract entries and killed
self.log.info("kill node 0")
self.nodes[0].stop(kill=True)
self.start_storage_node(0)
self.nodes[0].wait_for_rpc_connection()
chunk_data = b"\x03" * 256
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 3)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
for i in range(1, self.num_nodes):
self.nodes[i].stop(kill=True)
segment = submit_data(self.nodes[0], chunk_data)
self.log.info("segment: %s", segment)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
for i in range(1, self.num_nodes):
self.log.info("wait for node: %s", i)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(2)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 5: node[1..] synced contract entries and killed, sync disorder
self.nodes[0].stop(kill=True)
self.start_storage_node(0)
self.nodes[0].wait_for_rpc_connection()
chunk_data = b"\x04" * 256 * 1561
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 4)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
for i in range(1, 2):
self.nodes[i].stop(kill=True)
chunk_data1 = b"\x05" * 256
submissions1, data_root1 = create_submission(chunk_data1)
self.contract.submit(submissions1)
wait_until(lambda: self.contract.num_submissions() == 5)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1) is not None)
for i in range(2, self.num_nodes):
self.nodes[i].stop(kill=True)
segment = submit_data(self.nodes[0], chunk_data)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
segment = submit_data(self.nodes[0], chunk_data1)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1)["finalized"])
for i in range(1, self.num_nodes):
self.log.info("wait for node: %s", i)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1) is not None
)
self.nodes[i].admin_start_sync_file(4)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1)["finalized"]
)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(3)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
if __name__ == "__main__":
CrashTest().main()