0g-storage-node/tests/crash_test.py

158 lines
6.0 KiB
Python
Raw Normal View History

2024-01-03 10:24:52 +00:00
#!/usr/bin/env python3
from test_framework.test_framework import TestFramework
from utility.submission import create_submission, submit_data
from utility.utils import wait_until
class CrashTest(TestFramework):
def setup_params(self):
self.num_blockchain_nodes = 1
self.num_nodes = 4
def run_test(self):
# 1. all nodes are running
chunk_data = b"\x01" * 256
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 1)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
segment = submit_data(self.nodes[0], chunk_data)
self.log.info("segment: %s", segment)
for i in range(self.num_nodes):
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(0)
self.log.info("wait for node: %s", i)
2024-01-03 10:24:52 +00:00
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 2: first node runnging, other nodes killed
self.log.info("kill node")
# kill node to simulate node crash
for i in range(1, self.num_nodes):
self.nodes[i].stop(kill=True)
chunk_data = b"\x02" * 256 * 1024
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 2)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
segment = submit_data(self.nodes[0], chunk_data)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
self.start_storage_node(1)
self.nodes[1].wait_for_rpc_connection()
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root) is not None)
self.nodes[1].admin_start_sync_file(1)
2024-01-03 10:24:52 +00:00
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root)["finalized"])
for i in range(2, self.num_nodes):
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
2024-01-03 10:24:52 +00:00
self.nodes[i].admin_start_sync_file(1)
self.nodes[i].stop(kill=True)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(1)
2024-01-03 10:24:52 +00:00
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 4: node[1..] synced contract entries and killed
self.log.info("kill node 0")
self.nodes[0].stop(kill=True)
self.start_storage_node(0)
self.nodes[0].wait_for_rpc_connection()
chunk_data = b"\x03" * 256
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 3)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
for i in range(1, self.num_nodes):
self.nodes[i].stop(kill=True)
segment = submit_data(self.nodes[0], chunk_data)
self.log.info("segment: %s", segment)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
for i in range(1, self.num_nodes):
self.log.info("wait for node: %s", i)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(2)
2024-01-03 10:24:52 +00:00
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
# 5: node[1..] synced contract entries and killed, sync disorder
self.nodes[0].stop(kill=True)
self.start_storage_node(0)
self.nodes[0].wait_for_rpc_connection()
chunk_data = b"\x04" * 256 * 1561
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions)
wait_until(lambda: self.contract.num_submissions() == 4)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
for i in range(1, 2):
self.nodes[i].stop(kill=True)
chunk_data1 = b"\x05" * 256
submissions1, data_root1 = create_submission(chunk_data1)
self.contract.submit(submissions1)
wait_until(lambda: self.contract.num_submissions() == 5)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1) is not None)
for i in range(2, self.num_nodes):
self.nodes[i].stop(kill=True)
segment = submit_data(self.nodes[0], chunk_data)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
segment = submit_data(self.nodes[0], chunk_data1)
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1)["finalized"])
for i in range(1, self.num_nodes):
self.log.info("wait for node: %s", i)
self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1) is not None
)
self.nodes[i].admin_start_sync_file(4)
2024-01-03 10:24:52 +00:00
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1)["finalized"]
)
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(3)
2024-01-03 10:24:52 +00:00
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
)
if __name__ == "__main__":
CrashTest().main()