From 64120399f11705f75eac595b3f331d4724906e2b Mon Sep 17 00:00:00 2001 From: Bo QIU <35757521+boqiu@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:36:53 +0800 Subject: [PATCH] refactor sync relevant python tests (#301) --- tests/config/node_config.py | 1 + ...2_test.py => sync_auto_historical_test.py} | 10 ++- tests/sync_auto_random_test.py | 32 ---------- tests/sync_auto_sequential_test_ignore.py | 32 ---------- ..._auto_test_ignore.py => sync_auto_test.py} | 4 +- tests/{sync_test.py => sync_chunks_test.py} | 62 +++---------------- tests/sync_file_test.py | 51 +++++++++++++++ 7 files changed, 67 insertions(+), 125 deletions(-) rename tests/{sync_auto_random_v2_test.py => sync_auto_historical_test.py} (86%) delete mode 100644 tests/sync_auto_random_test.py delete mode 100644 tests/sync_auto_sequential_test_ignore.py rename tests/{sync_auto_test_ignore.py => sync_auto_test.py} (87%) rename tests/{sync_test.py => sync_chunks_test.py} (54%) mode change 100755 => 100644 create mode 100644 tests/sync_file_test.py diff --git a/tests/config/node_config.py b/tests/config/node_config.py index 71b97c5..a24b213 100644 --- a/tests/config/node_config.py +++ b/tests/config/node_config.py @@ -3,6 +3,7 @@ import os from web3 import Web3 ZGS_CONFIG = { + "network_enr_address": "127.0.0.1", "log_config_file": "log_config", "confirmation_block_count": 1, "discv5_disable_ip_limit": True, diff --git a/tests/sync_auto_random_v2_test.py b/tests/sync_auto_historical_test.py similarity index 86% rename from tests/sync_auto_random_v2_test.py rename to tests/sync_auto_historical_test.py index a00c559..5b4989c 100644 --- a/tests/sync_auto_random_v2_test.py +++ b/tests/sync_auto_historical_test.py @@ -3,17 +3,15 @@ from test_framework.test_framework import TestFramework from utility.utils import wait_until -class AutoRandomSyncV2Test(TestFramework): +class AutoSyncHistoricalTest(TestFramework): def setup_params(self): self.num_nodes = 4 - # Enable random auto sync v2 + # Enable auto sync for i in range(self.num_nodes): self.zgs_node_configs[i] = { "sync": { - "auto_sync_enabled": True, - "max_sequential_workers": 0, - "max_random_workers": 3, + "auto_sync_enabled": True } } @@ -41,4 +39,4 @@ class AutoRandomSyncV2Test(TestFramework): wait_until(lambda: self.nodes[self.num_nodes - 1].zgs_get_file_info(data_root_2)["finalized"]) if __name__ == "__main__": - AutoRandomSyncV2Test().main() + AutoSyncHistoricalTest().main() diff --git a/tests/sync_auto_random_test.py b/tests/sync_auto_random_test.py deleted file mode 100644 index 5b572f2..0000000 --- a/tests/sync_auto_random_test.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from test_framework.test_framework import TestFramework -from utility.utils import wait_until - -class AutoRandomSyncTest(TestFramework): - def setup_params(self): - self.num_nodes = 2 - - # Enable random auto sync only - for i in range(self.num_nodes): - self.zgs_node_configs[i] = { - "sync": { - "auto_sync_enabled": True, - "max_sequential_workers": 0, - "max_random_workers": 3, - } - } - - def run_test(self): - # Submit and upload files on node 0 - data_root_1 = self.__upload_file__(0, 256 * 1024) - data_root_2 = self.__upload_file__(0, 256 * 1024) - - # Files should be available on node 1 via auto sync - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1) is not None) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1)["finalized"]) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2) is not None) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2)["finalized"]) - -if __name__ == "__main__": - AutoRandomSyncTest().main() diff --git a/tests/sync_auto_sequential_test_ignore.py b/tests/sync_auto_sequential_test_ignore.py deleted file mode 100644 index c2103bd..0000000 --- a/tests/sync_auto_sequential_test_ignore.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from test_framework.test_framework import TestFramework -from utility.utils import wait_until - -class AutoSequentialSyncTest(TestFramework): - def setup_params(self): - self.num_nodes = 2 - - # Enable sequential auto sync only - for i in range(self.num_nodes): - self.zgs_node_configs[i] = { - "sync": { - "auto_sync_enabled": True, - "max_sequential_workers": 3, - "max_random_workers": 0, - } - } - - def run_test(self): - # Submit and upload files on node 0 - data_root_1 = self.__upload_file__(0, 256 * 1024) - data_root_2 = self.__upload_file__(0, 256 * 1024) - - # Files should be available on node 1 via auto sync - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1) is not None) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1)["finalized"]) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2) is not None) - wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2)["finalized"]) - -if __name__ == "__main__": - AutoSequentialSyncTest().main() diff --git a/tests/sync_auto_test_ignore.py b/tests/sync_auto_test.py similarity index 87% rename from tests/sync_auto_test_ignore.py rename to tests/sync_auto_test.py index cc7dd30..43ec235 100644 --- a/tests/sync_auto_test_ignore.py +++ b/tests/sync_auto_test.py @@ -11,9 +11,7 @@ class AutoSyncTest(TestFramework): for i in range(self.num_nodes): self.zgs_node_configs[i] = { "sync": { - "auto_sync_enabled": True, - "max_sequential_workers": 3, - "max_random_workers": 3, + "auto_sync_enabled": True } } diff --git a/tests/sync_test.py b/tests/sync_chunks_test.py old mode 100755 new mode 100644 similarity index 54% rename from tests/sync_test.py rename to tests/sync_chunks_test.py index e2e8f7f..f24418f --- a/tests/sync_test.py +++ b/tests/sync_chunks_test.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 import random -import time from test_framework.test_framework import TestFramework from utility.submission import data_to_segments -from utility.utils import ( - assert_equal, - wait_until, -) +from utility.utils import assert_equal, wait_until + +class SyncChunksTest(TestFramework): + """ + By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false, + and chunks sync should be triggered by rpc. + """ -class SyncTest(TestFramework): def setup_params(self): self.num_nodes = 2 @@ -21,49 +22,6 @@ class SyncTest(TestFramework): } def run_test(self): - # By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false, - # and file or chunks sync should be triggered by rpc. - self.__test_sync_file_by_rpc() - self.__test_sync_chunks_by_rpc() - - def __test_sync_file_by_rpc(self): - self.log.info("Begin to test file sync by rpc") - - client1 = self.nodes[0] - client2 = self.nodes[1] - - # stop client2, preventing it from receiving AnnounceFile - client2.shutdown() - - data_root = self.__upload_file__(0, 256 * 1024) - - # restart client2 - client2.start() - client2.wait_for_rpc_connection() - - # File should not be auto sync on node 2 and there is no cached file locations - wait_until(lambda: client2.zgs_get_file_info(data_root) is not None) - time.sleep(3) - assert_equal(client2.zgs_get_file_info(data_root)["finalized"], False) - # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore. - # assert(client2.admin_get_file_location(0) is None) - - # Trigger file sync by rpc - assert(client2.admin_start_sync_file(0) is None) - wait_until(lambda: client2.sync_status_is_completed_or_unknown(0)) - wait_until(lambda: client2.zgs_get_file_info(data_root)["finalized"]) - # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore. - # assert(client2.admin_get_file_location(0) is not None) - - # Validate data - assert_equal( - client2.zgs_download_segment(data_root, 0, 1024), - client1.zgs_download_segment(data_root, 0, 1024), - ) - - def __test_sync_chunks_by_rpc(self): - self.log.info("Begin to test chunks sync by rpc") - client1 = self.nodes[0] client2 = self.nodes[1] @@ -98,12 +56,12 @@ class SyncTest(TestFramework): self.nodes[0].wait_for_rpc_connection() # Trigger chunks sync by rpc - assert(client2.admin_start_sync_chunks(1, 1024, 2048) is None) - wait_until(lambda: client2.sync_status_is_completed_or_unknown(1)) + assert(client2.admin_start_sync_chunks(0, 1024, 2048) is None) + wait_until(lambda: client2.sync_status_is_completed_or_unknown(0)) wait_until(lambda: client2.zgs_download_segment_decoded(data_root, 1024, 2048) is not None) # Validate data assert_equal(client2.zgs_download_segment_decoded(data_root, 1024, 2048), chunk_data[1024*256:2048*256]) if __name__ == "__main__": - SyncTest().main() + SyncChunksTest().main() diff --git a/tests/sync_file_test.py b/tests/sync_file_test.py new file mode 100644 index 0000000..4a64f0e --- /dev/null +++ b/tests/sync_file_test.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import time + +from test_framework.test_framework import TestFramework +from utility.utils import assert_equal, wait_until + +class SyncFileTest(TestFramework): + """ + By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false, + and file sync should be triggered by rpc. + """ + + def setup_params(self): + self.num_nodes = 2 + + def run_test(self): + client1 = self.nodes[0] + client2 = self.nodes[1] + + # stop client2, preventing it from receiving AnnounceFile + client2.shutdown() + + data_root = self.__upload_file__(0, 256 * 1024) + + # restart client2 + client2.start() + client2.wait_for_rpc_connection() + + # File should not be auto sync on node 2 and there is no cached file locations + wait_until(lambda: client2.zgs_get_file_info(data_root) is not None) + time.sleep(3) + assert_equal(client2.zgs_get_file_info(data_root)["finalized"], False) + # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore. + # assert(client2.admin_get_file_location(0) is None) + + # Trigger file sync by rpc + assert(client2.admin_start_sync_file(0) is None) + wait_until(lambda: client2.sync_status_is_completed_or_unknown(0)) + wait_until(lambda: client2.zgs_get_file_info(data_root)["finalized"]) + # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore. + # assert(client2.admin_get_file_location(0) is not None) + + # Validate data + assert_equal( + client2.zgs_download_segment(data_root, 0, 1024), + client1.zgs_download_segment(data_root, 0, 1024), + ) + +if __name__ == "__main__": + SyncFileTest().main()