Fix python test (#8)

* Fix stdouterr issue when shutdown storage node

* Fix random failure for submission test

* add more info for failed rpc

* use single file instead of subprocess pipe to launch CLI

* Opt test_all for windows

* wait for log entry before file sync by admin
This commit is contained in:
Bo QIU 2024-01-24 20:42:25 +08:00 committed by GitHub
parent 7479f56474
commit 8b225fde43
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 78 additions and 50 deletions

View File

@ -84,6 +84,7 @@ class CliSubmissionTest(TestFramework):
continue continue
self.log.info("wait node %d", i) self.log.info("wait node %d", i)
wait_until(lambda: self.nodes[i].zgs_get_file_info(root) is not None)
self.nodes[i].admin_start_sync_file(submission_index - 1) self.nodes[i].admin_start_sync_file(submission_index - 1)
wait_until( wait_until(
lambda: self.nodes[i].sycn_status_is_completed_or_unknown( lambda: self.nodes[i].sycn_status_is_completed_or_unknown(
@ -91,7 +92,6 @@ class CliSubmissionTest(TestFramework):
) )
) )
wait_until(lambda: self.nodes[i].zgs_get_file_info(root) is not None)
wait_until(lambda: self.nodes[i].zgs_get_file_info(root)["finalized"]) wait_until(lambda: self.nodes[i].zgs_get_file_info(root)["finalized"])
# start_idx = random.randint(0, num_of_entris - 1) # start_idx = random.randint(0, num_of_entris - 1)

View File

@ -22,11 +22,11 @@ class CrashTest(TestFramework):
self.log.info("segment: %s", segment) self.log.info("segment: %s", segment)
for i in range(self.num_nodes): for i in range(self.num_nodes):
self.nodes[i].admin_start_sync_file(0)
self.log.info("wait for node: %s", i)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
) )
self.nodes[i].admin_start_sync_file(0)
self.log.info("wait for node: %s", i)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
) )
@ -48,22 +48,26 @@ class CrashTest(TestFramework):
self.start_storage_node(1) self.start_storage_node(1)
self.nodes[1].wait_for_rpc_connection() self.nodes[1].wait_for_rpc_connection()
self.nodes[1].admin_start_sync_file(1)
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root) is not None) wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root) is not None)
self.nodes[1].admin_start_sync_file(1)
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root)["finalized"]) wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root)["finalized"])
for i in range(2, self.num_nodes): for i in range(2, self.num_nodes):
self.start_storage_node(i) self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection() self.nodes[i].wait_for_rpc_connection()
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(1) self.nodes[i].admin_start_sync_file(1)
self.nodes[i].stop(kill=True) self.nodes[i].stop(kill=True)
self.start_storage_node(i) self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection() self.nodes[i].wait_for_rpc_connection()
self.nodes[i].admin_start_sync_file(1)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
) )
self.nodes[i].admin_start_sync_file(1)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
) )
@ -91,10 +95,10 @@ class CrashTest(TestFramework):
self.log.info("wait for node: %s", i) self.log.info("wait for node: %s", i)
self.start_storage_node(i) self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection() self.nodes[i].wait_for_rpc_connection()
self.nodes[i].admin_start_sync_file(2)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
) )
self.nodes[i].admin_start_sync_file(2)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
) )
@ -132,18 +136,18 @@ class CrashTest(TestFramework):
self.log.info("wait for node: %s", i) self.log.info("wait for node: %s", i)
self.start_storage_node(i) self.start_storage_node(i)
self.nodes[i].wait_for_rpc_connection() self.nodes[i].wait_for_rpc_connection()
self.nodes[i].admin_start_sync_file(4)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1) is not None lambda: self.nodes[i].zgs_get_file_info(data_root1) is not None
) )
self.nodes[i].admin_start_sync_file(4)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root1)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root1)["finalized"]
) )
self.nodes[i].admin_start_sync_file(3)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
) )
self.nodes[i].admin_start_sync_file(3)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
) )

View File

@ -81,6 +81,7 @@ class RpcTest(TestFramework):
for i in range(1, self.num_nodes): for i in range(1, self.num_nodes):
self.log.info("wait node %d", i) self.log.info("wait node %d", i)
wait_until(lambda: self.nodes[i].zgs_get_file_info(root) is not None)
self.nodes[i].admin_start_sync_file(n_files - 1) self.nodes[i].admin_start_sync_file(n_files - 1)
wait_until( wait_until(
lambda: self.nodes[i].sycn_status_is_completed_or_unknown( lambda: self.nodes[i].sycn_status_is_completed_or_unknown(
@ -88,7 +89,6 @@ class RpcTest(TestFramework):
) )
) )
wait_until(lambda: self.nodes[i].zgs_get_file_info(root) is not None)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(root)["finalized"]
) )

View File

@ -76,16 +76,19 @@ class SubmissionTest(TestFramework):
if node_idx == i: if node_idx == i:
continue continue
# Wait for log entry before file sync, otherwise, admin_startSyncFile will be failed.
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
self.nodes[i].admin_start_sync_file(submission_index - 1) self.nodes[i].admin_start_sync_file(submission_index - 1)
wait_until( wait_until(
lambda: self.nodes[i].sycn_status_is_completed_or_unknown( lambda: self.nodes[i].sycn_status_is_completed_or_unknown(
submission_index - 1 submission_index - 1
) )
) )
wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
)
wait_until( wait_until(
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"] lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
) )

View File

@ -5,6 +5,7 @@ import shutil
import stat import stat
import subprocess import subprocess
import sys import sys
import time
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
@ -18,6 +19,9 @@ __file_path__ = os.path.dirname(os.path.realpath(__file__))
CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux" CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
def print_testcase_result(color, glyph, script, start_time):
print(color[1] + glyph + " Testcase " + script + "\telapsed: " + str(int(time.time() - start_time)) + " seconds" + color[0], flush=True)
def run_single_test(py, script, test_dir, index, port_min, port_max): def run_single_test(py, script, test_dir, index, port_min, port_max):
try: try:
# Make sure python thinks it can write unicode to its stdout # Make sure python thinks it can write unicode to its stdout
@ -31,17 +35,16 @@ def run_single_test(py, script, test_dir, index, port_min, port_max):
CIRCLE = "o " CIRCLE = "o "
BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
if os.name == "posix": if os.name == "posix" or os.name == "nt":
# primitive formatting on supported # primitive formatting on supported
# terminal via ANSI escape sequences: # terminal via ANSI escape sequences:
BOLD = ("\033[0m", "\033[1m") BOLD = ("\033[0m", "\033[1m")
BLUE = ("\033[0m", "\033[0;34m") BLUE = ("\033[0m", "\033[0;34m")
RED = ("\033[0m", "\033[0;31m") RED = ("\033[0m", "\033[0;31m")
GREY = ("\033[0m", "\033[1;30m") GREY = ("\033[0m", "\033[1;30m")
print("Running " + script) print("Running " + script, flush=True)
port_min = port_min + (index * PORT_RANGE) % (port_max - port_min) port_min = port_min + (index * PORT_RANGE) % (port_max - port_min)
color = BLUE start_time = time.time()
glyph = TICK
try: try:
subprocess.check_output( subprocess.check_output(
args=[py, script, "--randomseed=1", f"--port-min={port_min}"], args=[py, script, "--randomseed=1", f"--port-min={port_min}"],
@ -49,15 +52,13 @@ def run_single_test(py, script, test_dir, index, port_min, port_max):
cwd=test_dir, cwd=test_dir,
) )
except subprocess.CalledProcessError as err: except subprocess.CalledProcessError as err:
color = RED print_testcase_result(RED, CROSS, script, start_time)
glyph = CROSS print("Output of " + script + "\n" + err.output.decode("utf-8"), flush=True)
print(color[1] + glyph + " Testcase " + script + color[0])
print("Output of " + script + "\n" + err.output.decode("utf-8"))
raise err raise err
print(color[1] + glyph + " Testcase " + script + color[0]) print_testcase_result(BLUE, TICK, script, start_time)
def run(): def run():
start_time = time.time()
dir_name = os.path.join(__file_path__, "tmp") dir_name = os.path.join(__file_path__, "tmp")
if not os.path.exists(dir_name): if not os.path.exists(dir_name):
os.makedirs(dir_name, exist_ok=True) os.makedirs(dir_name, exist_ok=True)
@ -95,7 +96,7 @@ def run():
"", # include test_dir itself "", # include test_dir itself
] ]
slow_tests = {} slow_tests = {"random_test.py", "same_root_test.py"}
long_manual_tests = {"fuzz_test.py"} long_manual_tests = {"fuzz_test.py"}
for subdir in test_subdirs: for subdir in test_subdirs:
@ -136,6 +137,8 @@ def run():
print("CalledProcessError " + repr(err)) print("CalledProcessError " + repr(err))
failed.add(script) failed.add(script)
print("Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
if len(failed) > 0: if len(failed) > 0:
print("The following test fails: ") print("The following test fails: ")
for c in failed: for c in failed:

View File

@ -149,7 +149,7 @@ class TestNode:
else: else:
self.process.terminate() self.process.terminate()
if wait: if wait:
self.wait_until_stopped() self.wait_until_stopped(close_stdout_stderr=False)
# Check that stderr is as expected # Check that stderr is as expected
self.stderr.seek(0) self.stderr.seek(0)
stderr = self.stderr.read().decode("utf-8").strip() stderr = self.stderr.read().decode("utf-8").strip()
@ -169,12 +169,18 @@ class TestNode:
) )
) )
self.stdout.close() self.__safe_close_stdout_stderr__()
self.stderr.close()
self.stdout = None
self.stderr = None
def is_node_stopped(self): def __safe_close_stdout_stderr__(self):
if self.stdout is not None:
self.stdout.close()
self.stdout = None
if self.stderr is not None:
self.stderr.close()
self.stderr = None
def __is_node_stopped__(self):
"""Checks whether the node has stopped. """Checks whether the node has stopped.
Returns True if the node has stopped. False otherwise. Returns True if the node has stopped. False otherwise.
@ -196,8 +202,10 @@ class TestNode:
self.return_code = return_code self.return_code = return_code
return True return True
def wait_until_stopped(self, timeout=20): def wait_until_stopped(self, close_stdout_stderr=True, timeout=20):
wait_until(self.is_node_stopped, timeout=timeout) wait_until(self.__is_node_stopped__, timeout=timeout)
if close_stdout_stderr:
self.__safe_close_stdout_stderr__()
class BlockchainNode(TestNode): class BlockchainNode(TestNode):

View File

@ -326,25 +326,35 @@ class TestFramework:
"--file", "--file",
] ]
proc = subprocess.Popen( output = tempfile.NamedTemporaryFile(dir=self.root_dir, delete=False, prefix="zgs_client_output_")
upload_args + [file_to_upload.name], output_name = output.name
text=True, output_fileno = output.fileno()
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
proc.wait()
root = None try:
lines = proc.stdout.readlines() proc = subprocess.Popen(
for line in lines: upload_args + [file_to_upload.name],
self.log.debug("line: %s", line) text=True,
if "root" in line: stdout=output_fileno,
index = line.find("root=") stderr=output_fileno,
root = line[index + 5 : -1] )
self.log.info("root: %s", root)
return_code = proc.wait(timeout=60)
assert proc.returncode == 0, "%s upload file failed, log: %s" % (self.cli_binary, lines) output.seek(0)
self.log.info("file uploaded") lines = output.readlines()
for line in lines:
line = line.decode("utf-8")
self.log.debug("line: %s", line)
if "root" in line:
index = line.find("root=")
root = line[index + 5 : -1]
except Exception as ex:
self.log.error("Failed to upload file via CLI tool, output: %s", output_name)
raise ex
finally:
output.close()
assert return_code == 0, "%s upload file failed, output: %s, log: %s" % (self.cli_binary, output_name, lines)
return root return root

View File

@ -25,7 +25,7 @@ class RpcCaller:
if isinstance(parsed, Ok): if isinstance(parsed, Ok):
return parsed.result return parsed.result
else: else:
print(parsed) print("Failed to call RPC, method = %s(%s), error = %s" % (self.method, *args, parsed))
except Exception as ex: except Exception as ex:
print(ex) print("Failed to call RPC, method = %s(%s), exception = %s" % (self.method, *args, ex))
return None return None