Skip to content

Commit

Permalink
Fix iterations reporting (ydb-platform#10792)
Browse files Browse the repository at this point in the history
  • Loading branch information
@iddqdex
iddqdex authored Oct 23, 2024
1 parent cb09349 commit 1d7807f
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 18 deletions.
33 changes: 21 additions & 12 deletions ydb/tests/olap/lib/ydb_cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import yatest.common
import json
import os
import re
from ydb.tests.olap.lib.ydb_cluster import YdbCluster
from ydb.tests.olap.lib.utils import get_external_param
from enum import StrEnum
Expand Down Expand Up @@ -39,12 +40,13 @@ class WorkloadRunResult:
def __init__(self):
self.stats: dict[str, dict[str, Any]] = {}
self.query_out: Optional[str] = None
self.stdout: Optional[str] = None
self.stderr: Optional[str] = None
self.stdout: str = ''
self.stderr: str = ''
self.error_message: str = ''
self.plans: Optional[list[YdbCliHelper.QueryPlan]] = None
self.explain_plan: Optional[YdbCliHelper.QueryPlan] = None
self.errors_by_iter: dict[int, str] = {}
self.time_by_iter: dict[int, float] = {}
self.traceback: Optional[TracebackType] = None

@property
Expand Down Expand Up @@ -81,26 +83,26 @@ def _add_error(self, msg: Optional[str]):
else:
self.result.error_message = msg

def _process_returncode(self, returncode, stderr: str) -> None:
def _process_returncode(self, returncode) -> None:
begin_str = f'{self.query_num}:'
end_str = 'Query text:'
iter_str = 'iteration '
begin_pos = stderr.find(begin_str)
begin_pos = self.result.stderr.find(begin_str)
if begin_pos >= 0:
while True:
begin_pos = stderr.find(iter_str, begin_pos)
begin_pos = self.result.stderr.find(iter_str, begin_pos)
if begin_pos < 0:
break
begin_pos += len(iter_str)
end_pos = stderr.find('\n', begin_pos)
end_pos = self.result.stderr.find('\n', begin_pos)
if end_pos < 0:
iter = int(stderr[begin_pos:])
begin_pos = len(stderr) - 1
iter = int(self.result.stderr[begin_pos:])
begin_pos = len(self.result.stderr) - 1
else:
iter = int(stderr[begin_pos:end_pos])
iter = int(self.result.stderr[begin_pos:end_pos])
begin_pos = end_pos + 1
end_pos = stderr.find(end_str, begin_pos)
msg = (stderr[begin_pos:] if end_pos < 0 else stderr[begin_pos:end_pos]).strip()
end_pos = self.result.stderr.find(end_str, begin_pos)
msg = (self.result.stderr[begin_pos:] if end_pos < 0 else self.result.stderr[begin_pos:end_pos]).strip()
self.result.errors_by_iter[iter] = msg
self._add_error(f'Iteration {iter}: {msg}')
if returncode != 0 and len(self.result.errors_by_iter) == 0:
Expand Down Expand Up @@ -167,6 +169,12 @@ def _check_nodes(self):
node_errors.append(f'Node {node} is down')
self._add_error('\n'.join(node_errors))

def _parse_stdout(self):
for line in self.result.stdout.splitlines():
m = re.search(r'iteration ([0-9]*):\s*ok\s*([\.0-9]*)s', line)
if m is not None:
self.result.time_by_iter[int(m.group(1))] = float(m.group(2))

def _get_cmd(self) -> list[str]:
cmd = YdbCliHelper.get_cli_command() + [
'-e', YdbCluster.ydb_endpoint,
Expand All @@ -192,7 +200,8 @@ def _exec_cli(self) -> None:
process = yatest.common.process.execute(self._get_cmd(), check_exit_code=False)
self.result.stdout = process.stdout.decode('utf-8', 'replace')
self.result.stderr = process.stderr.decode('utf-8', 'replace')
self._process_returncode(process.returncode, self.result.stderr)
self._process_returncode(process.returncode)
self._parse_stdout()

def process(self) -> YdbCliHelper.WorkloadRunResult:
try:
Expand Down
19 changes: 13 additions & 6 deletions ydb/tests/olap/load/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ def _get_duraton(stats, field):
result = stats.get(field)
return float(result) / 1e3 if result is not None else None

def _duration_text(duration: float | int):
s = f'{int(duration)}s ' if duration >= 1 else ''
return f'{s}{int(duration * 1000) % 1000}ms'

def _attach_plans(plan: YdbCliHelper.QueryPlan) -> None:
if plan.plan is not None:
allure.attach(json.dumps(plan.plan), 'Plan json', attachment_type=allure.attachment_type.JSON)
Expand All @@ -95,10 +99,15 @@ def _attach_plans(plan: YdbCliHelper.QueryPlan) -> None:
_attach_plans(result.explain_plan)

if result.plans is not None:
for i in range(self._get_iterations(query_num)):
for i in range(iterations):
s = allure.step(f'Iteration {i}')
if i in result.time_by_iter:
s.params['duration'] = _duration_text(result.time_by_iter[i])
try:
with allure.step(f'Iteration {i}'):
with s:
_attach_plans(result.plans[i])
if i in result.time_by_iter:
allure.dynamic.parameter('duration', _duration_text(result.time_by_iter[i]))
if i in result.errors_by_iter:
pytest.fail(result.errors_by_iter[i])
except BaseException:
Expand All @@ -120,9 +129,7 @@ def _attach_plans(plan: YdbCliHelper.QueryPlan) -> None:
allure.attach(result.stderr, 'Stderr', attachment_type=allure.attachment_type.TEXT)
for p in ['Mean']:
if p in stats:
value = int(stats[p])
s = f'{int(value / 1000)}s ' if value >= 1000 else ''
allure.dynamic.parameter(p, f'{s}{value % 1000}ms')
allure.dynamic.parameter(p, _duration_text(stats[p] / 1000.))
error_message = ''
success = True
if not result.success:
Expand Down Expand Up @@ -191,4 +198,4 @@ def run_workload_test(self, path: str, query_num: int) -> None:
check_canonical=self.check_canonical
)
allure_test_description(self.suite, self._test_name(query_num), refference_set=self.refference, start_time=start_time, end_time=time())
self.process_query_result(result, query_num, self.iterations, True)
self.process_query_result(result, query_num, self._get_iterations(query_num), True)

0 comments on commit 1d7807f

Please sign in to comment.