Add reconciliation test for test-run logs

Execute runner.sh for finding negative tests and compare test-run logs
of  new version and golden version from result directory. Also, these
suite consists of size comparing test and check that the log is not
empty.

test/negative_tests/conftest.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+
+import pytest
+from helpers import eliminate_support_info
+
+
+@pytest.fixture
+def gold_log(test_name) -> Path:
+    return eliminate_support_info(
+        f"./result/{test_name}.result.log", suffix="gold")
+
+
+@pytest.fixture
+def curr_log(test_name) -> Path:
+    return eliminate_support_info(f"./output/{test_name}", suffix="test")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--my-parameter", action="store", default=None,
+        help="Parameter description"
+    )
+
+
+@pytest.fixture(scope='session')
+def test_name(pytestconfig):
+    """ Description of changes triggered by parameter. """
+    param = pytestconfig.getoption("--my-parameter")
+    if param is None:
+        assert False
+    return param

test/negative_tests/engine.cfg

@@ -0,0 +1,15 @@
+{
+    "set_language.test.sql": {
+        "memtx": {"engine": "memtx"}
+    },
+    "setopt_delimeter.test.lua": {
+        "memtx": {"engine": "memtx"}
+    },
+    "worker_hang_when_gc_triggered_inside_colorer.test.lua": {
+        "vinyl": {"engine": "vinyl"}
+    },
+    "*": {
+        "memtx": {"engine": "memtx"},
+        "vinyl": {"engine": "vinyl"}
+    }
+}

test/negative_tests/helpers.py

@@ -0,0 +1,30 @@
+import re
+from pathlib import Path
+
+
+def delete_hash(line: str) -> str:
+    """Delete commit hash"""
+    return re.sub(r'[0-9a-z]{32}|[0-9a-z-]{36}', '', line)
+
+
+def delete_timestamp(line: str) -> str:
+    """Delete timestamp"""
+    return re.sub(r'[\d-]{10} [\d:.]{12}', '', line)
+
+
+def delete_instance(line: str) -> str:
+    """Delete instance number"""
+    return re.sub(r'\[\d*\]', '', line)
+
+
+def eliminate_support_info(path: str, suffix: str) -> Path:
+    """Delete information (hash, time, instance), which could be changed."""
+    if not suffix.startswith("_"):
+        suffix = "_" + suffix
+    with open(path+suffix, 'w') as file, open(path, 'r') as log:
+        for line in log:
+            line = delete_hash(line)
+            line = delete_instance(line)
+            line = delete_timestamp(line)
+            file.write(line.strip() + "\n")
+    return Path(path+suffix)

test/negative_tests/runner.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -eu
+
+rm -rf output
+mkdir output
+FILES_LIST="$(ls *.test.lua 2>/dev/null)"
+echo "$FILES_LIST"
+for test in $FILES_LIST; do
+    printf 'Starting %s\n' "${test}"
+
+    suite="${test%/}"
+    output_file="output/${test}"
+    ../../../test/test-run.py "${test}" > "${output_file}" 2>&1 || true
+    pytest ./test_reconcile_logs.py -vv --my-parameter "$suite"
+done

test/negative_tests/suite.ini

@@ -0,0 +1,7 @@
+[default]
+core = tarantool
+description = tarantool tests
+script = box.lua
+use_unix_sockets = True
+pretest_clean = True
+config = engine.cfg

test/negative_tests/test_reconcile_logs.py

@@ -0,0 +1,17 @@
+import pytest
+from pathlib import Path
+
+
+@pytest.mark.usefixtures("test_name")
+class TestReconciliationLogs:
+
+    def test_log_is_not_empty(self, curr_log: Path) -> None:
+        assert curr_log.stat().st_size
+
+    def test_log_sizes_are_equal(self, curr_log: Path, gold_log: Path) -> None:
+        assert curr_log.stat().st_size == gold_log.stat().st_size
+
+    def test_reconcile_logs(self, curr_log: Path, gold_log: Path) -> None:
+        with open(curr_log, 'r') as tl, open(gold_log, 'r') as gl:
+            for new_line, gold_line in zip(tl, gl):
+                assert new_line == gold_line