Deterministic requests in clickhouse workload (ydb-platform#6678)

GrigoriyPA · Jul 15, 2024 · a6eb07c · a6eb07c
1 parent 5a2b13b
commit a6eb07c
Show file tree

Hide file tree

Showing 5 changed files with 165 additions and 85 deletions.
diff --git a/ydb/library/workload/clickbench/click_bench_canonical/q42.result b/ydb/library/workload/clickbench/click_bench_canonical/q42.result
@@ -1,11 +1,11 @@
 "Minute","PageViews"
-"2013-07-15 12:40:00",513
-"2013-07-15 12:41:00",457
-"2013-07-15 12:42:00",470
-"2013-07-15 12:43:00",468
-"2013-07-15 12:44:00",453
-"2013-07-15 12:45:00",462
-"2013-07-15 12:46:00",481
-"2013-07-15 12:47:00",458
-"2013-07-15 12:48:00",466
-"2013-07-15 12:49:00",467
+"22898200",513
+"22898201",457
+"22898202",470
+"22898203",468
+"22898204",453
+"22898205",462
+"22898206",481
+"22898207",458
+"22898208",466
+"22898209",467
diff --git a/ydb/library/workload/clickbench/clickbench.cpp b/ydb/library/workload/clickbench/clickbench.cpp
@@ -82,7 +82,8 @@ TQueryInfoList TClickbenchWorkloadGenerator::GetWorkload(int type) {
             queries.emplace_back(fInput.ReadAll());
         }
     } else {
-        queries = StringSplitter(NResource::Find("click_bench_queries.sql")).Split(';').ToList<TString>();
+        const auto resourceName = Params.IsCheckCannonical() ? "queries-deterministic.sql" : "click_bench_queries.sql";
+        queries = StringSplitter(NResource::Find(resourceName)).Split(';').ToList<TString>();
     }
     auto strVariables = StringSplitter(Params.GetExternalVariablesString()).Split(';').SkipEmpty().ToList<TString>();
     TVector<TExternalVariable> vars;
@@ -130,6 +131,14 @@ TMap<ui32, TString> TClickbenchWorkloadGenerator::LoadExternalResults() const {
             TFileInput fInput(Params.GetExternalResultsDir() / i);
             result.emplace(qId, fInput.ReadAll());
         }
+    } else if (Params.IsCheckCannonical()) {
+        for(ui32 qId = 0; qId < 43; ++qId) {
+            const auto key = "click_bench_canonical/q" + ToString(qId) + ".result";
+            if (!NResource::Has(key)) {
+                continue;
+            }
+            result.emplace(qId, NResource::Find(key));
+        }
     }
     return result;
 }
@@ -153,6 +162,8 @@ void TClickbenchWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const EC
         opts.AddLongOption('q', "ext-query", "String with external queries. Separated by ';'")
             .DefaultValue("")
             .StoreResult(&ExternalQueries);
+        opts.AddLongOption('c', "check-cannonical", "Use deterministic queries and check results with cannonical ones.")
+            .NoArgument().StoreTrue(&CheckCannonicalFlag);
         break;
     default:
         break;

diff --git a/ydb/library/workload/clickbench/clickbench.h b/ydb/library/workload/clickbench/clickbench.h
@@ -18,6 +18,7 @@ class TClickbenchWorkloadParams final: public TWorkloadBaseParams {
     YDB_READONLY_DEF(TString, ExternalVariablesString);
     YDB_READONLY_DEF(TFsPath, ExternalQueriesDir);
     YDB_READONLY_DEF(TFsPath, DataFiles);
+    YDB_READONLY_FLAG(CheckCannonical, false);
 };
 
 class TClickbenchWorkloadGenerator final: public TWorkloadGeneratorBase {

diff --git a/ydb/library/workload/clickbench/ya.make b/ydb/library/workload/clickbench/ya.make
@@ -8,50 +8,51 @@ SRCS(
 
 RESOURCE(
     click_bench_queries.sql click_bench_queries.sql
+    ${ARCADIA_ROOT}/ydb/tests/functional/clickbench/data/queries-deterministic.sql queries-deterministic.sql
     click_bench_schema.sql click_bench_schema.sql
-    click_bench_canonical/q0.result q0.result
-    click_bench_canonical/q1.result q1.result
-    click_bench_canonical/q2.result q2.result
-    click_bench_canonical/q3.result q3.result
-    click_bench_canonical/q4.result q4.result
-    click_bench_canonical/q5.result q5.result
-    click_bench_canonical/q6.result q6.result
-    click_bench_canonical/q7.result q7.result
-    click_bench_canonical/q8.result q8.result
-    click_bench_canonical/q9.result q9.result
-    click_bench_canonical/q10.result q10.result
-    click_bench_canonical/q11.result q11.result
-    click_bench_canonical/q12.result q12.result
-    click_bench_canonical/q13.result q13.result
-    click_bench_canonical/q14.result q14.result
-    click_bench_canonical/q15.result q15.result
-    click_bench_canonical/q16.result q16.result
-    click_bench_canonical/q17.result q17.result
-    click_bench_canonical/q18.result q18.result
-    click_bench_canonical/q19.result q19.result
-    click_bench_canonical/q20.result q20.result
-    click_bench_canonical/q21.result q21.result
-    click_bench_canonical/q22.result q22.result
-    click_bench_canonical/q23.result q23.result
-    click_bench_canonical/q24.result q24.result
-    click_bench_canonical/q25.result q25.result
-    click_bench_canonical/q26.result q26.result
-    click_bench_canonical/q27.result q27.result
-    click_bench_canonical/q28.result q28.result
-    click_bench_canonical/q29.result q29.result
-    click_bench_canonical/q30.result q30.result
-    click_bench_canonical/q31.result q31.result
-    click_bench_canonical/q32.result q32.result
-    click_bench_canonical/q33.result q33.result
-    click_bench_canonical/q34.result q34.result
-    click_bench_canonical/q35.result q35.result
-    click_bench_canonical/q36.result q36.result
-    click_bench_canonical/q37.result q37.result
-    click_bench_canonical/q38.result q38.result
-    click_bench_canonical/q39.result q39.result
-    click_bench_canonical/q40.result q40.result
-    click_bench_canonical/q41.result q41.result
-    click_bench_canonical/q42.result q42.result
+    click_bench_canonical/q0.result click_bench_canonical/q0.result
+    click_bench_canonical/q1.result click_bench_canonical/q1.result
+    click_bench_canonical/q2.result click_bench_canonical/q2.result
+    click_bench_canonical/q3.result click_bench_canonical/q3.result
+    click_bench_canonical/q4.result click_bench_canonical/q4.result
+    click_bench_canonical/q5.result click_bench_canonical/q5.result
+    click_bench_canonical/q6.result click_bench_canonical/q6.result
+    click_bench_canonical/q7.result click_bench_canonical/q7.result
+    click_bench_canonical/q8.result click_bench_canonical/q8.result
+    click_bench_canonical/q9.result click_bench_canonical/q9.result
+    click_bench_canonical/q10.result click_bench_canonical/q10.result
+    click_bench_canonical/q11.result click_bench_canonical/q11.result
+    click_bench_canonical/q12.result click_bench_canonical/q12.result
+    click_bench_canonical/q13.result click_bench_canonical/q13.result
+    click_bench_canonical/q14.result click_bench_canonical/q14.result
+    click_bench_canonical/q15.result click_bench_canonical/q15.result
+    click_bench_canonical/q16.result click_bench_canonical/q16.result
+    click_bench_canonical/q17.result click_bench_canonical/q17.result
+    click_bench_canonical/q18.result click_bench_canonical/q18.result
+    click_bench_canonical/q19.result click_bench_canonical/q19.result
+    click_bench_canonical/q20.result click_bench_canonical/q20.result
+    click_bench_canonical/q21.result click_bench_canonical/q21.result
+    click_bench_canonical/q22.result click_bench_canonical/q22.result
+    click_bench_canonical/q23.result click_bench_canonical/q23.result
+    click_bench_canonical/q24.result click_bench_canonical/q24.result
+    click_bench_canonical/q25.result click_bench_canonical/q25.result
+    click_bench_canonical/q26.result click_bench_canonical/q26.result
+    click_bench_canonical/q27.result click_bench_canonical/q27.result
+    click_bench_canonical/q28.result click_bench_canonical/q28.result
+    click_bench_canonical/q29.result click_bench_canonical/q29.result
+    click_bench_canonical/q30.result click_bench_canonical/q30.result
+    click_bench_canonical/q31.result click_bench_canonical/q31.result
+    click_bench_canonical/q32.result click_bench_canonical/q32.result
+    click_bench_canonical/q33.result click_bench_canonical/q33.result
+    click_bench_canonical/q34.result click_bench_canonical/q34.result
+    click_bench_canonical/q35.result click_bench_canonical/q35.result
+    click_bench_canonical/q36.result click_bench_canonical/q36.result
+    click_bench_canonical/q37.result click_bench_canonical/q37.result
+    click_bench_canonical/q38.result click_bench_canonical/q38.result
+    click_bench_canonical/q39.result click_bench_canonical/q39.result
+    click_bench_canonical/q40.result click_bench_canonical/q40.result
+    click_bench_canonical/q41.result click_bench_canonical/q41.result
+    click_bench_canonical/q42.result click_bench_canonical/q42.result
 )
 
 PEERDIR(

diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp
@@ -129,9 +129,12 @@ void ThrowOnError(const TStatus& status) {
 }
 
 bool HasCharsInString(const TString& str) {
-    for (auto c : str) {
-        if (std::isalpha(c)) {
-            return true;
+    for(TStringBuf q(str), line; q.ReadLine(line);) {
+        line = line.NextTok("--");
+        for (auto c: line) {
+            if (std::isalpha(c)) {
+                return true;
+            }
         }
     }
     return false;
@@ -370,41 +373,105 @@ bool CompareValueImpl(const T& valResult, TStringBuf vExpected) {
     return valResult == valExpected;
 }
 
-bool CompareValue(const NYdb::TValue& v, TStringBuf vExpected) {
-    const auto& vp = v.GetProto();
-    if (vp.has_bool_value()) {
-        return CompareValueImpl<bool>(vp.bool_value(), vExpected);
-    }
-    if (vp.has_int32_value()) {
-        return CompareValueImpl<i32>(vp.int32_value(), vExpected);
-    }
-    if (vp.has_uint32_value()) {
-        return CompareValueImpl<ui32>(vp.uint32_value(), vExpected);
-    }
-    if (vp.has_int64_value()) {
-        return CompareValueImpl<i64>(vp.int64_value(), vExpected);
-    }
-    if (vp.has_uint64_value()) {
-        return CompareValueImpl<ui64>(vp.uint64_value(), vExpected);
-    }
-    if (vp.has_float_value()) {
-        return CompareValueImpl<float>(vp.float_value(), vExpected);
+template <class T>
+bool CompareValueImplFloat(const T& valResult, TStringBuf vExpected, const double floatPrecesion) {
+    T valExpected;
+    if (!TryFromString<T>(vExpected, valExpected)) {
+        Cerr << "cannot parse expected as " << typeid(valResult).name() << "(" << vExpected << ")" << Endl;
+        return false;
     }
-    if (vp.has_double_value()) {
-        return CompareValueImpl<double>(vp.double_value(), vExpected);
+    return valResult > (1 - floatPrecesion) * valExpected && valResult < (1 + floatPrecesion) * valExpected;
+}
+
+bool CompareValueImplDatetime(const TInstant& valResult, TStringBuf vExpected, TDuration unit) {
+    TInstant expected;
+    if (!TInstant::TryParseIso8601(vExpected, expected)) {
+        i64 i;
+        if (!TryFromString(vExpected, i)) {
+            Cerr << "cannot parse expected as " << typeid(valResult).name() << "(" << vExpected << ")" << Endl;
+            return false;
+        }
+        expected = TInstant::Zero() + i * unit;
     }
-    if (vp.has_text_value()) {
-        return CompareValueImpl<TString>(TString(vp.text_value().data(), vp.text_value().size()), vExpected);
+    return valResult == expected;
+}
+
+template<class T>
+bool CompareValueImplDatetime64(const T& valResult, TStringBuf vExpected, TDuration unit) {
+    T valExpected;
+    if (!TryFromString<T>(vExpected, valExpected)) {
+        TInstant expected;
+        if (!TInstant::TryParseIso8601(vExpected, expected)) {
+            Cerr << "cannot parse expected as " << typeid(valResult).name() << "(" << vExpected << ")" << Endl;
+            return false;
+        }
+        valExpected = expected.GetValue() / unit.GetValue();
     }
-    if (vp.has_null_flag_value()) {
-        return vExpected == "";
+    return valResult == valExpected;
+}
+
+bool CompareValue(const NYdb::TValue& v, TStringBuf vExpected, double floatPrecession) {
+    TValueParser vp(v);
+    TTypeParser tp(v.GetType());
+    if (tp.GetKind() == TTypeParser::ETypeKind::Optional) {
+        if (vp.IsNull()) {
+            return vExpected == "";
+        }
+        vp.OpenOptional();
+        tp.OpenOptional();
+    }
+    switch (tp.GetPrimitive()) {
+    case EPrimitiveType::Bool:
+        return CompareValueImpl(vp.GetBool(), vExpected);
+    case EPrimitiveType::Int8:
+        return CompareValueImpl(vp.GetInt8(), vExpected);
+    case EPrimitiveType::Uint8:
+        return CompareValueImpl(vp.GetUint8(), vExpected);
+    case EPrimitiveType::Int16:
+        return CompareValueImpl(vp.GetInt16(), vExpected);
+    case EPrimitiveType::Uint16:
+        return CompareValueImpl(vp.GetUint16(), vExpected);
+    case EPrimitiveType::Int32:
+        return CompareValueImpl(vp.GetInt32(), vExpected);
+    case EPrimitiveType::Uint32:
+        return CompareValueImpl(vp.GetUint32(), vExpected);
+    case EPrimitiveType::Int64:
+        return CompareValueImpl(vp.GetInt64(), vExpected);
+    case EPrimitiveType::Uint64:
+        return CompareValueImpl(vp.GetUint64(), vExpected);
+    case EPrimitiveType::Float:
+        return CompareValueImplFloat(vp.GetFloat(), vExpected, floatPrecession);
+    case EPrimitiveType::Double:
+        return CompareValueImplFloat(vp.GetDouble(), vExpected, floatPrecession);
+    case EPrimitiveType::Date:
+        return CompareValueImplDatetime(vp.GetDate(), vExpected, TDuration::Days(1));
+    case EPrimitiveType::Datetime:
+        return CompareValueImplDatetime(vp.GetDatetime(), vExpected, TDuration::Seconds(1));
+    case EPrimitiveType::Timestamp:
+        return CompareValueImplDatetime(vp.GetTimestamp(), vExpected, TDuration::MicroSeconds(1));
+    case EPrimitiveType::Interval:
+        return CompareValueImpl(vp.GetInterval(), vExpected);
+    case EPrimitiveType::Date32:
+        return CompareValueImplDatetime64(vp.GetDate32(), vExpected, TDuration::Days(1));
+    case EPrimitiveType::Datetime64:
+        return CompareValueImplDatetime64(vp.GetDatetime64(), vExpected, TDuration::Seconds(1));
+    case EPrimitiveType::Timestamp64:
+        return CompareValueImplDatetime64(vp.GetTimestamp64(), vExpected, TDuration::MicroSeconds(1));
+    case EPrimitiveType::Interval64:
+        return CompareValueImpl(vp.GetInterval64(), vExpected);
+    case EPrimitiveType::String:
+        return CompareValueImpl(vp.GetString(), vExpected);
+    case EPrimitiveType::Utf8:
+        return CompareValueImpl(vp.GetUtf8(), vExpected);
+    default:
+        Cerr << "unexpected type for comparision: " << v.GetProto().DebugString() << Endl;
+        return false;
     }
-    Cerr << "unexpected type for comparision: " << vp.DebugString() << Endl;
-    return false;
 }
 
 
 bool TQueryResultInfo::IsExpected(std::string_view expected) const {
+    constexpr double floatPrecesion = 0.0001;
     if (expected.empty()) {
         return true;
     }
@@ -452,7 +519,7 @@ bool TQueryResultInfo::IsExpected(std::string_view expected) const {
                 return false;
             }
             TStringBuf cItem = splitter.Consume();
-            if (!CompareValue(resultValue, cItem)) {
+            if (!CompareValue(resultValue, cItem, floatPrecesion)) {
                 Cerr << "has diff: " << resultValue.GetProto().DebugString() << ";EXPECTED:" << cItem << Endl;
                 return false;
             }