From 57fdf1af48f684e3be6f240b361a66467bac808a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 11 Jan 2025 16:10:49 +0800 Subject: [PATCH 01/49] fix: mixed mode query can not calc nthread --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 566dff5a..c5beb87e 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -687,7 +687,7 @@ static int multi_thread_specified_mixed_query(uint16_t iface, char* dbName) { benchArrayDestroy(delay_list); code = 0; g_queryInfo.specifiedQueryInfo.totalQueried = - g_queryInfo.specifiedQueryInfo.sqls->size * g_queryInfo.specifiedQueryInfo.queryTimes * thread; + g_queryInfo.specifiedQueryInfo.sqls->size * g_queryInfo.specifiedQueryInfo.queryTimes; OVER: tmfree(pids); From 2b645f21d3a4fe7c125d5588b484493c58b4f39d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 11 Jan 2025 16:13:46 +0800 Subject: [PATCH 02/49] fix: modify case with mixed-query no calc thread --- tests/taosbenchmark/bugs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/taosbenchmark/bugs.py b/tests/taosbenchmark/bugs.py index f668b3c1..610e9989 100644 --- a/tests/taosbenchmark/bugs.py +++ b/tests/taosbenchmark/bugs.py @@ -156,6 +156,7 @@ def bugsTS(self, benchmark): # TS-5846 keys = ["completed total queries: 40"] self.benchmarkQuery(benchmark, "./taosbenchmark/json/TS-5846-Query.json", keys) + keys = ["completed total queries: 20"] self.benchmarkQuery(benchmark, "./taosbenchmark/json/TS-5846-Mixed-Query.json", keys) # bugs td From 98b320b19efe226b390fd2ba624a5408baa5e7b1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 11 Jan 2025 18:10:33 +0800 Subject: [PATCH 03/49] normal query mode add each sql QPS metric --- src/benchQuery.c | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index c5beb87e..e0c926bb 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -489,6 +489,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { g_arguments->terminate = true; } + int64_t start = toolsGetTimestampUs(); // wait threads execute finished one by one for (int j = 0; j < threadCnt ; j++) { pthread_join(pids[j], NULL); @@ -512,6 +513,11 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { pThreadInfo->query_delay_list = NULL; } } + int64_t spend = toolsGetTimestampUs() - start; + if(spend == 0) { + // avoid xx/spend expr throw error + spend = 1; + } // cancel or need exit check if (g_fail || g_arguments->terminate) { @@ -540,17 +546,20 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } avg_delay /= nConcurrent; qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); - infoPrintNoTimestamp("complete query with %d threads and %" PRIu64 - " query delay " - "avg: \t%.6fs " - "min: \t%.6fs " - "max: \t%.6fs " - "p90: \t%.6fs " - "p95: \t%.6fs " - "p99: \t%.6fs " + infoPrintNoTimestamp("complete query with %d threads and %" PRIu64 " " + "sql %"PRIu64" spend %.6fs QPS: %.3f " + "query delay " + "avg: %.6fs " + "min: %.6fs " + "max: %.6fs " + "p90: %.6fs " + "p95: %.6fs " + "p99: %.6fs " "SQL command: %s" "\n", - nConcurrent, query_times, avg_delay / 1E6, /* avg */ + nConcurrent, query_times, + i, spend/1E6, totalQueryTimes / (spend/1E6), + avg_delay / 1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ /* p90 */ @@ -560,16 +569,19 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { /* p99 */ sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); infoPrintNoTimestampToFile("complete query with %d threads and %" PRIu64 + "sql %"PRIu64" spend %.6fs QPS: %.3f " " query delay " - "avg: \t%.6fs " - "min: \t%.6fs " - "max: \t%.6fs " - "p90: \t%.6fs " - "p95: \t%.6fs " - "p99: \t%.6fs " + "avg: %.6fs " + "min: %.6fs " + "max: %.6fs " + "p90: %.6fs " + "p95: %.6fs " + "p99: %.6fs " "SQL command: %s" "\n", - nConcurrent, query_times, avg_delay / 1E6, /* avg */ + nConcurrent, query_times, + i, spend/1E6, totalQueryTimes / (spend/1E6), + avg_delay / 1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ /* p90 */ From dfedf5aabe90a787fade376bc10d099878b443d8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 11 Jan 2025 18:12:21 +0800 Subject: [PATCH 04/49] sql index start from 1 --- src/benchQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index e0c926bb..070c7cf8 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -558,7 +558,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { "SQL command: %s" "\n", nConcurrent, query_times, - i, spend/1E6, totalQueryTimes / (spend/1E6), + i + 1, spend/1E6, totalQueryTimes / (spend/1E6), avg_delay / 1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ @@ -580,7 +580,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { "SQL command: %s" "\n", nConcurrent, query_times, - i, spend/1E6, totalQueryTimes / (spend/1E6), + i + 1, spend/1E6, totalQueryTimes / (spend/1E6), avg_delay / 1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ From 6b8bfe2e9b3f997633f02542c0c741cbb3d4d8c0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 13 Jan 2025 14:35:53 +0800 Subject: [PATCH 05/49] use buffer to write screen and file --- src/benchQuery.c | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 070c7cf8..75cd65c2 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -546,7 +546,9 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } avg_delay /= nConcurrent; qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); - infoPrintNoTimestamp("complete query with %d threads and %" PRIu64 " " + int32_t bufLen = strlen(sql->command) + 512; + char * buf = benchCalloc(bufLen); + snprintf(buf , bufLen, "complete query with %d threads and %" PRIu64 " " "sql %"PRIu64" spend %.6fs QPS: %.3f " "query delay " "avg: %.6fs " @@ -567,29 +569,10 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { /* p95 */ sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)] / 1E6, /* p99 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); - infoPrintNoTimestampToFile("complete query with %d threads and %" PRIu64 - "sql %"PRIu64" spend %.6fs QPS: %.3f " - " query delay " - "avg: %.6fs " - "min: %.6fs " - "max: %.6fs " - "p90: %.6fs " - "p95: %.6fs " - "p99: %.6fs " - "SQL command: %s" - "\n", - nConcurrent, query_times, - i + 1, spend/1E6, totalQueryTimes / (spend/1E6), - avg_delay / 1E6, /* avg */ - sql->delay_list[0] / 1E6, /* min */ - sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ - /* p90 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)] / 1E6, - /* p95 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)] / 1E6, - /* p99 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); + sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); + infoPrintNoTimestamp("%s", buf); + infoPrintNoTimestampToFile("%s", buf); + tmfree(buf); } g_queryInfo.specifiedQueryInfo.totalQueried = From e5cea34e60cb758a6ca8c3d1f7f31682821b3358 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 13 Jan 2025 14:47:06 +0800 Subject: [PATCH 06/49] fix: benchCalloc mistake --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 75cd65c2..a26c485f 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -547,7 +547,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { avg_delay /= nConcurrent; qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); int32_t bufLen = strlen(sql->command) + 512; - char * buf = benchCalloc(bufLen); + char * buf = benchCalloc(bufLen, sizeof(char), false); snprintf(buf , bufLen, "complete query with %d threads and %" PRIu64 " " "sql %"PRIu64" spend %.6fs QPS: %.3f " "query delay " From 12fca3dc51f68c2335c40d4c9286d1c7d260861e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 11:52:16 +0800 Subject: [PATCH 07/49] enh: organize code for query --- src/benchJsonOpt.c | 5 ++ src/benchQuery.c | 213 +++++++++++++++++++++++++++------------------ 2 files changed, 134 insertions(+), 84 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 57b18748..1c27037f 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1076,6 +1076,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { return -1; } + // read from super table tools_cJSON *continueIfFail = tools_cJSON_GetObjectItem(stbInfo, "continue_if_fail"); // yes, no, if (tools_cJSON_IsString(continueIfFail)) { @@ -1090,6 +1091,9 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { continueIfFail->valuestring); return -1; } + } else { + // default value is common specialed + superTable->continueIfFail = g_arguments->continueIfFail; } // start_fillback_time @@ -1560,6 +1564,7 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { } } + // read from common tools_cJSON *continueIfFail = tools_cJSON_GetObjectItem(json, "continue_if_fail"); // yes, no, if (tools_cJSON_IsString(continueIfFail)) { diff --git a/src/benchQuery.c b/src/benchQuery.c index a26c485f..0ab83bf9 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -755,6 +755,115 @@ void *queryKiller(void *arg) { return NULL; } +// +int killSlowQuery() { + pthread_t pidKiller = {0}; + int32_t ret = pthread_create(&pidKiller, NULL, queryKiller, NULL); + if (ret != 0) { + errorPrint("pthread_create failed create queryKiller thread. error code =%d \n", ret); + return -1; + } + pthread_join(pidKiller, NULL); + toolsMsleep(1000); + return 0; +} + +// fetch super table child name from server +int fetchChildTableName(char *dbName, char *stbName) { + SBenchConn* conn = initBenchConn(); + if (conn == NULL) { + return -1; + } + + // get child count + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + if (3 == g_majorVersionOfClient) { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", + dbName, stbName); + } else { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT COUNT(TBNAME) FROM %s.%s", + dbName, superQueryInfo.stbName); + } + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return -1; + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == strlen((char *)(row[0]))) { + errorPrint("stable %s have no child table\n", stbName); + taos_free_result(res); + closeBenchConn(conn); + return -1; + } + char temp[256] = {0}; + taos_print_row(temp, row, fields, num_fields); + + // set child table count + g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); + } + infoPrint("%s's childTblCount: %" PRId64 "\n", stbName, g_queryInfo.superQueryInfo.childTblCount); + taos_free_result(res); + + // malloc memory with child table count + g_queryInfo.superQueryInfo.childTblName = + benchCalloc(g_queryInfo.superQueryInfo.childTblCount, + sizeof(char *), false); + // fetch child table name + if (getAllChildNameOfSuperTable( + conn->taos, dbName, stbName, + g_queryInfo.superQueryInfo.childTblName, + g_queryInfo.superQueryInfo.childTblCount)) { + // faild + tmfree(g_queryInfo.superQueryInfo.childTblName); + closeBenchConn(conn); + return -1; + } + closeBenchConn(conn); + + // succ + return 0; +} + +// total query for end +void totalQuery(int64_t spends) { + + double tInS = (double)spends / 1000; + + // specifiedQuery + if (g_queryInfo.specifiedQueryInfo.totalQueried) { + infoPrint("Total specified queries: %" PRIu64 "\n", + g_queryInfo.specifiedQueryInfo.totalQueried); + } + + // superQuery + if (g_queryInfo.superQueryInfo.totalQueried) { + infoPrint("Total super queries: %" PRIu64 "\n", + g_queryInfo.superQueryInfo.totalQueried); + } + + // total QPS + uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + + g_queryInfo.superQueryInfo.totalQueried; + infoPrint( + "Spend %.4f second completed total queries: %" PRIu64 + ", the QPS of all threads: %10.3f\n\n", + tInS, totalQueried, (double)totalQueried / tInS); + infoPrintToFile( + "Spend %.4f second completed total queries: %" PRIu64 + ", the QPS of all threads: %10.3f\n\n", + tInS, totalQueried, (double)totalQueried / tInS); + +} + int queryTestProcess() { prompt(0); @@ -762,13 +871,15 @@ int queryTestProcess() { encodeAuthBase64(); } - pthread_t pidKiller = {0}; + // kill sql for executing seconds over "kill_slow_query_threshold" if (g_queryInfo.iface == TAOSC_IFACE && g_queryInfo.killQueryThreshold) { - pthread_create(&pidKiller, NULL, queryKiller, NULL); - pthread_join(pidKiller, NULL); - toolsMsleep(1000); + int32_t ret = killSlowQuery(); + if (ret != 0) { + return ret; + } } + // covert addr if (g_queryInfo.iface == REST_IFACE) { if (convertHostToServAddr(g_arguments->host, g_arguments->port + TSDB_PORT_HTTP, @@ -778,105 +889,39 @@ int queryTestProcess() { } } + // fetch child name if super table if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - SBenchConn* conn = initBenchConn(); - if (conn == NULL) { - return -1; - } - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (3 == g_majorVersionOfClient) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(TBNAME) FROM %s.%s", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - closeBenchConn(conn); - return -1; - } - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == strlen((char *)(row[0]))) { - errorPrint("stable %s have no child table\n", - g_queryInfo.superQueryInfo.stbName); - taos_free_result(res); - closeBenchConn(conn); - return -1; - } - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); - } - infoPrint("%s's childTblCount: %" PRId64 "\n", - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblCount); - taos_free_result(res); - g_queryInfo.superQueryInfo.childTblName = - benchCalloc(g_queryInfo.superQueryInfo.childTblCount, - sizeof(char *), false); - if (getAllChildNameOfSuperTable( - conn->taos, g_queryInfo.dbName, - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblName, - g_queryInfo.superQueryInfo.childTblCount)) { - tmfree(g_queryInfo.superQueryInfo.childTblName); - closeBenchConn(conn); - return -1; - } - closeBenchConn(conn); + int32_t ret = fetchChildTableName(g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); } + + // + // start running + // + + // specified table uint64_t startTs = toolsGetTimestampMs(); if (g_queryInfo.specifiedQueryInfo.mixed_query) { + // mixed if (multi_thread_specified_mixed_query(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } else { + // no mixied if (multi_thread_specified_table_query(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } + + // super table if (multi_thread_super_table_query(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } - // workaround to use separate taos connection; - uint64_t endTs = toolsGetTimestampMs(); - int64_t t = endTs - startTs; - double tInS = (double)t / 1000.0; - // specifiedQuery - if (g_queryInfo.specifiedQueryInfo.totalQueried) { - infoPrint("Total specified queries: %" PRIu64 "\n", - g_queryInfo.specifiedQueryInfo.totalQueried); - } - - // superQuery - if (g_queryInfo.superQueryInfo.totalQueried) { - infoPrint("Total super queries: %" PRIu64 "\n", - g_queryInfo.superQueryInfo.totalQueried); - } - - // total QPS - uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried - + g_queryInfo.superQueryInfo.totalQueried; - infoPrint( - "Spend %.4f second completed total queries: %" PRIu64 - ", the QPS of all threads: %10.3f\n\n", - tInS, totalQueried, (double)totalQueried / tInS); - infoPrintToFile( - "Spend %.4f second completed total queries: %" PRIu64 - ", the QPS of all threads: %10.3f\n\n", - tInS, totalQueried, (double)totalQueried / tInS); + // total + totalQuery(toolsGetTimestampMs() - startTs); return 0; } From 28d07b7eec68fba459a9531259dc5b74645d52d7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 12:16:16 +0800 Subject: [PATCH 08/49] fix: change filename with uniform name --- src/benchQuery.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 0ab83bf9..bc7059ec 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -61,10 +61,11 @@ int selectAndGetResult(threadInfo *pThreadInfo, char *command) { return ret; } -static void *mixedQuery(void *sarg) { +// spec query mixed thread +static void *specQueryMixThread(void *sarg) { queryThreadInfo *pThreadInfo = (queryThreadInfo*)sarg; #ifdef LINUX - prctl(PR_SET_NAME, "mixedQuery"); + prctl(PR_SET_NAME, "specQueryMixThread"); #endif // use db if (g_queryInfo.dbName) { @@ -156,10 +157,11 @@ static void *mixedQuery(void *sarg) { return NULL; } -static void *specifiedTableQuery(void *sarg) { +// spec query thread +static void *specQueryThread(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; #ifdef LINUX - prctl(PR_SET_NAME, "specTableQuery"); + prctl(PR_SET_NAME, "specQueryThread"); #endif uint64_t st = 0; uint64_t et = 0; @@ -260,11 +262,12 @@ static void *specifiedTableQuery(void *sarg) { return NULL; } -static void *superTableQuery(void *sarg) { +// super table query thread +static void *stbQueryThread(void *sarg) { char *sqlstr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); threadInfo *pThreadInfo = (threadInfo *)sarg; #ifdef LINUX - prctl(PR_SET_NAME, "superTableQuery"); + prctl(PR_SET_NAME, "stbQueryThread"); #endif uint64_t st = 0; @@ -320,7 +323,8 @@ static void *superTableQuery(void *sarg) { return NULL; } -static int multi_thread_super_table_query(uint16_t iface, char* dbName) { +// super table query +static int stbQuery(uint16_t iface, char* dbName) { int ret = -1; pthread_t * pidsOfSub = NULL; threadInfo *infosOfSub = NULL; @@ -368,7 +372,7 @@ static int multi_thread_super_table_query(uint16_t iface, char* dbName) { goto OVER; } } - pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); + pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -424,8 +428,8 @@ void freeSpecialQueryInfo() { g_queryInfo.specifiedQueryInfo.sqls = NULL; } - -static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { +// spec query thread +static int specQuery(uint16_t iface, char* dbName) { pthread_t * pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from specify table @@ -479,7 +483,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } } - pthread_create(pids + j, NULL, specifiedTableQuery, pThreadInfo); + pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); threadCnt++; } @@ -585,7 +589,8 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { return 0; } -static int multi_thread_specified_mixed_query(uint16_t iface, char* dbName) { +// spec query mix +static int specQueryMix(uint16_t iface, char* dbName) { int code = -1; int thread = g_queryInfo.specifiedQueryInfo.concurrent; pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); @@ -621,7 +626,7 @@ static int multi_thread_specified_mixed_query(uint16_t iface, char* dbName) { goto OVER; } } - pthread_create(pids + i, NULL, mixedQuery, pQueryThreadInfo); + pthread_create(pids + i, NULL, specQueryMixThread, pQueryThreadInfo); } int64_t start = toolsGetTimestampUs(); @@ -903,20 +908,20 @@ int queryTestProcess() { uint64_t startTs = toolsGetTimestampMs(); if (g_queryInfo.specifiedQueryInfo.mixed_query) { // mixed - if (multi_thread_specified_mixed_query(g_queryInfo.iface, + if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } else { // no mixied - if (multi_thread_specified_table_query(g_queryInfo.iface, + if (specQuery(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } // super table - if (multi_thread_super_table_query(g_queryInfo.iface, + if (stbQuery(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } From 201865dfccb9cc13325156f24bae22121296f972 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 12:36:43 +0800 Subject: [PATCH 09/49] fix: build is ok --- src/benchQuery.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index bc7059ec..fc5dd276 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -789,7 +789,7 @@ int fetchChildTableName(char *dbName, char *stbName) { } else { snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(TBNAME) FROM %s.%s", - dbName, superQueryInfo.stbName); + dbName, stbName); } TAOS_RES *res = taos_query(conn->taos, cmd); int32_t code = taos_errno(res); @@ -898,6 +898,10 @@ int queryTestProcess() { if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { int32_t ret = fetchChildTableName(g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); + if (ret != 0) { + errorPrint("fetchChildTableName dbName=%s stb=%s failed.", g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); + return -1; + } } // From 3b353d7ed30e8c5038e71e937cca52435b6fa75c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 16:11:18 +0800 Subject: [PATCH 10/49] fix: origanize query code over --- inc/bench.h | 6 +- src/benchQuery.c | 379 +++++++++++++++++++++++++---------------------- src/benchUtil.c | 13 +- 3 files changed, 213 insertions(+), 185 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 2fb08859..0a8cedf2 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -863,7 +863,11 @@ typedef struct SQueryThreadInfo_S { int sockfd; SBenchConn* conn; int64_t total_delay; -} queryThreadInfo; + + // error rate + uint64_t nSucc; + uint64_t nFail; +} qThreadInfo; typedef struct STSmaThreadInfo_S { char* dbName; diff --git a/src/benchQuery.c b/src/benchQuery.c index fc5dd276..de379455 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -15,7 +15,8 @@ extern int g_majorVersionOfClient; -int selectAndGetResult(threadInfo *pThreadInfo, char *command) { +// query and get result other true is no test sql +int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { int ret = 0; if (g_arguments->terminate) { @@ -30,40 +31,78 @@ int selectAndGetResult(threadInfo *pThreadInfo, char *command) { 0, g_arguments->port, false, pThreadInfo->sockfd, pThreadInfo->filePath); if (0 != retCode) { - errorPrint("====restful return fail, threadID[%u]\n", - threadID); + errorPrint("====restful return fail, threadID[%u]\n", threadID); ret = -1; } } else { + // query TAOS *taos = pThreadInfo->conn->taos; int64_t rows = 0; TAOS_RES *res = taos_query(taos, command); int code = taos_errno(res); if (res == NULL || code) { - if (YES_IF_FAILED == g_arguments->continueIfFail) { - warnPrint("failed to execute sql:%s, " - "code: 0x%08x, reason:%s\n", - command, code, taos_errstr(res)); - } else { - errorPrint("failed to execute sql:%s, " - "code: 0x%08x, reason:%s\n", - command, code, taos_errstr(res)); - ret = -1; - } + // failed query + errorPrint("failed to execute sql:%s, " + "code: 0x%08x, reason:%s\n", + command, code, taos_errstr(res)); + ret = -1; } else { - //if (strlen(pThreadInfo->filePath) > 0) { + // succ query + if (!other) rows = fetchResult(res, pThreadInfo); - //} } - taos_free_result(res); + + // free result + if (res) { + taos_free_result(res); + } debugPrint("query sql:%s rows:%"PRId64"\n", command, rows); } + + // record succ or fail count + if (ret !=0) { + if (!other) + pThreadInfo->nSucc ++; + if (YES_IF_FAILED == g_arguments->continueIfFail) { + // force continue , so ignore this error + ret = 0; + } + } else if(!other) { + pThreadInfo->nFail ++; + } + return ret; } +// interlligent sleep +void autoSleep(uint64_t st, uint64_t et) { + if (g_queryInfo.specifiedQueryInfo.queryInterval && + (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { + toolsMsleep((int32_t)( + g_queryInfo.specifiedQueryInfo.queryInterval*1000 + - (et - st))); // ms + } +} + +// reset +int32_t resetQueryCache(qThreadInfo* pThreadInfo) { + // execute sql + if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE"), true) { + errorPrint("%s() LN%d, reset query cache failed\n", + __func__, __LINE__); + break; + } +} + + + +// +// --------------------------------- second levle funtion for Thread ----------------------------------- +// + // spec query mixed thread static void *specQueryMixThread(void *sarg) { - queryThreadInfo *pThreadInfo = (queryThreadInfo*)sarg; + qThreadInfo *pThreadInfo = (qThreadInfo*)sarg; #ifdef LINUX prctl(PR_SET_NAME, "specQueryMixThread"); #endif @@ -84,58 +123,30 @@ static void *specQueryMixThread(void *sarg) { for (int i = pThreadInfo->start_sql; i <= pThreadInfo->end_sql; ++i) { SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); for (int j = 0; j < queryTimes; ++j) { + // check user cancel test if (g_arguments->terminate) { return NULL; } + + // reset cache if (g_queryInfo.reset_query_cache) { - if (queryDbExecCall(pThreadInfo->conn, - "RESET QUERY CACHE")) { - errorPrint("%s() LN%d, reset query cache failed\n", - __func__, __LINE__); + if (resetQueryCache(pThreadInfo)) { + errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); return NULL; } } + + // execute sql st = toolsGetTimestampUs(); - if (g_queryInfo.iface == REST_IFACE) { - int retCode = postProceSql(sql->command, g_queryInfo.dbName, - 0, g_queryInfo.iface, 0, - g_arguments->port, - false, pThreadInfo->sockfd, ""); - if (retCode) { - errorPrint("thread[%d]: restful query <%s> failed\n", - pThreadInfo->threadId, sql->command); - continue; - } - } else { - TAOS_RES *res = taos_query(pThreadInfo->conn->taos, - sql->command); - if (res == NULL || taos_errno(res) != 0) { - if (YES_IF_FAILED == g_arguments->continueIfFail) { - warnPrint( - "thread[%d]: failed to execute sql :%s, " - "code: 0x%x, reason: %s\n", - pThreadInfo->threadId, - sql->command, - taos_errno(res), taos_errstr(res)); - } else { - errorPrint( - "thread[%d]: failed to execute sql :%s, " - "code: 0x%x, reason: %s\n", - pThreadInfo->threadId, - sql->command, - taos_errno(res), taos_errstr(res)); - if (TSDB_CODE_RPC_NETWORK_UNAVAIL == - taos_errno(res)) { - taos_free_result(res); - return NULL; - } - } - taos_free_result(res); - continue; - } - taos_free_result(res); + int ret = selectAndGetResult(pThreadInfo, sql->command, false); + if (ret) { + g_fail = true; + errorPrint("failed call mix selectAndGetResult, i=%d j=%d", i, j); + return NULL; } et = toolsGetTimestampUs(); + + // delay int64_t* delay = benchCalloc(1, sizeof(int64_t), false); *delay = et - st; debugPrint("%s() LN%d, delay: %"PRId64"\n", @@ -159,7 +170,7 @@ static void *specQueryMixThread(void *sarg) { // spec query thread static void *specQueryThread(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; + qThreadInfo *pThreadInfo = (qThreadInfo *)sarg; #ifdef LINUX prctl(PR_SET_NAME, "specQueryThread"); #endif @@ -200,25 +211,23 @@ static void *specQueryThread(void *sarg) { return NULL; } - if (g_queryInfo.specifiedQueryInfo.queryInterval && - (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { - toolsMsleep((int32_t)( - g_queryInfo.specifiedQueryInfo.queryInterval*1000 - - (et - st))); // ms - } + // sleep + autoSleep(st, et); + + // reset cache if (g_queryInfo.reset_query_cache) { - // execute sql - if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE")) { - errorPrint("%s() LN%d, reset query cache failed\n", - __func__, __LINE__); + if (resetQueryCache(pThreadInfo)) { + errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); return NULL; } } st = toolsGetTimestampUs(); - int ret = selectAndGetResult(pThreadInfo, sql->command); + int ret = selectAndGetResult(pThreadInfo, sql->command, false); if (ret) { g_fail = true; + errorPrint("failed call spec selectAndGetResult, index=%d\n", index); + break; } et = toolsGetTimestampUs(); @@ -256,8 +265,9 @@ static void *specQueryThread(void *sarg) { return NULL; } } - qsort(pThreadInfo->query_delay_list, queryTimes, - sizeof(uint64_t), compare); + + // delay + qsort(pThreadInfo->query_delay_list, queryTimes, sizeof(uint64_t), compare); pThreadInfo->avg_delay = (double)totalDelay / queryTimes; return NULL; } @@ -265,7 +275,7 @@ static void *specQueryThread(void *sarg) { // super table query thread static void *stbQueryThread(void *sarg) { char *sqlstr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - threadInfo *pThreadInfo = (threadInfo *)sarg; + qThreadInfo *pThreadInfo = (qThreadInfo *)sarg; #ifdef LINUX prctl(PR_SET_NAME, "stbQueryThread"); #endif @@ -278,32 +288,45 @@ static void *stbQueryThread(void *sarg) { uint64_t lastPrintTime = toolsGetTimestampMs(); while (queryTimes--) { - if (g_queryInfo.superQueryInfo.queryInterval - && ((et - st) < - (int64_t)g_queryInfo.superQueryInfo.queryInterval*1000)) { - toolsMsleep((int32_t) - (g_queryInfo.superQueryInfo.queryInterval*1000 - - (et - st))); + + // sleep + autoSleep(st, et); + + // reset cache + if (g_queryInfo.reset_query_cache) { + if (resetQueryCache(pThreadInfo)) { + errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); + return NULL; + } } + // execute st = toolsGetTimestampMs(); - for (int i = (int)pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { + // for each table + for (int i = (int)pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + // for each sql for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr, 0, TSDB_MAX_ALLOWED_SQL_LEN); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, - i); + // get real child name sql + if(replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i)) { + // fault + tmfree(sqlstr); + return NULL; + } + if (g_queryInfo.superQueryInfo.result[j][0] != '\0') { snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - if (selectAndGetResult(pThreadInfo, sqlstr)) { + if (selectAndGetResult(pThreadInfo, sqlstr, false)) { + // found error + errorPrint("failed call stb selectAndGetResult, i=%d j=%d\n", i, j); g_fail = true; + tmfree(sqlstr); + return NULL } - pThreadInfo->totalQueried++; - int64_t currentPrintTime = toolsGetTimestampMs(); int64_t endTs = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30 * 1000) { @@ -323,11 +346,49 @@ static void *stbQueryThread(void *sarg) { return NULL; } +// +// --------------------------------- firse level function ------------------------------ +// + +// init conn +int32_t initQueryConn(qThreadInfo * pThreadInfo, int iface) { + // create conn + if (iface == REST_IFACE) { + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + pThreadInfo->sockfd = sockfd; + } else { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + return -1; + } + } + + return 0; +} + +// close conn +void closeQueryConn(qThreadInfo * pThreadInfo, int iface) { + if (iface == REST_IFACE) { +#ifdef WINDOWS + closesocket(pThreadInfo->sockfd); + WSACleanup(); +#else + close(pThreadInfo->sockfd); +#endif + } else { + closeBenchConn(pThreadInfo->conn); + pThreadInfo->conn = NULL; + } +} + // super table query static int stbQuery(uint16_t iface, char* dbName) { int ret = -1; - pthread_t * pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; + pthread_t * pidsOfSub = NULL; + qThreadInfo *infosOfSub = NULL; //==== create sub threads for query from all sub table of the super table if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { @@ -335,7 +396,7 @@ static int stbQuery(uint16_t iface, char* dbName) { *sizeof(pthread_t), false); infosOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt - *sizeof(threadInfo), false); + *sizeof(qThreadInfo), false); int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; int threads = g_queryInfo.superQueryInfo.threadCnt; @@ -353,24 +414,16 @@ static int stbQuery(uint16_t iface, char* dbName) { uint64_t tableFrom = 0; for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infosOfSub + i; + qThreadInfo *pThreadInfo = infosOfSub + i; pThreadInfo->threadID = i; pThreadInfo->start_table_from = tableFrom; pThreadInfo->ntables = i < b ? a + 1 : a; pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; tableFrom = pThreadInfo->end_table_to + 1; - if (iface == REST_IFACE) { - int sockfd = createSockFd(); - if (sockfd < 0) { - goto OVER; - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - goto OVER; - } + // create conn + if (initQueryConn(pThreadInfo, iface)){ + goto OVER; } pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); } @@ -379,19 +432,15 @@ static int stbQuery(uint16_t iface, char* dbName) { for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { if (!g_arguments->terminate) pthread_join(pidsOfSub[i], NULL); - threadInfo *pThreadInfo = infosOfSub + i; - if (iface == REST_IFACE) { - destroySockFd(pThreadInfo->sockfd); - } else { - closeBenchConn(pThreadInfo->conn); - } + qThreadInfo *pThreadInfo = infosOfSub + i; + // close conn + closeQueryConn(pThreadInfo, iface); if (g_fail) { goto OVER; } } for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; ++i) { - g_queryInfo.superQueryInfo.totalQueried - += infosOfSub[i].totalQueried; + g_queryInfo.superQueryInfo.totalQueried += infosOfSub[i].totalQueried; } } else { return 0; @@ -431,7 +480,7 @@ void freeSpecialQueryInfo() { // spec query thread static int specQuery(uint16_t iface, char* dbName) { pthread_t * pids = NULL; - threadInfo *infos = NULL; + qThreadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; @@ -447,13 +496,13 @@ static int specQuery(uint16_t iface, char* dbName) { // malloc funciton global memory pids = benchCalloc(1, nConcurrent * sizeof(pthread_t), false); - infos = benchCalloc(1, nConcurrent * sizeof(threadInfo), false); + infos = benchCalloc(1, nConcurrent * sizeof(qThreadInfo), false); bool exeError = false; for (uint64_t i = 0; i < nSqlCount; i++) { // reset memset(pids, 0, nConcurrent * sizeof(pthread_t)); - memset(infos, 0, nConcurrent * sizeof(threadInfo)); + memset(infos, 0, nConcurrent * sizeof(qThreadInfo)); // get execute sql SSQL *sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); @@ -461,26 +510,14 @@ static int specQuery(uint16_t iface, char* dbName) { // create threads int threadCnt = 0; for (int j = 0; j < nConcurrent; j++) { - threadInfo *pThreadInfo = infos + j; + qThreadInfo *pThreadInfo = infos + j; pThreadInfo->threadID = i * nConcurrent + j; pThreadInfo->querySeq = i; - if (iface == REST_IFACE) { - int sockfd = createSockFd(); - // int iMode = 1; - // ioctl(sockfd, FIONBIO, &iMode); - if (sockfd < 0) { - exeError = true; - - break; - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - destroySockFd(pThreadInfo->sockfd); - exeError = true; - break; - } + + // create conn + if (initQueryConn(pThreadInfo, iface)) { + exeError = true; + break; } pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); @@ -497,18 +534,8 @@ static int specQuery(uint16_t iface, char* dbName) { // wait threads execute finished one by one for (int j = 0; j < threadCnt ; j++) { pthread_join(pids[j], NULL); - threadInfo *pThreadInfo = infos + j; - if (iface == REST_IFACE) { -#ifdef WINDOWS - closesocket(pThreadInfo->sockfd); - WSACleanup(); -#else - close(pThreadInfo->sockfd); -#endif - } else { - closeBenchConn(pThreadInfo->conn); - pThreadInfo->conn = NULL; - } + qThreadInfo *pThreadInfo = infos + j; + closeQueryConn(pThreadInfo, iface); // need exit in loop if (g_fail || g_arguments->terminate) { @@ -538,7 +565,7 @@ static int specQuery(uint16_t iface, char* dbName) { uint64_t totalQueryTimes = query_times * nConcurrent; double avg_delay = 0.0; for (int j = 0; j < nConcurrent; j++) { - threadInfo *pThreadInfo = infos + j; + qThreadInfo *pThreadInfo = infos + j; avg_delay += pThreadInfo->avg_delay; for (uint64_t k = 0; k < g_queryInfo.specifiedQueryInfo.queryTimes; k++) { sql->delay_list[j * query_times + k] = pThreadInfo->query_delay_list[k]; @@ -591,13 +618,16 @@ static int specQuery(uint16_t iface, char* dbName) { // spec query mix static int specQueryMix(uint16_t iface, char* dbName) { - int code = -1; - int thread = g_queryInfo.specifiedQueryInfo.concurrent; - pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); - queryThreadInfo *infos = benchCalloc(thread, sizeof(queryThreadInfo), true); + // init + int code = -1; + int thread = g_queryInfo.specifiedQueryInfo.concurrent; + pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); + qThreadInfo *infos = benchCalloc(thread, sizeof(qThreadInfo), true); + + // concurent calc int total_sql_num = g_queryInfo.specifiedQueryInfo.sqls->size; - int start_sql = 0; - int a = total_sql_num / thread; + int start_sql = 0; + int a = total_sql_num / thread; if (a < 1) { thread = total_sql_num; a = 1; @@ -606,27 +636,24 @@ static int specQueryMix(uint16_t iface, char* dbName) { if (thread != 0) { b = total_sql_num % thread; } + + // + // running + // for (int i = 0; i < thread; ++i) { - queryThreadInfo *pQueryThreadInfo = infos + i; - pQueryThreadInfo->threadId = i; - pQueryThreadInfo->start_sql = start_sql; - pQueryThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; - start_sql = pQueryThreadInfo->end_sql + 1; - pQueryThreadInfo->total_delay = 0; - pQueryThreadInfo->query_delay_list = benchArrayInit(1, sizeof(int64_t)); - if (iface == REST_IFACE) { - int sockfd = createSockFd(); - if (sockfd < 0) { - goto OVER; - } - pQueryThreadInfo->sockfd = sockfd; - } else { - pQueryThreadInfo->conn = initBenchConn(); - if (pQueryThreadInfo->conn == NULL) { - goto OVER; - } + qThreadInfo *pThreadInfo = infos + i; + pThreadInfo->threadId = i; + pThreadInfo->start_sql = start_sql; + pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; + start_sql = pThreadInfo->end_sql + 1; + pThreadInfo->total_delay = 0; + pThreadInfo->query_delay_list = benchArrayInit(1, sizeof(int64_t)); + + // create conn + if (initQueryConn(pThreadInfo, iface)){ + goto OVER; } - pthread_create(pids + i, NULL, specQueryMixThread, pQueryThreadInfo); + pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); } int64_t start = toolsGetTimestampUs(); @@ -639,23 +666,15 @@ static int specQueryMix(uint16_t iface, char* dbName) { BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); int64_t total_delay = 0; for (int i = 0; i < thread; ++i) { - queryThreadInfo * pThreadInfo = infos + i; + qThreadInfo * pThreadInfo = infos + i; benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, pThreadInfo->query_delay_list->size, true); total_delay += pThreadInfo->total_delay; tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; - if (iface == REST_IFACE) { -#ifdef WINDOWS - closesocket(pThreadInfo->sockfd); - WSACleanup(); -#else - close(pThreadInfo->sockfd); -#endif - } else { - closeBenchConn(pThreadInfo->conn); - } + // close conn + closeQueryConn(pThreadInfo, iface); } qsort(delay_list->pData, delay_list->size, delay_list->elemSize, compare); if (delay_list->size) { diff --git a/src/benchUtil.c b/src/benchUtil.c index 8baa108d..d47c284b 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -240,10 +240,14 @@ static void appendResultBufToFile(char *resultBuf, char * filePath) { tmfclose(fp); } -void replaceChildTblName(char *inSql, char *outSql, int tblIndex) { - char sourceString[32] = "xxxx"; - char *pos = strstr(inSql, sourceString); - if (0 == pos) return; +int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { + // child table mark + char mark[32] = "xxxx"; + char *pos = strstr(inSql, mark); + if (0 == pos) { + errorPrint("sql format error, super table query not found child table replace mark string '%s'", mark); + return -1; + } char subTblName[TSDB_TABLE_NAME_LEN]; snprintf(subTblName, TSDB_TABLE_NAME_LEN, @@ -253,6 +257,7 @@ void replaceChildTblName(char *inSql, char *outSql, int tblIndex) { tstrncpy(outSql, inSql, pos - inSql + 1); snprintf(outSql + strlen(outSql), TSDB_MAX_ALLOWED_SQL_LEN -1, "%s%s", subTblName, pos + strlen(sourceString)); + return 0; } int64_t toolsGetTimestamp(int32_t precision) { From 1de5a31cab8f9ad107f74d81cd422b4ec283ba47 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 17:57:22 +0800 Subject: [PATCH 11/49] enh: calc totalQueries is finished --- src/benchQuery.c | 216 ++++++++++++++++++++++++++++------------------- 1 file changed, 129 insertions(+), 87 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index de379455..a1936a72 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -147,15 +147,18 @@ static void *specQueryMixThread(void *sarg) { et = toolsGetTimestampUs(); // delay - int64_t* delay = benchCalloc(1, sizeof(int64_t), false); - *delay = et - st; - debugPrint("%s() LN%d, delay: %"PRId64"\n", - __func__, __LINE__, *delay); - - pThreadInfo->total_delay += (et - st); - if(benchArrayPush(pThreadInfo->query_delay_list, delay) == NULL){ - tmfree(delay); + if (ret == 0) { + int64_t* delay = benchCalloc(1, sizeof(int64_t), false); + *delay = et - st; + debugPrint("%s() LN%d, delay: %"PRId64"\n", __func__, __LINE__, *delay); + + pThreadInfo->total_delay += (et - st); + if(benchArrayPush(pThreadInfo->query_delay_list, delay) == NULL){ + tmfree(delay); + } } + + // show int64_t currentPrintTs = toolsGetTimestampMs(); if (currentPrintTs - lastPrintTs > 10 * 1000) { infoPrint("thread[%d] has currently complete query %d times\n", @@ -235,8 +238,8 @@ static void *specQueryThread(void *sarg) { debugPrint("%s() LN%d, delay: %"PRId64"\n", __func__, __LINE__, delay); if (ret == 0) { + // only succ add delay list pThreadInfo->query_delay_list[index] = delay; - pThreadInfo->totalQueried++; } index++; totalDelay += delay; @@ -327,15 +330,17 @@ static void *stbQueryThread(void *sarg) { return NULL } - int64_t currentPrintTime = toolsGetTimestampMs(); - int64_t endTs = toolsGetTimestampMs(); + int64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30 * 1000) { + int64_t endTs = toolsGetTimestampMs(); + uint64_t totalQueried = pThreadInfo->nSucc; + if(g_arguments->continueIfFail == YES_IF_FAILED) { + totalQueried += pThreadInfo->nFail; + } infoPrint( - "thread[%d] has currently completed queries: %" PRIu64 - ", QPS: %10.3f\n", - pThreadInfo->threadID, pThreadInfo->totalQueried, - (double)(pThreadInfo->totalQueried / - ((endTs - startTs) / 1000.0))); + "thread[%d] has currently completed queries: %" PRIu64 ", QPS: %10.3f\n", + pThreadInfo->threadID, totalQueried, + (double)(totalQueried / ((endTs - startTs) / 1000.0))); lastPrintTime = currentPrintTime; } } @@ -389,61 +394,68 @@ static int stbQuery(uint16_t iface, char* dbName) { int ret = -1; pthread_t * pidsOfSub = NULL; qThreadInfo *infosOfSub = NULL; - //==== create sub threads for query from all sub table of the super table - if ((g_queryInfo.superQueryInfo.sqlCount > 0) - && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt - *sizeof(pthread_t), - false); - infosOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt - *sizeof(qThreadInfo), false); - - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - uint64_t tableFrom = 0; - for (int i = 0; i < threads; i++) { - qThreadInfo *pThreadInfo = infosOfSub + i; - pThreadInfo->threadID = i; - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = - i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - // create conn - if (initQueryConn(pThreadInfo, iface)){ - goto OVER; - } - pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); - } - g_queryInfo.superQueryInfo.threadCnt = threads; - - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - if (!g_arguments->terminate) - pthread_join(pidsOfSub[i], NULL); - qThreadInfo *pThreadInfo = infosOfSub + i; - // close conn - closeQueryConn(pThreadInfo, iface); - if (g_fail) { - goto OVER; - } + g_queryInfo.superQueryInfo.totalQueried = 0; + + // check + if ((g_queryInfo.superQueryInfo.sqlCount == 0) + || (g_queryInfo.superQueryInfo.threadCnt == 0)) { + return 0; + } + + // malloc + pidsOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt + *sizeof(pthread_t), + false); + infosOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt + *sizeof(qThreadInfo), false); + + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int nConcurrent = g_queryInfo.superQueryInfo.threadCnt; + + int64_t a = ntables / nConcurrent; + if (a < 1) { + nConcurrent = (int)ntables; + a = 1; + } + + int64_t b = 0; + if (nConcurrent != 0) { + b = ntables % nConcurrent; + } + + uint64_t tableFrom = 0; + int threadCnt = 0; + for (int i = 0; i < nConcurrent; i++) { + qThreadInfo *pThreadInfo = infosOfSub + i; + pThreadInfo->threadID = i; + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = + i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + // create conn + if (initQueryConn(pThreadInfo, iface)){ + g_fail = true; + break; } - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; ++i) { - g_queryInfo.superQueryInfo.totalQueried += infosOfSub[i].totalQueried; + pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); + threadCnt ++; + } + // real thread count + g_queryInfo.superQueryInfo.threadCnt = threadCnt; + + for (int i = 0; i < threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + qThreadInfo *pThreadInfo = infosOfSub + i; + // add succ + g_queryInfo.superQueryInfo.totalQueried += pThreadInfo->nSucc; + if (g_arguments->continueIfFail == YES_IF_FAILED) { + // "yes" need add fail cnt + g_queryInfo.superQueryInfo.totalQueried += pThreadInfo->nFail; } - } else { - return 0; + + // close conn + closeQueryConn(pThreadInfo, iface); } ret = 0; @@ -484,6 +496,7 @@ static int specQuery(uint16_t iface, char* dbName) { //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; + g_queryInfo.specifiedQueryInfo.totalQueried = 0; // check invaid if(nSqlCount == 0 || nConcurrent == 0 ) { @@ -571,6 +584,12 @@ static int specQuery(uint16_t iface, char* dbName) { sql->delay_list[j * query_times + k] = pThreadInfo->query_delay_list[k]; } + // total queries + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; + if (g_arguments->continueIfFail == YES_IF_FAILED) { + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + } + // free BArray tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; @@ -606,8 +625,6 @@ static int specQuery(uint16_t iface, char* dbName) { tmfree(buf); } - g_queryInfo.specifiedQueryInfo.totalQueried = - nSqlCount * g_queryInfo.specifiedQueryInfo.queryTimes * nConcurrent; tmfree((char *)pids); tmfree((char *)infos); @@ -619,9 +636,9 @@ static int specQuery(uint16_t iface, char* dbName) { // spec query mix static int specQueryMix(uint16_t iface, char* dbName) { // init - int code = -1; - int thread = g_queryInfo.specifiedQueryInfo.concurrent; - pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); + int code = -1; + int thread = g_queryInfo.specifiedQueryInfo.concurrent; + pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); qThreadInfo *infos = benchCalloc(thread, sizeof(qThreadInfo), true); // concurent calc @@ -642,9 +659,9 @@ static int specQueryMix(uint16_t iface, char* dbName) { // for (int i = 0; i < thread; ++i) { qThreadInfo *pThreadInfo = infos + i; - pThreadInfo->threadId = i; - pThreadInfo->start_sql = start_sql; - pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; + pThreadInfo->threadId = i; + pThreadInfo->start_sql = start_sql; + pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; start_sql = pThreadInfo->end_sql + 1; pThreadInfo->total_delay = 0; pThreadInfo->query_delay_list = benchArrayInit(1, sizeof(int64_t)); @@ -665,18 +682,35 @@ static int specQueryMix(uint16_t iface, char* dbName) { // statistic BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); int64_t total_delay = 0; + g_queryInfo.specifiedQueryInfo.totalQueried = 0; + for (int i = 0; i < thread; ++i) { qThreadInfo * pThreadInfo = infos + i; + + // total queries + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; + if(g_arguments->continueIfFail == YES_IF_FAILED) { + // yes need add failed count + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + } + + // append delay benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, pThreadInfo->query_delay_list->size, true); total_delay += pThreadInfo->total_delay; + + // free delay tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; // close conn closeQueryConn(pThreadInfo, iface); } + + // sort qsort(delay_list->pData, delay_list->size, delay_list->elemSize, compare); + + // show delay min max if (delay_list->size) { infoPrint( "spend %.6fs using " @@ -705,8 +739,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { } benchArrayDestroy(delay_list); code = 0; - g_queryInfo.specifiedQueryInfo.totalQueried = - g_queryInfo.specifiedQueryInfo.sqls->size * g_queryInfo.specifiedQueryInfo.queryTimes; + OVER: tmfree(pids); @@ -877,15 +910,24 @@ void totalQuery(int64_t spends) { // total QPS uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + g_queryInfo.superQueryInfo.totalQueried; - infoPrint( - "Spend %.4f second completed total queries: %" PRIu64 - ", the QPS of all threads: %10.3f\n\n", - tInS, totalQueried, (double)totalQueried / tInS); - infoPrintToFile( - "Spend %.4f second completed total queries: %" PRIu64 - ", the QPS of all threads: %10.3f\n\n", - tInS, totalQueried, (double)totalQueried / tInS); + // error rate + char errRate[128] = ""; + if(g_arguments->continueIfFail == YES_IF_FAILED) { + uint16_t totalFail = g_queryInfo.specifiedQueryInfo.nFail + g_queryInfo.superQueryInfo.nFail; + if (totalQueried > 0) { + snprintf(errRate, " Error rate:%.3f%%", ((float)totalFail * 100)/totalQueried); + } + } + + // show + char buf[512] = ""; + snprintf(buf, sizeof(buf), + "Spend %.4f second completed total queries: %" PRIu64 + ", the QPS of all threads: %10.3f%s\n\n", + tInS, totalQueried, (double)totalQueried / tInS, errRate); + infoPrint("%s", buf); + infoPrintToFile("%s", buf); } int queryTestProcess() { From 85e38d994fb3c9ac97abcca84c8a3839715ab589 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 18 Jan 2025 18:10:17 +0800 Subject: [PATCH 12/49] fix: modify filePath variant --- inc/bench.h | 5 +++-- src/benchQuery.c | 2 +- src/benchSubscribe.c | 8 ++++---- src/benchUtil.c | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 0a8cedf2..e26cd61b 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -858,7 +858,7 @@ typedef struct SThreadInfo_S { typedef struct SQueryThreadInfo_S { int start_sql; int end_sql; - int threadId; + int threadID; BArray* query_delay_list; int sockfd; SBenchConn* conn; @@ -867,6 +867,7 @@ typedef struct SQueryThreadInfo_S { // error rate uint64_t nSucc; uint64_t nFail; + char filePath[MAX_PATH_LEN]; } qThreadInfo; typedef struct STSmaThreadInfo_S { @@ -911,7 +912,7 @@ int convertStringToDatatype(char *type, int length); unsigned int taosRandom(); void tmfree(void *buf); void tmfclose(FILE *fp); -int64_t fetchResult(TAOS_RES *res, threadInfo *pThreadInfo); +int64_t fetchResult(TAOS_RES *res, char *filePath); void prompt(bool NonStopMode); void ERROR_EXIT(const char *msg); int getServerVersionRest(int16_t rest_port); diff --git a/src/benchQuery.c b/src/benchQuery.c index a1936a72..48243929 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -49,7 +49,7 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { } else { // succ query if (!other) - rows = fetchResult(res, pThreadInfo); + rows = fetchResult(res, pThreadInfo->filePath); } // free result diff --git a/src/benchSubscribe.c b/src/benchSubscribe.c index 011d5458..7f0d7e15 100644 --- a/src/benchSubscribe.c +++ b/src/benchSubscribe.c @@ -23,7 +23,7 @@ static void stable_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, return; } - if (param) fetchResult(res, (threadInfo *)param); + if (param) fetchResult(res, ((threadInfo *)param)->filePath); // tao_unsubscribe() will free result. } @@ -35,7 +35,7 @@ static void specified_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, return; } - if (param) fetchResult(res, (threadInfo *)param); + if (param) fetchResult(res, ((threadInfo *)param)->filePath); // tao_unsubscribe() will free result. } @@ -127,7 +127,7 @@ static void *specifiedSubscribe(void *sarg) { } fetchResult( g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo); + pThreadInfo->filePath); g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID]++; if ((g_queryInfo.specifiedQueryInfo @@ -247,7 +247,7 @@ static void *superSubscribe(void *sarg) { .result[pThreadInfo->querySeq], pThreadInfo->threadID); } - fetchResult(res, pThreadInfo); + fetchResult(res, pThreadInfo->filePath); consumed[tsubSeq]++; if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) && diff --git a/src/benchUtil.c b/src/benchUtil.c index d47c284b..46f888a9 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -825,14 +825,14 @@ int postProceSql(char *sqlstr, char* dbName, int precision, int iface, } // fetch result fo file or nothing -int64_t fetchResult(TAOS_RES *res, threadInfo *pThreadInfo) { +int64_t fetchResult(TAOS_RES *res, char * filePath) { TAOS_ROW row = NULL; int num_fields = 0; int64_t totalLen = 0; TAOS_FIELD *fields = 0; int64_t rows = 0; char *databuf = NULL; - bool toFile = strlen(pThreadInfo->filePath) > 0; + bool toFile = strlen(filePath) > 0; if(toFile) { @@ -865,7 +865,7 @@ int64_t fetchResult(TAOS_RES *res, threadInfo *pThreadInfo) { // end if (toFile) { - appendResultBufToFile(databuf, pThreadInfo->filePath); + appendResultBufToFile(databuf, filePath); free(databuf); } return rows; From 5704eb5f38bb343b46d64b60bdb4aa02e88d4086 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 08:22:47 +0800 Subject: [PATCH 13/49] fix: add totalFail to total error rate --- inc/bench.h | 28 +++++++++++++++++++--------- src/benchQuery.c | 23 +++++++++++++++-------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index e26cd61b..42f1a0f0 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -648,6 +648,8 @@ typedef struct SpecifiedQueryInfo_S { TAOS_RES *res[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; bool mixed_query; + // error rate + uint64_t totalFail; } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { @@ -669,6 +671,8 @@ typedef struct SuperQueryInfo_S { TAOS_SUB *tsub[MAX_QUERY_SQL_COUNT]; char ** childTblName; uint64_t totalQueried; + // error rate + uint64_t totalFail; } SuperQueryInfo; typedef struct SQueryMetaInfo_S { @@ -856,18 +860,24 @@ typedef struct SThreadInfo_S { } threadInfo; typedef struct SQueryThreadInfo_S { - int start_sql; - int end_sql; - int threadID; - BArray* query_delay_list; - int sockfd; SBenchConn* conn; - int64_t total_delay; + int32_t start_sql; + int32_t end_sql; + int32_t threadID; + BArray* query_delay_list; + int32_t sockfd; + int64_t total_delay; + + char filePath[MAX_PATH_LEN]; + uint64_t start_table_from; + uint64_t end_table_to; + double avg_delay; + uint64_t ntables; + uint64_t *query_delay_list; // error rate - uint64_t nSucc; - uint64_t nFail; - char filePath[MAX_PATH_LEN]; + uint64_t nSucc; + uint64_t nFail; } qThreadInfo; typedef struct STSmaThreadInfo_S { diff --git a/src/benchQuery.c b/src/benchQuery.c index 48243929..dba95740 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -87,7 +87,7 @@ void autoSleep(uint64_t st, uint64_t et) { // reset int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // execute sql - if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE"), true) { + if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", true) { errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); break; @@ -111,7 +111,7 @@ static void *specQueryMixThread(void *sarg) { if (pThreadInfo->conn && pThreadInfo->conn->taos && taos_select_db(pThreadInfo->conn->taos, g_queryInfo.dbName)) { - errorPrint("thread[%d]: failed to select database(%s)\n", pThreadInfo->threadId, g_queryInfo.dbName); + errorPrint("thread[%d]: failed to select database(%s)\n", pThreadInfo->threadID, g_queryInfo.dbName); return NULL; } } @@ -162,7 +162,7 @@ static void *specQueryMixThread(void *sarg) { int64_t currentPrintTs = toolsGetTimestampMs(); if (currentPrintTs - lastPrintTs > 10 * 1000) { infoPrint("thread[%d] has currently complete query %d times\n", - pThreadInfo->threadId, + pThreadInfo->threadID, (int)pThreadInfo->query_delay_list->size); lastPrintTs = currentPrintTs; } @@ -333,6 +333,7 @@ static void *stbQueryThread(void *sarg) { int64_t currentPrintTime = toolsGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30 * 1000) { int64_t endTs = toolsGetTimestampMs(); + // real total uint64_t totalQueried = pThreadInfo->nSucc; if(g_arguments->continueIfFail == YES_IF_FAILED) { totalQueried += pThreadInfo->nFail; @@ -395,6 +396,7 @@ static int stbQuery(uint16_t iface, char* dbName) { pthread_t * pidsOfSub = NULL; qThreadInfo *infosOfSub = NULL; g_queryInfo.superQueryInfo.totalQueried = 0; + g_queryInfo.superQueryInfo.totalFail = 0; // check if ((g_queryInfo.superQueryInfo.sqlCount == 0) @@ -452,6 +454,7 @@ static int stbQuery(uint16_t iface, char* dbName) { if (g_arguments->continueIfFail == YES_IF_FAILED) { // "yes" need add fail cnt g_queryInfo.superQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.superQueryInfo.totalFail += pThreadInfo->nFail; } // close conn @@ -497,6 +500,7 @@ static int specQuery(uint16_t iface, char* dbName) { int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; g_queryInfo.specifiedQueryInfo.totalQueried = 0; + g_queryInfo.specifiedQueryInfo.totalFail = 0; // check invaid if(nSqlCount == 0 || nConcurrent == 0 ) { @@ -587,7 +591,8 @@ static int specQuery(uint16_t iface, char* dbName) { // total queries g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; if (g_arguments->continueIfFail == YES_IF_FAILED) { - g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; } // free BArray @@ -659,7 +664,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { // for (int i = 0; i < thread; ++i) { qThreadInfo *pThreadInfo = infos + i; - pThreadInfo->threadId = i; + pThreadInfo->threadID = i; pThreadInfo->start_sql = start_sql; pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; start_sql = pThreadInfo->end_sql + 1; @@ -683,15 +688,17 @@ static int specQueryMix(uint16_t iface, char* dbName) { BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); int64_t total_delay = 0; g_queryInfo.specifiedQueryInfo.totalQueried = 0; + g_queryInfo.specifiedQueryInfo.totalFail = 0; for (int i = 0; i < thread; ++i) { qThreadInfo * pThreadInfo = infos + i; // total queries g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; - if(g_arguments->continueIfFail == YES_IF_FAILED) { + if (g_arguments->continueIfFail == YES_IF_FAILED) { // yes need add failed count g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; } // append delay @@ -914,9 +921,9 @@ void totalQuery(int64_t spends) { // error rate char errRate[128] = ""; if(g_arguments->continueIfFail == YES_IF_FAILED) { - uint16_t totalFail = g_queryInfo.specifiedQueryInfo.nFail + g_queryInfo.superQueryInfo.nFail; + uint16_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; if (totalQueried > 0) { - snprintf(errRate, " Error rate:%.3f%%", ((float)totalFail * 100)/totalQueried); + snprintf(errRate, sizeof(errRate), " Error Rate:%.3f%%", ((float)totalFail * 100)/totalQueried); } } From e2369b25fecdefaf7c2d347b075840a8a38f95ef Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 13:00:39 +0800 Subject: [PATCH 14/49] enh: query three mode uniform --- inc/bench.h | 7 +- src/benchQuery.c | 314 ++++++++++++++++++++++++++++------------------- src/benchUtil.c | 7 +- 3 files changed, 200 insertions(+), 128 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 42f1a0f0..9f6e5448 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -873,7 +873,7 @@ typedef struct SQueryThreadInfo_S { uint64_t end_table_to; double avg_delay; uint64_t ntables; - uint64_t *query_delay_list; + uint64_t querySeq; // error rate uint64_t nSucc; @@ -914,7 +914,7 @@ int getInfoFromJsonFile(); /* demoUtil.c */ int compare(const void *a, const void *b); void encodeAuthBase64(); -void replaceChildTblName(char *inSql, char *outSql, int tblIndex); +int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex); void setupForAnsiEscape(void); void resetAfterAnsiEscape(void); char * convertDatatypeToString(int type); @@ -942,7 +942,8 @@ int getAllChildNameOfSuperTable(TAOS *taos, char *dbName, char *stbName, int64_t childTblCountOfSuperTbl); void* benchCalloc(size_t nmemb, size_t size, bool record); BArray* benchArrayInit(size_t size, size_t elemSize); -void* benchArrayPush(BArray* pArray, void* pData); +void* benchArrayPush(BArray* pArray, void* pData); // free pData for auto +void* benchArrayPushNoFree(BArray* pArray, void* pData); // not free pData void* benchArrayDestroy(BArray* pArray); void benchArrayClear(BArray* pArray); void* benchArrayGet(const BArray* pArray, size_t index); diff --git a/src/benchQuery.c b/src/benchQuery.c index dba95740..ce0893b6 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -87,7 +87,7 @@ void autoSleep(uint64_t st, uint64_t et) { // reset int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // execute sql - if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", true) { + if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", true)) { errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); break; @@ -100,6 +100,25 @@ int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // --------------------------------- second levle funtion for Thread ----------------------------------- // +// show rela qps +int64_t showRealQPS(qThreadInfo* pThreadInfo, int64_t lastPrintTime, int64_t startTs) { + int64_t now = toolsGetTimestampMs(); + if (now - lastPrintTime > 30 * 1000) { + // real total + uint64_t totalQueried = pThreadInfo->nSucc; + if(g_arguments->continueIfFail == YES_IF_FAILED) { + totalQueried += pThreadInfo->nFail; + } + infoPrint( + "thread[%d] has currently completed queries: %" PRIu64 ", QPS: %10.3f\n", + pThreadInfo->threadID, totalQueried, + (double)(totalQueried / ((now - startTs) / 1000.0))); + return now; + } else { + return lastPrintTime; + } +} + // spec query mixed thread static void *specQueryMixThread(void *sarg) { qThreadInfo *pThreadInfo = (qThreadInfo*)sarg; @@ -116,16 +135,19 @@ static void *specQueryMixThread(void *sarg) { } } - int64_t lastPrintTs = toolsGetTimestampMs(); - int64_t st; - int64_t et; + int64_t st = 0; + int64_t et = 0; + int64_t startTs = toolsGetTimestampMs(); + int64_t lastPrintTime = startTs; uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(int64_t)); for (int i = pThreadInfo->start_sql; i <= pThreadInfo->end_sql; ++i) { SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); for (int j = 0; j < queryTimes; ++j) { - // check user cancel test - if (g_arguments->terminate) { - return NULL; + // use cancel + if(g_arguments->terminate) { + infoPrint("%s\n", "user cancel , so exit testing."); + break; } // reset cache @@ -146,6 +168,9 @@ static void *specQueryMixThread(void *sarg) { } et = toolsGetTimestampUs(); + // sleep + autoSleep(st, et); + // delay if (ret == 0) { int64_t* delay = benchCalloc(1, sizeof(int64_t), false); @@ -158,16 +183,13 @@ static void *specQueryMixThread(void *sarg) { } } - // show - int64_t currentPrintTs = toolsGetTimestampMs(); - if (currentPrintTs - lastPrintTs > 10 * 1000) { - infoPrint("thread[%d] has currently complete query %d times\n", - pThreadInfo->threadID, - (int)pThreadInfo->query_delay_list->size); - lastPrintTs = currentPrintTs; - } + // real show + lastPrintTime = showRealQPS(pThreadInfo, lastPrintTime, startTs); } } + + // delay sort + qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -179,9 +201,6 @@ static void *specQueryThread(void *sarg) { #endif uint64_t st = 0; uint64_t et = 0; - uint64_t minDelay = UINT64_MAX; - uint64_t maxDelay = 0; - uint64_t totalDelay = 0; int32_t index = 0; // use db @@ -195,13 +214,12 @@ static void *specQueryThread(void *sarg) { } uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; - pThreadInfo->query_delay_list = benchCalloc(queryTimes, - sizeof(uint64_t), false); - uint64_t lastPrintTime = toolsGetTimestampMs(); - uint64_t startTs = toolsGetTimestampMs(); + pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(int64_t)); + + uint64_t startTs = toolsGetTimestampMs(); + uint64_t lastPrintTime = startTs; - SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, - pThreadInfo->querySeq); + SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, pThreadInfo->querySeq); if (sql->result[0] != '\0') { snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d", @@ -209,14 +227,12 @@ static void *specQueryThread(void *sarg) { } while (index < queryTimes) { - // check cancel - if (g_arguments->terminate) { - return NULL; + // use cancel + if(g_arguments->terminate) { + infoPrint("%s\n", "user cancel , so exit testing."); + break; } - // sleep - autoSleep(st, et); - // reset cache if (g_queryInfo.reset_query_cache) { if (resetQueryCache(pThreadInfo)) { @@ -225,6 +241,7 @@ static void *specQueryThread(void *sarg) { } } + // execute sql st = toolsGetTimestampUs(); int ret = selectAndGetResult(pThreadInfo, sql->command, false); if (ret) { @@ -232,46 +249,26 @@ static void *specQueryThread(void *sarg) { errorPrint("failed call spec selectAndGetResult, index=%d\n", index); break; } - et = toolsGetTimestampUs(); - int64_t delay = et - st; - debugPrint("%s() LN%d, delay: %"PRId64"\n", __func__, __LINE__, delay); + + // sleep + autoSleep(st, et); + + uint64_t delay = et - st; + debugPrint("%s() LN%d, delay: %"PRIu64"\n", __func__, __LINE__, delay); if (ret == 0) { // only succ add delay list - pThreadInfo->query_delay_list[index] = delay; + benchArrayPushNoFree(pThreadInfo->query_delay_list, &delay); } index++; - totalDelay += delay; - if (delay > maxDelay) { - maxDelay = delay; - } - if (delay < minDelay) { - minDelay = delay; - } - - uint64_t currentPrintTime = toolsGetTimestampMs(); - uint64_t endTs = toolsGetTimestampMs(); - - if ((ret == 0) && (currentPrintTime - lastPrintTime > 30 * 1000)) { - infoPrint( - "thread[%d] has currently completed queries: %" PRIu64 - ", QPS: %10.6f\n", - pThreadInfo->threadID, pThreadInfo->totalQueried, - (double)(pThreadInfo->totalQueried / - ((endTs - startTs) / 1000.0))); - lastPrintTime = currentPrintTime; - } - if (-2 == ret) { - toolsMsleep(1000); - return NULL; - } + // real show + lastPrintTime = showRealQPS(pThreadInfo, lastPrintTime, startTs); } - // delay - qsort(pThreadInfo->query_delay_list, queryTimes, sizeof(uint64_t), compare); - pThreadInfo->avg_delay = (double)totalDelay / queryTimes; + // delay sort + qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -284,16 +281,19 @@ static void *stbQueryThread(void *sarg) { #endif uint64_t st = 0; - uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval*1000; + uint64_t et = 0; uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes; + pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(uint64_t)); + uint64_t startTs = toolsGetTimestampMs(); - - uint64_t lastPrintTime = toolsGetTimestampMs(); + uint64_t lastPrintTime = startTs; while (queryTimes--) { - - // sleep - autoSleep(st, et); + // use cancel + if(g_arguments->terminate) { + infoPrint("%s\n", "user cancel , so exit testing."); + break; + } // reset cache if (g_queryInfo.reset_query_cache) { @@ -307,11 +307,24 @@ static void *stbQueryThread(void *sarg) { st = toolsGetTimestampMs(); // for each table for (int i = (int)pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + // use cancel + if(g_arguments->terminate) { + infoPrint("%s\n", "user cancel , so exit testing."); + break; + } + // for each sql for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr, 0, TSDB_MAX_ALLOWED_SQL_LEN); + // use cancel + if(g_arguments->terminate) { + infoPrint("%s\n", "user cancel , so exit testing."); + break; + } + + // get real child name sql - if(replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i)) { + if (replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i)) { // fault tmfree(sqlstr); return NULL; @@ -322,33 +335,37 @@ static void *stbQueryThread(void *sarg) { g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - if (selectAndGetResult(pThreadInfo, sqlstr, false)) { + + // execute sql + uint64_t s = toolsGetTimestampUs(); + int ret = selectAndGetResult(pThreadInfo, sqlstr, false); + if (ret) { // found error errorPrint("failed call stb selectAndGetResult, i=%d j=%d\n", i, j); g_fail = true; tmfree(sqlstr); return NULL } - - int64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30 * 1000) { - int64_t endTs = toolsGetTimestampMs(); - // real total - uint64_t totalQueried = pThreadInfo->nSucc; - if(g_arguments->continueIfFail == YES_IF_FAILED) { - totalQueried += pThreadInfo->nFail; - } - infoPrint( - "thread[%d] has currently completed queries: %" PRIu64 ", QPS: %10.3f\n", - pThreadInfo->threadID, totalQueried, - (double)(totalQueried / ((endTs - startTs) / 1000.0))); - lastPrintTime = currentPrintTime; + uint64_t delay = toolsGetTimestampUs() - s; + debugPrint("%s() LN%d, delay: %"PRIu64"\n", __func__, __LINE__, delay); + if (ret == 0) { + // only succ add delay list + benchArrayPushNoFree(pThreadInfo->query_delay_list, &delay); } + + // show real QPS + lastPrintTime = showRealQPS(pThreadInfo, lastPrintTime, startTs); } } et = toolsGetTimestampMs(); + + // sleep + autoSleep(st, et); } tmfree(sqlstr); + + // delay sort + qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -437,12 +454,23 @@ static int stbQuery(uint16_t iface, char* dbName) { tableFrom = pThreadInfo->end_table_to + 1; // create conn if (initQueryConn(pThreadInfo, iface)){ - g_fail = true; break; } - pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); + int ret = pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); + if (ret != 0) { + errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + break; + } threadCnt ++; } + + bool needExit = false; + // if failed, set termainte flag true like ctrl+c exit + if (threadCnt != nConcurrent ) { + needExit = true; + g_arguments->terminate = true; + } + // real thread count g_queryInfo.superQueryInfo.threadCnt = threadCnt; @@ -494,10 +522,9 @@ void freeSpecialQueryInfo() { // spec query thread static int specQuery(uint16_t iface, char* dbName) { - pthread_t * pids = NULL; + pthread_t *pids = NULL; qThreadInfo *infos = NULL; - //==== create sub threads for query from specify table - int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; + int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; g_queryInfo.specifiedQueryInfo.totalQueried = 0; g_queryInfo.specifiedQueryInfo.totalFail = 0; @@ -507,7 +534,7 @@ static int specQuery(uint16_t iface, char* dbName) { if(nSqlCount == 0) warnPrint("specified table query sql count is %" PRIu64 ".\n", nSqlCount); if(nConcurrent == 0) - warnPrint("concurrent is %d , specified_table_query->concurrent is zero. \n", nConcurrent); + warnPrint("nConcurrent is %d , specified_table_query->nConcurrent is zero. \n", nConcurrent); return 0; } @@ -517,6 +544,10 @@ static int specQuery(uint16_t iface, char* dbName) { bool exeError = false; for (uint64_t i = 0; i < nSqlCount; i++) { + if( g_arguments->terminate ) { + break; + } + // reset memset(pids, 0, nConcurrent * sizeof(pthread_t)); memset(infos, 0, nConcurrent * sizeof(qThreadInfo)); @@ -533,17 +564,21 @@ static int specQuery(uint16_t iface, char* dbName) { // create conn if (initQueryConn(pThreadInfo, iface)) { - exeError = true; break; } - pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); + int ret = pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); + if (ret != 0) { + errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + break; + } threadCnt++; } + bool needExit = false; // if failed, set termainte flag true like ctrl+c exit - if (exeError) { - errorPrint(" i=%" PRIu64 " create thread occur error, so wait exit ...\n", i); + if (threadCnt != nConcurrent ) { + needExit = true; g_arguments->terminate = true; } @@ -555,9 +590,9 @@ static int specQuery(uint16_t iface, char* dbName) { closeQueryConn(pThreadInfo, iface); // need exit in loop - if (g_fail || g_arguments->terminate) { + if (needExit) { // free BArray - tmfree(pThreadInfo->query_delay_list); + benchArrayDestroy(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; } } @@ -567,8 +602,9 @@ static int specQuery(uint16_t iface, char* dbName) { spend = 1; } - // cancel or need exit check - if (g_fail || g_arguments->terminate) { + // create + if (needExit) { + errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); // free current funciton malloc memory tmfree((char *)pids); tmfree((char *)infos); @@ -579,9 +615,9 @@ static int specQuery(uint16_t iface, char* dbName) { // execute successfully uint64_t query_times = g_queryInfo.specifiedQueryInfo.queryTimes; - uint64_t totalQueryTimes = query_times * nConcurrent; + uint64_t totalQueryTimes = query_times * threadCnt; double avg_delay = 0.0; - for (int j = 0; j < nConcurrent; j++) { + for (int j = 0; j < threadCnt; j++) { qThreadInfo *pThreadInfo = infos + j; avg_delay += pThreadInfo->avg_delay; for (uint64_t k = 0; k < g_queryInfo.specifiedQueryInfo.queryTimes; k++) { @@ -599,7 +635,7 @@ static int specQuery(uint16_t iface, char* dbName) { tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; } - avg_delay /= nConcurrent; + avg_delay /= threadCnt; qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); int32_t bufLen = strlen(sql->command) + 512; char * buf = benchCalloc(bufLen, sizeof(char), false); @@ -614,7 +650,7 @@ static int specQuery(uint16_t iface, char* dbName) { "p99: %.6fs " "SQL command: %s" "\n", - nConcurrent, query_times, + threadCnt, query_times, i + 1, spend/1E6, totalQueryTimes / (spend/1E6), avg_delay / 1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ @@ -642,55 +678,86 @@ static int specQuery(uint16_t iface, char* dbName) { static int specQueryMix(uint16_t iface, char* dbName) { // init int code = -1; - int thread = g_queryInfo.specifiedQueryInfo.concurrent; - pthread_t * pids = benchCalloc(thread, sizeof(pthread_t), true); - qThreadInfo *infos = benchCalloc(thread, sizeof(qThreadInfo), true); + int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; + pthread_t * pids = benchCalloc(nConcurrent, sizeof(pthread_t), true); + qThreadInfo *infos = benchCalloc(nConcurrent, sizeof(qThreadInfo), true); // concurent calc int total_sql_num = g_queryInfo.specifiedQueryInfo.sqls->size; int start_sql = 0; - int a = total_sql_num / thread; + int a = total_sql_num / nConcurrent; if (a < 1) { - thread = total_sql_num; + nConcurrent = total_sql_num; a = 1; } int b = 0; - if (thread != 0) { - b = total_sql_num % thread; + if (nConcurrent != 0) { + b = total_sql_num % nConcurrent; } // // running // - for (int i = 0; i < thread; ++i) { + int threadCnt = 0; + for (int i = 0; i < nConcurrent; ++i) { qThreadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; pThreadInfo->start_sql = start_sql; pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; start_sql = pThreadInfo->end_sql + 1; pThreadInfo->total_delay = 0; - pThreadInfo->query_delay_list = benchArrayInit(1, sizeof(int64_t)); // create conn if (initQueryConn(pThreadInfo, iface)){ - goto OVER; + break; + } + // main run + int ret = pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); + if (ret != 0) { + errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + break; } - pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); + + threadCnt ++; + } + + bool needExit = false; + // if failed, set termainte flag true like ctrl+c exit + if (threadCnt != nConcurrent) { + needExit = true; + g_arguments->terminate = true; } int64_t start = toolsGetTimestampUs(); - for (int i = 0; i < thread; ++i) { + for (int i = 0; i < threadCnt; ++i) { pthread_join(pids[i], NULL); + closeQueryConn(); + if (needExit) { + benchArrayDestroy(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + } } int64_t end = toolsGetTimestampUs(); + // create + if (needExit) { + errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); + // free current funciton malloc memory + tmfree((char *)pids); + tmfree((char *)infos); + // free sqls + freeSpecialQueryInfo(); + return -1; + } + // statistic BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); int64_t total_delay = 0; g_queryInfo.specifiedQueryInfo.totalQueried = 0; g_queryInfo.specifiedQueryInfo.totalFail = 0; - for (int i = 0; i < thread; ++i) { + // clear + for (int i = 0; i < nConcurrent; ++i) { qThreadInfo * pThreadInfo = infos + i; // total queries @@ -707,7 +774,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { total_delay += pThreadInfo->total_delay; // free delay - tmfree(pThreadInfo->query_delay_list); + benchArrayDestroy(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; // close conn @@ -729,7 +796,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { "p99: %.6fs, " "max: %.6fs\n", (end - start)/1E6, - thread, (int)delay_list->size, + threadCnt, (int)delay_list->size, *(int64_t *)(benchArrayGet(delay_list, 0))/1E6, (double)total_delay/delay_list->size/1E6, *(int64_t *)(benchArrayGet(delay_list, @@ -747,10 +814,12 @@ static int specQueryMix(uint16_t iface, char* dbName) { benchArrayDestroy(delay_list); code = 0; - -OVER: tmfree(pids); tmfree(infos); + + // free sqls + freeSpecialQueryInfo(); + return code; } @@ -980,21 +1049,18 @@ int queryTestProcess() { uint64_t startTs = toolsGetTimestampMs(); if (g_queryInfo.specifiedQueryInfo.mixed_query) { // mixed - if (specQueryMix(g_queryInfo.iface, - g_queryInfo.dbName)) { + if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } else { // no mixied - if (specQuery(g_queryInfo.iface, - g_queryInfo.dbName)) { + if (specQuery(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } } // super table - if (stbQuery(g_queryInfo.iface, - g_queryInfo.dbName)) { + if (stbQuery(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } diff --git a/src/benchUtil.c b/src/benchUtil.c index 46f888a9..8f05c831 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1107,7 +1107,7 @@ static int32_t benchArrayEnsureCap(BArray* pArray, size_t newCap) { } void* benchArrayAddBatch(BArray* pArray, void* pData, int32_t elems, bool free) { - if (pData == NULL) { + if (pData == NULL || elems <=0) { return NULL; } @@ -1128,6 +1128,11 @@ FORCE_INLINE void* benchArrayPush(BArray* pArray, void* pData) { return benchArrayAddBatch(pArray, pData, 1, true); } +FORCE_INLINE void* benchArrayPushNoFree(BArray* pArray, void* pData) { + return benchArrayAddBatch(pArray, pData, 1, false); +} + + void* benchArrayDestroy(BArray* pArray) { if (pArray) { tmfree(pArray->pData); From 2fe79cbed26f7fa63a3533123cbe116ef5c3925a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 21:22:09 +0800 Subject: [PATCH 15/49] fix: three query mode logic is ok need test --- inc/bench.h | 14 ++ src/benchQuery.c | 434 +++++++++++++++-------------------------------- src/benchUtil.c | 205 +++++++++++++++++++++- 3 files changed, 353 insertions(+), 300 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 9f6e5448..3fe2d2e5 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -1031,4 +1031,18 @@ bool isRest(int32_t iface); // get group index about dbname.tbname int32_t calcGroupIndex(char* dbName, char* tbName, int32_t groupCnt); + +// ------------ benchQuery util ------------- +void freeSpecialQueryInfo(); +// init conn +int32_t initQueryConn(qThreadInfo * pThreadInfo, int iface); +// close conn +void closeQueryConn(qThreadInfo * pThreadInfo, int iface); + +void *queryKiller(void *arg); +// kill show +int killSlowQuery(); +// fetch super table child name from server +int fetchChildTableName(char *dbName, char *stbName); + #endif // INC_BENCH_H_ diff --git a/src/benchQuery.c b/src/benchQuery.c index ce0893b6..276ba3c5 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -373,45 +373,73 @@ static void *stbQueryThread(void *sarg) { // --------------------------------- firse level function ------------------------------ // -// init conn -int32_t initQueryConn(qThreadInfo * pThreadInfo, int iface) { - // create conn - if (iface == REST_IFACE) { - int sockfd = createSockFd(); - if (sockfd < 0) { - return -1; - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - return -1; - } +void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { + // valid check + if(threadInfo == NULL || threadCnt == 0) { + return ; } + + // statistic + BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); + int64_t total_delay = 0; + g_queryInfo.specifiedQueryInfo.totalQueried = 0; + g_queryInfo.specifiedQueryInfo.totalFail = 0; - return 0; -} + // clear + for (int i = 0; i < threadCnt; ++i) { + qThreadInfo * pThreadInfo = infos + i; + + // append delay + benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, + pThreadInfo->query_delay_list->size, true); + total_delay += pThreadInfo->total_delay; -// close conn -void closeQueryConn(qThreadInfo * pThreadInfo, int iface) { - if (iface == REST_IFACE) { -#ifdef WINDOWS - closesocket(pThreadInfo->sockfd); - WSACleanup(); -#else - close(pThreadInfo->sockfd); -#endif + // free delay + benchArrayDestroy(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + + } + + // sort + qsort(delay_list->pData, delay_list->size, delay_list->elemSize, compare); + + // show delay min max + if (delay_list->size) { + infoPrint( + "spend %.6fs using " + "%d threads complete query %d times, " + "min delay: %.6fs, " + "avg delay: %.6fs, " + "p90: %.6fs, " + "p95: %.6fs, " + "p99: %.6fs, " + "max: %.6fs\n", + spend/1E6, + threadCnt, (int)delay_list->size, + *(int64_t *)(benchArrayGet(delay_list, 0))/1E6, + (double)total_delay/delay_list->size/1E6, + *(int64_t *)(benchArrayGet(delay_list, + (int32_t)(delay_list->size * 0.9)))/1E6, + *(int64_t *)(benchArrayGet(delay_list, + (int32_t)(delay_list->size * 0.95)))/1E6, + *(int64_t *)(benchArrayGet(delay_list, + (int32_t)(delay_list->size * 0.99)))/1E6, + *(int64_t *)(benchArrayGet(delay_list, + (int32_t)(delay_list->size - 1)))/1E6); } else { - closeBenchConn(pThreadInfo->conn); - pThreadInfo->conn = NULL; + errorPrint("%s() LN%d, delay_list size: %"PRId64"\n", + __func__, __LINE__, (int64_t)delay_list->size); } + benchArrayDestroy(delay_list); } +// // super table query +// static int stbQuery(uint16_t iface, char* dbName) { int ret = -1; pthread_t * pidsOfSub = NULL; - qThreadInfo *infosOfSub = NULL; + qThreadInfo *threadInfos = NULL; g_queryInfo.superQueryInfo.totalQueried = 0; g_queryInfo.superQueryInfo.totalFail = 0; @@ -425,7 +453,7 @@ static int stbQuery(uint16_t iface, char* dbName) { pidsOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt *sizeof(pthread_t), false); - infosOfSub = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt + threadInfos = benchCalloc(1, g_queryInfo.superQueryInfo.threadCnt *sizeof(qThreadInfo), false); int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; @@ -445,7 +473,7 @@ static int stbQuery(uint16_t iface, char* dbName) { uint64_t tableFrom = 0; int threadCnt = 0; for (int i = 0; i < nConcurrent; i++) { - qThreadInfo *pThreadInfo = infosOfSub + i; + qThreadInfo *pThreadInfo = threadInfos + i; pThreadInfo->threadID = i; pThreadInfo->start_table_from = tableFrom; pThreadInfo->ntables = i < b ? a + 1 : a; @@ -469,14 +497,16 @@ static int stbQuery(uint16_t iface, char* dbName) { if (threadCnt != nConcurrent ) { needExit = true; g_arguments->terminate = true; + goto OVER: } // real thread count g_queryInfo.superQueryInfo.threadCnt = threadCnt; + int64_t start = toolsGetTimestampUs(); for (int i = 0; i < threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); - qThreadInfo *pThreadInfo = infosOfSub + i; + qThreadInfo *pThreadInfo = threadInfos + i; // add succ g_queryInfo.superQueryInfo.totalQueried += pThreadInfo->nSucc; if (g_arguments->continueIfFail == YES_IF_FAILED) { @@ -488,11 +518,20 @@ static int stbQuery(uint16_t iface, char* dbName) { // close conn closeQueryConn(pThreadInfo, iface); } + int64_t end = toolsGetTimestampUs(); + + if (needExit) { + goto OVER: + } + + // total show + totalChildQuery(threadInfos, threadCnt, end - start); ret = 0; + OVER: tmfree((char *)pidsOfSub); - tmfree((char *)infosOfSub); + tmfree((char *)threadInfos); for (int64_t i = 0; i < g_queryInfo.superQueryInfo.childTblCount; ++i) { tmfree(g_queryInfo.superQueryInfo.childTblName[i]); @@ -501,27 +540,11 @@ static int stbQuery(uint16_t iface, char* dbName) { return ret; } -// free g_queryInfo.specailQueryInfo memory , can re-call -void freeSpecialQueryInfo() { - // can re-call - if (g_queryInfo.specifiedQueryInfo.sqls == NULL) { - return; - } - - // loop free each item memory - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqls->size; ++i) { - SSQL *sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); - tmfree(sql->command); - tmfree(sql->delay_list); - } - - // free Array - benchArrayDestroy(g_queryInfo.specifiedQueryInfo.sqls); - g_queryInfo.specifiedQueryInfo.sqls = NULL; -} - -// spec query thread +// +// specQuery +// static int specQuery(uint16_t iface, char* dbName) { + int ret = -1; pthread_t *pids = NULL; qThreadInfo *infos = NULL; int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; @@ -538,7 +561,7 @@ static int specQuery(uint16_t iface, char* dbName) { return 0; } - // malloc funciton global memory + // malloc threads memory pids = benchCalloc(1, nConcurrent * sizeof(pthread_t), false); infos = benchCalloc(1, nConcurrent * sizeof(qThreadInfo), false); @@ -605,38 +628,46 @@ static int specQuery(uint16_t iface, char* dbName) { // create if (needExit) { errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); - // free current funciton malloc memory - tmfree((char *)pids); - tmfree((char *)infos); - // free global - freeSpecialQueryInfo(); - return -1; - } - - // execute successfully - uint64_t query_times = g_queryInfo.specifiedQueryInfo.queryTimes; - uint64_t totalQueryTimes = query_times * threadCnt; - double avg_delay = 0.0; + goto OVER: + } + ret = 0; + + // + // show QPS and P90 ... + // + uint64_t n = 0; + double delays = 0; + double avg_delay = 0.0; + uint64_t totalQueried = 0; + uint64_t totalFail = 0; for (int j = 0; j < threadCnt; j++) { qThreadInfo *pThreadInfo = infos + j; - avg_delay += pThreadInfo->avg_delay; - for (uint64_t k = 0; k < g_queryInfo.specifiedQueryInfo.queryTimes; k++) { - sql->delay_list[j * query_times + k] = pThreadInfo->query_delay_list[k]; + + // total one sql + for (uint64_t k = 0; k < pThreadInfo->query_delay_list->size; k++) { + int64_t * delay = benchArrayGet(pThreadInfo->query_delay_list, k); + sql->delay_list[n++] = *delay; + delays += delay; } // total queries - g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; + totalQueried += pThreadInfo->nSucc; if (g_arguments->continueIfFail == YES_IF_FAILED) { - g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; - g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; + totalQueried += pThreadInfo->nFail; + totalFail += pThreadInfo->nFail; } - // free BArray - tmfree(pThreadInfo->query_delay_list); + // free BArray query_delay_list + benchArrayDestroy(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; } + + // appand current sql + g_queryInfo.specifiedQueryInfo.totalQueried += totalQueried; + g_queryInfo.specifiedQueryInfo.totalFail += totalFail; + avg_delay /= threadCnt; - qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); + qsort(sql->delay_list, n, sizeof(uint64_t), compare); int32_t bufLen = strlen(sql->command) + 512; char * buf = benchCalloc(bufLen, sizeof(char), false); snprintf(buf , bufLen, "complete query with %d threads and %" PRIu64 " " @@ -648,33 +679,37 @@ static int specQuery(uint16_t iface, char* dbName) { "p90: %.6fs " "p95: %.6fs " "p99: %.6fs " - "SQL command: %s" - "\n", - threadCnt, query_times, - i + 1, spend/1E6, totalQueryTimes / (spend/1E6), - avg_delay / 1E6, /* avg */ - sql->delay_list[0] / 1E6, /* min */ - sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ + "SQL command: %s \n", + threadCnt, totalQueried, + i + 1, spend/1E6, totalQueried / (spend/1E6), + avg_delay / 1E6, /* avg */ + sql->delay_list[0] / 1E6, /* min */ + sql->delay_list[n - 1] / 1E6, /* max */ /* p90 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)] / 1E6, + sql->delay_list[(uint64_t)(n * 0.90)] / 1E6, /* p95 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)] / 1E6, + sql->delay_list[(uint64_t)(n * 0.95)] / 1E6, /* p99 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); + sql->delay_list[(uint64_t)(n * 0.99)] / 1E6, + sql->command); + infoPrintNoTimestamp("%s", buf); infoPrintNoTimestampToFile("%s", buf); tmfree(buf); } +OVER: tmfree((char *)pids); tmfree((char *)infos); // free specialQueryInfo freeSpecialQueryInfo(); - return 0; + return ret; } -// spec query mix +// +// specQueryMix +// static int specQueryMix(uint16_t iface, char* dbName) { // init int code = -1; @@ -732,6 +767,16 @@ static int specQueryMix(uint16_t iface, char* dbName) { for (int i = 0; i < threadCnt; ++i) { pthread_join(pids[i], NULL); closeQueryConn(); + + // total queries + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; + if (g_arguments->continueIfFail == YES_IF_FAILED) { + // yes need add failed count + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; + } + + // destory if (needExit) { benchArrayDestroy(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; @@ -742,78 +787,14 @@ static int specQueryMix(uint16_t iface, char* dbName) { // create if (needExit) { errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); - // free current funciton malloc memory - tmfree((char *)pids); - tmfree((char *)infos); - // free sqls - freeSpecialQueryInfo(); - return -1; + goto OVER: } // statistic - BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); - int64_t total_delay = 0; - g_queryInfo.specifiedQueryInfo.totalQueried = 0; - g_queryInfo.specifiedQueryInfo.totalFail = 0; - - // clear - for (int i = 0; i < nConcurrent; ++i) { - qThreadInfo * pThreadInfo = infos + i; - - // total queries - g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; - if (g_arguments->continueIfFail == YES_IF_FAILED) { - // yes need add failed count - g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; - g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; - } - - // append delay - benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, - pThreadInfo->query_delay_list->size, true); - total_delay += pThreadInfo->total_delay; - - // free delay - benchArrayDestroy(pThreadInfo->query_delay_list); - pThreadInfo->query_delay_list = NULL; - - // close conn - closeQueryConn(pThreadInfo, iface); - } - - // sort - qsort(delay_list->pData, delay_list->size, delay_list->elemSize, compare); - - // show delay min max - if (delay_list->size) { - infoPrint( - "spend %.6fs using " - "%d threads complete query %d times, " - "min delay: %.6fs, " - "avg delay: %.6fs, " - "p90: %.6fs, " - "p95: %.6fs, " - "p99: %.6fs, " - "max: %.6fs\n", - (end - start)/1E6, - threadCnt, (int)delay_list->size, - *(int64_t *)(benchArrayGet(delay_list, 0))/1E6, - (double)total_delay/delay_list->size/1E6, - *(int64_t *)(benchArrayGet(delay_list, - (int32_t)(delay_list->size * 0.9)))/1E6, - *(int64_t *)(benchArrayGet(delay_list, - (int32_t)(delay_list->size * 0.95)))/1E6, - *(int64_t *)(benchArrayGet(delay_list, - (int32_t)(delay_list->size * 0.99)))/1E6, - *(int64_t *)(benchArrayGet(delay_list, - (int32_t)(delay_list->size - 1)))/1E6); - } else { - errorPrint("%s() LN%d, delay_list size: %"PRId64"\n", - __func__, __LINE__, (int64_t)delay_list->size); - } - benchArrayDestroy(delay_list); + totalChildQuery(infos, threadCnt, end - start); code = 0; +OVER: tmfree(pids); tmfree(infos); @@ -823,154 +804,8 @@ static int specQueryMix(uint16_t iface, char* dbName) { return code; } -#define KILLID_LEN 20 - -void *queryKiller(void *arg) { - char host[MAX_HOSTNAME_LEN] = {0}; - tstrncpy(host, g_arguments->host, MAX_HOSTNAME_LEN); - - while (true) { - TAOS *taos = taos_connect(g_arguments->host, g_arguments->user, - g_arguments->password, NULL, g_arguments->port); - if (NULL == taos) { - errorPrint("Slow query killer thread " - "failed to connect to the server %s\n", - g_arguments->host); - return NULL; - } - - char command[TSDB_MAX_ALLOWED_SQL_LEN] = - "SELECT kill_id,exec_usec,sql FROM performance_schema.perf_queries"; - TAOS_RES *res = taos_query(taos, command); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(command, code, res); - } - - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - int32_t *lengths = taos_fetch_lengths(res); - if (lengths[0] <= 0) { - infoPrint("No valid query found by %s\n", command); - } else { - int64_t execUSec = *(int64_t*)row[1]; - - if (execUSec > g_queryInfo.killQueryThreshold * 1000000) { - char sql[SHORT_1K_SQL_BUFF_LEN] = {0}; - tstrncpy(sql, (char*)row[2], - min(strlen((char*)row[2])+1, - SHORT_1K_SQL_BUFF_LEN)); - - char killId[KILLID_LEN] = {0}; - tstrncpy(killId, (char*)row[0], - min(strlen((char*)row[0])+1, KILLID_LEN)); - char killCommand[KILLID_LEN + 15] = {0}; - snprintf(killCommand, KILLID_LEN + 15, - "KILL QUERY '%s'", killId); - TAOS_RES *resKill = taos_query(taos, killCommand); - int32_t codeKill = taos_errno(resKill); - if (codeKill) { - printErrCmdCodeStr(killCommand, codeKill, resKill); - } else { - infoPrint("%s succeed, sql: %s killed!\n", - killCommand, sql); - taos_free_result(resKill); - } - } - } - } - - taos_free_result(res); - taos_close(taos); - toolsMsleep(g_queryInfo.killQueryInterval*1000); - } - - return NULL; -} - -// -int killSlowQuery() { - pthread_t pidKiller = {0}; - int32_t ret = pthread_create(&pidKiller, NULL, queryKiller, NULL); - if (ret != 0) { - errorPrint("pthread_create failed create queryKiller thread. error code =%d \n", ret); - return -1; - } - pthread_join(pidKiller, NULL); - toolsMsleep(1000); - return 0; -} - -// fetch super table child name from server -int fetchChildTableName(char *dbName, char *stbName) { - SBenchConn* conn = initBenchConn(); - if (conn == NULL) { - return -1; - } - - // get child count - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (3 == g_majorVersionOfClient) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", - dbName, stbName); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(TBNAME) FROM %s.%s", - dbName, stbName); - } - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - closeBenchConn(conn); - return -1; - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == strlen((char *)(row[0]))) { - errorPrint("stable %s have no child table\n", stbName); - taos_free_result(res); - closeBenchConn(conn); - return -1; - } - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - - // set child table count - g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); - } - infoPrint("%s's childTblCount: %" PRId64 "\n", stbName, g_queryInfo.superQueryInfo.childTblCount); - taos_free_result(res); - - // malloc memory with child table count - g_queryInfo.superQueryInfo.childTblName = - benchCalloc(g_queryInfo.superQueryInfo.childTblCount, - sizeof(char *), false); - // fetch child table name - if (getAllChildNameOfSuperTable( - conn->taos, dbName, stbName, - g_queryInfo.superQueryInfo.childTblName, - g_queryInfo.superQueryInfo.childTblCount)) { - // faild - tmfree(g_queryInfo.superQueryInfo.childTblName); - closeBenchConn(conn); - return -1; - } - closeBenchConn(conn); - - // succ - return 0; -} - // total query for end void totalQuery(int64_t spends) { - - double tInS = (double)spends / 1000; - // specifiedQuery if (g_queryInfo.specifiedQueryInfo.totalQueried) { infoPrint("Total specified queries: %" PRIu64 "\n", @@ -997,6 +832,7 @@ void totalQuery(int64_t spends) { } // show + double tInS = (double)spends / 1000; char buf[512] = ""; snprintf(buf, sizeof(buf), "Spend %.4f second completed total queries: %" PRIu64 diff --git a/src/benchUtil.c b/src/benchUtil.c index 8f05c831..4ca90ec3 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1557,4 +1557,207 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { return h1; } -#endif \ No newline at end of file +#endif + + +// +// ---------------- benchQuery util ---------------------- +// + +// init conn +int32_t initQueryConn(qThreadInfo * pThreadInfo, int iface) { + // create conn + if (iface == REST_IFACE) { + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + pThreadInfo->sockfd = sockfd; + } else { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + return -1; + } + } + + return 0; +} + +// close conn +void closeQueryConn(qThreadInfo * pThreadInfo, int iface) { + if (iface == REST_IFACE) { +#ifdef WINDOWS + closesocket(pThreadInfo->sockfd); + WSACleanup(); +#else + close(pThreadInfo->sockfd); +#endif + } else { + closeBenchConn(pThreadInfo->conn); + pThreadInfo->conn = NULL; + } +} + + +// free g_queryInfo.specailQueryInfo memory , can re-call +void freeSpecialQueryInfo() { + // can re-call + if (g_queryInfo.specifiedQueryInfo.sqls == NULL) { + return; + } + + // loop free each item memory + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqls->size; ++i) { + SSQL *sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); + tmfree(sql->command); + tmfree(sql->delay_list); + } + + // free Array + benchArrayDestroy(g_queryInfo.specifiedQueryInfo.sqls); + g_queryInfo.specifiedQueryInfo.sqls = NULL; +} + + +#define KILLID_LEN 20 + +void *queryKiller(void *arg) { + char host[MAX_HOSTNAME_LEN] = {0}; + tstrncpy(host, g_arguments->host, MAX_HOSTNAME_LEN); + + while (true) { + TAOS *taos = taos_connect(g_arguments->host, g_arguments->user, + g_arguments->password, NULL, g_arguments->port); + if (NULL == taos) { + errorPrint("Slow query killer thread " + "failed to connect to the server %s\n", + g_arguments->host); + return NULL; + } + + char command[TSDB_MAX_ALLOWED_SQL_LEN] = + "SELECT kill_id,exec_usec,sql FROM performance_schema.perf_queries"; + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(command, code, res); + } + + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + int32_t *lengths = taos_fetch_lengths(res); + if (lengths[0] <= 0) { + infoPrint("No valid query found by %s\n", command); + } else { + int64_t execUSec = *(int64_t*)row[1]; + + if (execUSec > g_queryInfo.killQueryThreshold * 1000000) { + char sql[SHORT_1K_SQL_BUFF_LEN] = {0}; + tstrncpy(sql, (char*)row[2], + min(strlen((char*)row[2])+1, + SHORT_1K_SQL_BUFF_LEN)); + + char killId[KILLID_LEN] = {0}; + tstrncpy(killId, (char*)row[0], + min(strlen((char*)row[0])+1, KILLID_LEN)); + char killCommand[KILLID_LEN + 15] = {0}; + snprintf(killCommand, KILLID_LEN + 15, + "KILL QUERY '%s'", killId); + TAOS_RES *resKill = taos_query(taos, killCommand); + int32_t codeKill = taos_errno(resKill); + if (codeKill) { + printErrCmdCodeStr(killCommand, codeKill, resKill); + } else { + infoPrint("%s succeed, sql: %s killed!\n", + killCommand, sql); + taos_free_result(resKill); + } + } + } + } + + taos_free_result(res); + taos_close(taos); + toolsMsleep(g_queryInfo.killQueryInterval*1000); + } + + return NULL; +} + +// kill show +int killSlowQuery() { + pthread_t pidKiller = {0}; + int32_t ret = pthread_create(&pidKiller, NULL, queryKiller, NULL); + if (ret != 0) { + errorPrint("pthread_create failed create queryKiller thread. error code =%d \n", ret); + return -1; + } + pthread_join(pidKiller, NULL); + toolsMsleep(1000); + return 0; +} + +// fetch super table child name from server +int fetchChildTableName(char *dbName, char *stbName) { + SBenchConn* conn = initBenchConn(); + if (conn == NULL) { + return -1; + } + + // get child count + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + if (3 == g_majorVersionOfClient) { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", + dbName, stbName); + } else { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT COUNT(TBNAME) FROM %s.%s", + dbName, stbName); + } + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return -1; + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == strlen((char *)(row[0]))) { + errorPrint("stable %s have no child table\n", stbName); + taos_free_result(res); + closeBenchConn(conn); + return -1; + } + char temp[256] = {0}; + taos_print_row(temp, row, fields, num_fields); + + // set child table count + g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); + } + infoPrint("%s's childTblCount: %" PRId64 "\n", stbName, g_queryInfo.superQueryInfo.childTblCount); + taos_free_result(res); + + // malloc memory with child table count + g_queryInfo.superQueryInfo.childTblName = + benchCalloc(g_queryInfo.superQueryInfo.childTblCount, + sizeof(char *), false); + // fetch child table name + if (getAllChildNameOfSuperTable( + conn->taos, dbName, stbName, + g_queryInfo.superQueryInfo.childTblName, + g_queryInfo.superQueryInfo.childTblCount)) { + // faild + tmfree(g_queryInfo.superQueryInfo.childTblName); + closeBenchConn(conn); + return -1; + } + closeBenchConn(conn); + + // succ + return 0; +} From b01c091573e7696dc6b76bcb19099727c2eaf10d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 21:42:38 +0800 Subject: [PATCH 16/49] fix: build error for first test --- src/benchQuery.c | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 276ba3c5..5f204cbc 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -88,10 +88,11 @@ void autoSleep(uint64_t st, uint64_t et) { int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // execute sql if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", true)) { - errorPrint("%s() LN%d, reset query cache failed\n", - __func__, __LINE__); - break; + errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); + return -1; } + // succ + return 0; } @@ -188,8 +189,6 @@ static void *specQueryMixThread(void *sarg) { } } - // delay sort - qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -267,8 +266,6 @@ static void *specQueryThread(void *sarg) { lastPrintTime = showRealQPS(pThreadInfo, lastPrintTime, startTs); } - // delay sort - qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -344,7 +341,7 @@ static void *stbQueryThread(void *sarg) { errorPrint("failed call stb selectAndGetResult, i=%d j=%d\n", i, j); g_fail = true; tmfree(sqlstr); - return NULL + return NULL; } uint64_t delay = toolsGetTimestampUs() - s; debugPrint("%s() LN%d, delay: %"PRIu64"\n", __func__, __LINE__, delay); @@ -364,8 +361,6 @@ static void *stbQueryThread(void *sarg) { } tmfree(sqlstr); - // delay sort - qsort(pThreadInfo->query_delay_list, pThreadInfo->query_delay_list->size, sizeof(uint64_t), compare); return NULL; } @@ -375,7 +370,7 @@ static void *stbQueryThread(void *sarg) { void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // valid check - if(threadInfo == NULL || threadCnt == 0) { + if (infos == NULL || threadCnt == 0) { return ; } @@ -484,9 +479,9 @@ static int stbQuery(uint16_t iface, char* dbName) { if (initQueryConn(pThreadInfo, iface)){ break; } - int ret = pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); - if (ret != 0) { - errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + int code = pthread_create(pidsOfSub + i, NULL, stbQueryThread, pThreadInfo); + if (code != 0) { + errorPrint("failed stbQueryThread create. error code =%d \n", code); break; } threadCnt ++; @@ -497,7 +492,7 @@ static int stbQuery(uint16_t iface, char* dbName) { if (threadCnt != nConcurrent ) { needExit = true; g_arguments->terminate = true; - goto OVER: + goto OVER; } // real thread count @@ -521,7 +516,7 @@ static int stbQuery(uint16_t iface, char* dbName) { int64_t end = toolsGetTimestampUs(); if (needExit) { - goto OVER: + goto OVER; } // total show @@ -547,7 +542,7 @@ static int specQuery(uint16_t iface, char* dbName) { int ret = -1; pthread_t *pids = NULL; qThreadInfo *infos = NULL; - int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; + int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; g_queryInfo.specifiedQueryInfo.totalQueried = 0; g_queryInfo.specifiedQueryInfo.totalFail = 0; @@ -590,9 +585,9 @@ static int specQuery(uint16_t iface, char* dbName) { break; } - int ret = pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); - if (ret != 0) { - errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + int code = pthread_create(pids + j, NULL, specQueryThread, pThreadInfo); + if (code != 0) { + errorPrint("failed specQueryThread create. error code =%d \n", code); break; } threadCnt++; @@ -712,7 +707,7 @@ static int specQuery(uint16_t iface, char* dbName) { // static int specQueryMix(uint16_t iface, char* dbName) { // init - int code = -1; + int ret = -1; int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; pthread_t * pids = benchCalloc(nConcurrent, sizeof(pthread_t), true); qThreadInfo *infos = benchCalloc(nConcurrent, sizeof(qThreadInfo), true); @@ -747,9 +742,9 @@ static int specQueryMix(uint16_t iface, char* dbName) { break; } // main run - int ret = pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); - if (ret != 0) { - errorPrint("failed specQueryMixThread create. error code =%d \n", ret); + int code = pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); + if (code != 0) { + errorPrint("failed specQueryMixThread create. error code =%d \n", code); break; } @@ -792,7 +787,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { // statistic totalChildQuery(infos, threadCnt, end - start); - code = 0; + ret = 0; OVER: tmfree(pids); @@ -801,7 +796,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { // free sqls freeSpecialQueryInfo(); - return code; + return ret; } // total query for end From c80a20375d2cfabf8b69ad5cc97c46ac7422692f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 21:54:06 +0800 Subject: [PATCH 17/49] fix: lost pThreadInfo variant --- src/benchQuery.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 5f204cbc..eb17ddc6 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -560,7 +560,6 @@ static int specQuery(uint16_t iface, char* dbName) { pids = benchCalloc(1, nConcurrent * sizeof(pthread_t), false); infos = benchCalloc(1, nConcurrent * sizeof(qThreadInfo), false); - bool exeError = false; for (uint64_t i = 0; i < nSqlCount; i++) { if( g_arguments->terminate ) { break; @@ -623,7 +622,7 @@ static int specQuery(uint16_t iface, char* dbName) { // create if (needExit) { errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); - goto OVER: + goto OVER; } ret = 0; @@ -642,7 +641,7 @@ static int specQuery(uint16_t iface, char* dbName) { for (uint64_t k = 0; k < pThreadInfo->query_delay_list->size; k++) { int64_t * delay = benchArrayGet(pThreadInfo->query_delay_list, k); sql->delay_list[n++] = *delay; - delays += delay; + delays += *delay; } // total queries @@ -708,7 +707,7 @@ static int specQuery(uint16_t iface, char* dbName) { static int specQueryMix(uint16_t iface, char* dbName) { // init int ret = -1; - int nConcurrent = g_queryInfo.specifiedQueryInfo.nConcurrent; + int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; pthread_t * pids = benchCalloc(nConcurrent, sizeof(pthread_t), true); qThreadInfo *infos = benchCalloc(nConcurrent, sizeof(qThreadInfo), true); @@ -761,7 +760,8 @@ static int specQueryMix(uint16_t iface, char* dbName) { int64_t start = toolsGetTimestampUs(); for (int i = 0; i < threadCnt; ++i) { pthread_join(pids[i], NULL); - closeQueryConn(); + qThreadInfo *pThreadInfo = infos + i; + closeQueryConn(pThreadInfo, iface); // total queries g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; @@ -782,7 +782,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { // create if (needExit) { errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); - goto OVER: + goto OVER; } // statistic From f13b50b23575d3f46b4300a3791a6d6e641ee95c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 22:01:57 +0800 Subject: [PATCH 18/49] fix: benchUtil.c extern variant not declare --- inc/bench.h | 1 + src/benchMain.c | 2 +- src/benchUtil.c | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 3fe2d2e5..d579d696 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -898,6 +898,7 @@ extern bool g_fail; extern char configDir[]; extern tools_cJSON * root; extern uint64_t g_memoryUsage; +extern int32_t g_majorVersionOfClient; #define min(a, b) (((a) < (b)) ? (a) : (b)) #define BARRAY_GET_ELEM(array, index) \ diff --git a/src/benchMain.c b/src/benchMain.c index 7f590df9..46f60869 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -25,7 +25,7 @@ tools_cJSON* root; #define CLIENT_INFO_LEN 20 static char g_client_info[CLIENT_INFO_LEN] = {0}; -int g_majorVersionOfClient = 0; +int32_t g_majorVersionOfClient = 0; // set flag if command passed, see ARG_OPT_ ??? uint64_t g_argFlag = 0; diff --git a/src/benchUtil.c b/src/benchUtil.c index 4ca90ec3..eb91c6ef 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -245,7 +245,7 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { char mark[32] = "xxxx"; char *pos = strstr(inSql, mark); if (0 == pos) { - errorPrint("sql format error, super table query not found child table replace mark string '%s'", mark); + errorPrint("sql format error, sql not found mark string '%s'", mark); return -1; } @@ -256,7 +256,7 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { tstrncpy(outSql, inSql, pos - inSql + 1); snprintf(outSql + strlen(outSql), TSDB_MAX_ALLOWED_SQL_LEN -1, - "%s%s", subTblName, pos + strlen(sourceString)); + "%s%s", subTblName, pos + strlen(mark)); return 0; } @@ -846,7 +846,7 @@ int64_t fetchResult(TAOS_RES *res, char * filePath) { if (toFile) { if (totalLen >= (FETCH_BUFFER_SIZE - HEAD_BUFF_LEN * 2)) { // buff is full - appendResultBufToFile(databuf, pThreadInfo->filePath); + appendResultBufToFile(databuf, filePath); totalLen = 0; memset(databuf, 0, FETCH_BUFFER_SIZE); } From 9c4c8063e1b897a271ab3f05f0a53189149a288b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 22:03:36 +0800 Subject: [PATCH 19/49] fix: remove benchQuery.c declare define --- src/benchQuery.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index eb17ddc6..c713f3f4 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -13,8 +13,6 @@ #include #include "benchLog.h" -extern int g_majorVersionOfClient; - // query and get result other true is no test sql int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { int ret = 0; From 218fba1edce628fd2c16e02db8391207d56c891c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 22:57:06 +0800 Subject: [PATCH 20/49] fix: kill query buffer overflow --- src/benchQuery.c | 21 ++++++++++++++------- src/benchUtil.c | 7 +++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index c713f3f4..48f53207 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -17,9 +17,12 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { int ret = 0; + // user cancel if (g_arguments->terminate) { return -1; } + + // execute sql uint32_t threadID = pThreadInfo->threadID; char dbName[TSDB_DB_NAME_LEN] = {0}; tstrncpy(dbName, g_queryInfo.dbName, TSDB_DB_NAME_LEN); @@ -57,16 +60,20 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { debugPrint("query sql:%s rows:%"PRId64"\n", command, rows); } - // record succ or fail count - if (ret !=0) { + // record count + if (ret ==0) { + // succ if (!other) pThreadInfo->nSucc ++; + } else { + // fail + if (!other) + pThreadInfo->nFail ++; + + // continue option if (YES_IF_FAILED == g_arguments->continueIfFail) { - // force continue , so ignore this error - ret = 0; + ret = 0; // force continue } - } else if(!other) { - pThreadInfo->nFail ++; } return ret; @@ -102,7 +109,7 @@ int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // show rela qps int64_t showRealQPS(qThreadInfo* pThreadInfo, int64_t lastPrintTime, int64_t startTs) { int64_t now = toolsGetTimestampMs(); - if (now - lastPrintTime > 30 * 1000) { + if (now - lastPrintTime > 10 * 1000) { // real total uint64_t totalQueried = pThreadInfo->nSucc; if(g_arguments->continueIfFail == YES_IF_FAILED) { diff --git a/src/benchUtil.c b/src/benchUtil.c index 9d14249f..e7f85566 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1621,7 +1621,7 @@ void freeSpecialQueryInfo() { } -#define KILLID_LEN 20 +#define KILLID_LEN 64 void *queryKiller(void *arg) { char host[MAX_HOSTNAME_LEN] = {0}; @@ -1662,9 +1662,8 @@ void *queryKiller(void *arg) { char killId[KILLID_LEN] = {0}; tstrncpy(killId, (char*)row[0], min(strlen((char*)row[0])+1, KILLID_LEN)); - char killCommand[KILLID_LEN + 15] = {0}; - snprintf(killCommand, KILLID_LEN + 15, - "KILL QUERY '%s'", killId); + char killCommand[KILLID_LEN + 32] = {0}; + snprintf(killCommand, sizeof(killCommand), "KILL QUERY '%s'", killId); TAOS_RES *resKill = taos_query(taos, killCommand); int32_t codeKill = taos_errno(resKill); if (codeKill) { From c8cd304cdd43416da307d7ac48c218a1f8ccf69c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 19 Jan 2025 23:49:12 +0800 Subject: [PATCH 21/49] fix: buff calc over flow --- src/benchMain.c | 5 ----- src/benchQuery.c | 2 +- src/benchUtil.c | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/benchMain.c b/src/benchMain.c index 46f60869..afbc1096 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -43,11 +43,6 @@ void* benchCancelHandler(void* arg) { g_arguments->terminate = true; toolsMsleep(10); - if (g_arguments->in_prompt || INSERT_TEST != g_arguments->test_mode) { - toolsMsleep(100); - postFreeResource(); - exit(EXIT_SUCCESS); - } return NULL; } #endif diff --git a/src/benchQuery.c b/src/benchQuery.c index 48f53207..1a18a6ec 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -233,7 +233,7 @@ static void *specQueryThread(void *sarg) { while (index < queryTimes) { // use cancel if(g_arguments->terminate) { - infoPrint("%s\n", "user cancel , so exit testing."); + infoPrint("thread[%d] user cancel , so exit testing.\n", pThreadInfo->threadID); break; } diff --git a/src/benchUtil.c b/src/benchUtil.c index e7f85566..c593d47a 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1314,7 +1314,7 @@ void destroySockFd(int sockfd) { FORCE_INLINE void printErrCmdCodeStr(char *cmd, int32_t code, TAOS_RES *res) { char buff[512]; char *msg = cmd; - if (strlen(cmd) > sizeof(msg)) { + if (strlen(cmd) >= sizeof(buff)) { memcpy(buff, cmd, 500); buff[500] = 0; strcat(buff, "..."); From af7870e9a21ec1811c8e4cf2dc168d1d990ec37a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 17:37:04 +0800 Subject: [PATCH 22/49] fix: spec query and super query only have one or must have one --- inc/bench.h | 2 +- src/benchJsonOpt.c | 218 ++++++++++++++++++++++++++------------------- src/benchMain.c | 6 +- 3 files changed, 133 insertions(+), 93 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index d579d696..522d3146 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -911,7 +911,7 @@ void initArgument(); void queryAggrFunc(); void parseFieldDatatype(char *dataType, BArray *fields, bool isTag); /* demoJsonOpt.c */ -int getInfoFromJsonFile(); +int readQueryJson(char * file); /* demoUtil.c */ int compare(const void *a, const void *b); void encodeAuthBase64(); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 1c27037f..56eec595 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1770,79 +1770,8 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) { return code; } -static int getMetaFromQueryJsonFile(tools_cJSON *json) { - int32_t code = -1; - - tools_cJSON *telnet_tcp_port = - tools_cJSON_GetObjectItem(json, "telnet_tcp_port"); - if (tools_cJSON_IsNumber(telnet_tcp_port)) { - g_arguments->telnet_tcp_port = (uint16_t)telnet_tcp_port->valueint; - } - - tools_cJSON *gQueryTimes = tools_cJSON_GetObjectItem(json, "query_times"); - if (tools_cJSON_IsNumber(gQueryTimes)) { - g_queryInfo.query_times = gQueryTimes->valueint; - } else { - g_queryInfo.query_times = 1; - } - - tools_cJSON *gKillSlowQueryThreshold = - tools_cJSON_GetObjectItem(json, "kill_slow_query_threshold"); - if (tools_cJSON_IsNumber(gKillSlowQueryThreshold)) { - g_queryInfo.killQueryThreshold = gKillSlowQueryThreshold->valueint; - } else { - g_queryInfo.killQueryThreshold = 0; - } - - tools_cJSON *gKillSlowQueryInterval = - tools_cJSON_GetObjectItem(json, "kill_slow_query_interval"); - if (tools_cJSON_IsNumber(gKillSlowQueryInterval)) { - g_queryInfo.killQueryInterval = gKillSlowQueryInterval ->valueint; - } else { - g_queryInfo.killQueryInterval = 1; /* by default, interval 1s */ - } - - tools_cJSON *resetCache = - tools_cJSON_GetObjectItem(json, "reset_query_cache"); - if (tools_cJSON_IsString(resetCache)) { - if (0 == strcasecmp(resetCache->valuestring, "yes")) { - g_queryInfo.reset_query_cache = true; - } - } else { - g_queryInfo.reset_query_cache = false; - } - - tools_cJSON *respBuffer = - tools_cJSON_GetObjectItem(json, "response_buffer"); - if (tools_cJSON_IsNumber(respBuffer)) { - g_queryInfo.response_buffer = respBuffer->valueint; - } else { - g_queryInfo.response_buffer = RESP_BUF_LEN; - } - - tools_cJSON *dbs = tools_cJSON_GetObjectItem(json, "databases"); - if (tools_cJSON_IsString(dbs)) { - g_queryInfo.dbName = dbs->valuestring; - } - - tools_cJSON *queryMode = tools_cJSON_GetObjectItem(json, "query_mode"); - if (tools_cJSON_IsString(queryMode)) { - if (0 == strcasecmp(queryMode->valuestring, "rest")) { - g_queryInfo.iface = REST_IFACE; - } else if (0 == strcasecmp(queryMode->valuestring, "taosc")) { - g_queryInfo.iface = TAOSC_IFACE; - } else { - errorPrint("Invalid query_mode value: %s\n", - queryMode->valuestring); - goto PARSE_OVER; - } - } - // init sqls - g_queryInfo.specifiedQueryInfo.sqls = benchArrayInit(1, sizeof(SSQL)); - - // specified_table_query - tools_cJSON *specifiedQuery = - tools_cJSON_GetObjectItem(json, "specified_table_query"); +// Spec Query +int32_t readSpecQueryJson(tools_cJSON * specifiedQuery) { g_queryInfo.specifiedQueryInfo.concurrent = 1; if (tools_cJSON_IsObject(specifiedQuery)) { tools_cJSON *queryInterval = @@ -1873,7 +1802,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { } else { errorPrint("Invalid mixed_query value: %s\n", mixedQueryObj->valuestring); - goto PARSE_OVER; + return -1; } } @@ -1955,7 +1884,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { if (fp == NULL) { errorPrint("failed to open file: %s\n", sqlFileObj->valuestring); - goto PARSE_OVER; + return -1; } char *buf = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); while (fgets(buf, TSDB_MAX_ALLOWED_SQL_LEN, fp)) { @@ -2040,16 +1969,19 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { } } else { errorPrint("%s", "Invalid sql in json\n"); - goto PARSE_OVER; + return -1; } } } } } - // super_table_query - tools_cJSON *superQuery = - tools_cJSON_GetObjectItem(json, "super_table_query"); + // succ + return 0; +} + +// Super Query +int32_t readSuperQueryJson(tools_cJSON * superQuery) { g_queryInfo.superQueryInfo.threadCnt = 1; if (!superQuery || superQuery->type != tools_cJSON_Object) { g_queryInfo.superQueryInfo.sqlCount = 0; @@ -2175,7 +2107,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { errorPrint( "failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); - goto PARSE_OVER; + return -1; } g_queryInfo.superQueryInfo.sqlCount = superSqlSize; @@ -2201,11 +2133,118 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { } } } + + // succ + return 0; +} - code = 0; +// read query json +static int getMetaFromQueryJsonFile(tools_cJSON *json) { + int32_t code = -1; -PARSE_OVER: - return code; + // read common + tools_cJSON *telnet_tcp_port = + tools_cJSON_GetObjectItem(json, "telnet_tcp_port"); + if (tools_cJSON_IsNumber(telnet_tcp_port)) { + g_arguments->telnet_tcp_port = (uint16_t)telnet_tcp_port->valueint; + } + + tools_cJSON *gQueryTimes = tools_cJSON_GetObjectItem(json, "query_times"); + if (tools_cJSON_IsNumber(gQueryTimes)) { + g_queryInfo.query_times = gQueryTimes->valueint; + } else { + g_queryInfo.query_times = 1; + } + + tools_cJSON *gKillSlowQueryThreshold = + tools_cJSON_GetObjectItem(json, "kill_slow_query_threshold"); + if (tools_cJSON_IsNumber(gKillSlowQueryThreshold)) { + g_queryInfo.killQueryThreshold = gKillSlowQueryThreshold->valueint; + } else { + g_queryInfo.killQueryThreshold = 0; + } + + tools_cJSON *gKillSlowQueryInterval = + tools_cJSON_GetObjectItem(json, "kill_slow_query_interval"); + if (tools_cJSON_IsNumber(gKillSlowQueryInterval)) { + g_queryInfo.killQueryInterval = gKillSlowQueryInterval ->valueint; + } else { + g_queryInfo.killQueryInterval = 1; /* by default, interval 1s */ + } + + tools_cJSON *resetCache = + tools_cJSON_GetObjectItem(json, "reset_query_cache"); + if (tools_cJSON_IsString(resetCache)) { + if (0 == strcasecmp(resetCache->valuestring, "yes")) { + g_queryInfo.reset_query_cache = true; + } + } else { + g_queryInfo.reset_query_cache = false; + } + + tools_cJSON *respBuffer = + tools_cJSON_GetObjectItem(json, "response_buffer"); + if (tools_cJSON_IsNumber(respBuffer)) { + g_queryInfo.response_buffer = respBuffer->valueint; + } else { + g_queryInfo.response_buffer = RESP_BUF_LEN; + } + + tools_cJSON *dbs = tools_cJSON_GetObjectItem(json, "databases"); + if (tools_cJSON_IsString(dbs)) { + g_queryInfo.dbName = dbs->valuestring; + } + + tools_cJSON *queryMode = tools_cJSON_GetObjectItem(json, "query_mode"); + if (tools_cJSON_IsString(queryMode)) { + if (0 == strcasecmp(queryMode->valuestring, "rest")) { + g_queryInfo.iface = REST_IFACE; + } else if (0 == strcasecmp(queryMode->valuestring, "taosc")) { + g_queryInfo.iface = TAOSC_IFACE; + } else { + errorPrint("Invalid query_mode value: %s\n", + queryMode->valuestring); + return -1; + } + } + // init sqls + g_queryInfo.specifiedQueryInfo.sqls = benchArrayInit(1, sizeof(SSQL)); + + // specified_table_query + tools_cJSON *specifiedQuery = tools_cJSON_GetObjectItem(json, "specified_table_query"); + if (specifiedQuery) { + code = readSpecQueryJson(specifiedQuery); + if(code) { + errorPrint("failed to readSpecQueryJson code=%d", code); + return code; + } + } + + // super_table_query + tools_cJSON *superQuery = tools_cJSON_GetObjectItem(json, "super_table_query"); + if (superQuery) { + code = readSuperQueryJson(superQuery); + if(code) { + errorPrint("failed to readSuperQueryJson code=%d", code); + return code; + } + } + + // only have one + const char* errType = "json config invalid:"; + if (specifiedQuery && superQuery) { + errorPrint("%s only appear one for 'specified_table_query' and 'super_table_query' ", errType); + return -1; + } + + // must have one + if (specifiedQuery == NULL && superQuery == NULL ) { + errorPrint("%s must have one for 'specified_table_query' or 'super_table_query' ", errType); + return -1; + } + + // succ + return 0; } #ifdef TD_VER_COMPATIBLE_3_0_0_0 @@ -2378,8 +2417,7 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { } #endif -int getInfoFromJsonFile() { - char * file = g_arguments->metaFile; +int readQueryJson(char * file) { int32_t code = -1; FILE * fp = fopen(file, "r"); if (!fp) { @@ -2430,20 +2468,18 @@ int getInfoFromJsonFile() { // read common item code = getMetaFromCommonJsonFile(root); if (INSERT_TEST == g_arguments->test_mode || CSVFILE_TEST == g_arguments->test_mode) { + // insert code = getMetaFromInsertJsonFile(root); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 } else if (QUERY_TEST == g_arguments->test_mode) { -#else - } else { -#endif + // query memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); code = getMetaFromQueryJsonFile(root); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 } else if (SUBSCRIBE_TEST == g_arguments->test_mode) { + // subscribe memset(&g_tmqInfo, 0, sizeof(STmqMetaInfo)); code = getMetaFromTmqJsonFile(root); -#endif } + PARSE_OVER: free(content); fclose(fp); diff --git a/src/benchMain.c b/src/benchMain.c index afbc1096..75e8c094 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -123,7 +123,11 @@ int main(int argc, char* argv[]) { #endif if (g_arguments->metaFile) { g_arguments->totalChildTables = 0; - if (getInfoFromJsonFile()) exit(EXIT_FAILURE); + if (readQueryJson(g_arguments->metaFile)) { + errorPrint("failed to readQueryJson %s", g_arguments->metaFile); + exitLog(); + return -1; + } } else { modifyArgument(); } From fafebda3e8c166b0f83569290ca145c6ace6d8e8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 17:45:50 +0800 Subject: [PATCH 23/49] fix: add \n to log line --- inc/bench.h | 2 +- src/benchJsonOpt.c | 12 ++++++------ src/benchMain.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 522d3146..86d28a04 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -911,7 +911,7 @@ void initArgument(); void queryAggrFunc(); void parseFieldDatatype(char *dataType, BArray *fields, bool isTag); /* demoJsonOpt.c */ -int readQueryJson(char * file); +int readJsonConfig(char * file); /* demoUtil.c */ int compare(const void *a, const void *b); void encodeAuthBase64(); diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 56eec595..174be89e 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -2215,7 +2215,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { if (specifiedQuery) { code = readSpecQueryJson(specifiedQuery); if(code) { - errorPrint("failed to readSpecQueryJson code=%d", code); + errorPrint("failed to readSpecQueryJson code=%d \n", code); return code; } } @@ -2225,7 +2225,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { if (superQuery) { code = readSuperQueryJson(superQuery); if(code) { - errorPrint("failed to readSuperQueryJson code=%d", code); + errorPrint("failed to readSuperQueryJson code=%d \n", code); return code; } } @@ -2233,14 +2233,14 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { // only have one const char* errType = "json config invalid:"; if (specifiedQuery && superQuery) { - errorPrint("%s only appear one for 'specified_table_query' and 'super_table_query' ", errType); + errorPrint("%s only appear one for 'specified_table_query' and 'super_table_query' \n", errType); return -1; } // must have one if (specifiedQuery == NULL && superQuery == NULL ) { - errorPrint("%s must have one for 'specified_table_query' or 'super_table_query' ", errType); - return -1; + errorPrint("%s must have one for 'specified_table_query' or 'super_table_query' \n", errType); + return -1; } // succ @@ -2417,7 +2417,7 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { } #endif -int readQueryJson(char * file) { +int readJsonConfig(char * file) { int32_t code = -1; FILE * fp = fopen(file, "r"); if (!fp) { diff --git a/src/benchMain.c b/src/benchMain.c index 75e8c094..86ad795d 100644 --- a/src/benchMain.c +++ b/src/benchMain.c @@ -123,8 +123,8 @@ int main(int argc, char* argv[]) { #endif if (g_arguments->metaFile) { g_arguments->totalChildTables = 0; - if (readQueryJson(g_arguments->metaFile)) { - errorPrint("failed to readQueryJson %s", g_arguments->metaFile); + if (readJsonConfig(g_arguments->metaFile)) { + errorPrint("failed to readJsonConfig %s\n", g_arguments->metaFile); exitLog(); return -1; } From 2feb7968db2e5e79d90c5f35dfff785382c95359 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 19:42:38 +0800 Subject: [PATCH 24/49] fix: ci case for multi query conflict --- src/benchQuery.c | 2 +- tests/taosbenchmark/json/queryTaosc.json | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 1a18a6ec..7e8fbadb 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -391,7 +391,7 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // append delay benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, - pThreadInfo->query_delay_list->size, true); + pThreadInfo->query_delay_list->size, false); total_delay += pThreadInfo->total_delay; // free delay diff --git a/tests/taosbenchmark/json/queryTaosc.json b/tests/taosbenchmark/json/queryTaosc.json index d1e64e64..bcc2852b 100644 --- a/tests/taosbenchmark/json/queryTaosc.json +++ b/tests/taosbenchmark/json/queryTaosc.json @@ -22,16 +22,5 @@ "result": "./query_res1.txt" } ] - }, - "super_table_query": { - "stblname": "stb1", - "query_interval": 1, - "threads": 3, - "sqls": [ - { - "sql": "select last_row(ts) from xxxx", - "result": "./query_res2.txt" - } - ] } } From 4c4d4b7207e4ad67a0f839f3dfe599dfe7e83264 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 20:01:44 +0800 Subject: [PATCH 25/49] fix: split query to alone --- tests/taosbenchmark/json/queryTaosc1.json | 23 +++++++++++++++++++ .../taosdemoTestQueryWithJson.py | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 tests/taosbenchmark/json/queryTaosc1.json diff --git a/tests/taosbenchmark/json/queryTaosc1.json b/tests/taosbenchmark/json/queryTaosc1.json new file mode 100644 index 00000000..a3caa1c5 --- /dev/null +++ b/tests/taosbenchmark/json/queryTaosc1.json @@ -0,0 +1,23 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "taosc", + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/taosbenchmark/taosdemoTestQueryWithJson.py index b3341f21..e6ec2bf1 100644 --- a/tests/taosbenchmark/taosdemoTestQueryWithJson.py +++ b/tests/taosbenchmark/taosdemoTestQueryWithJson.py @@ -117,6 +117,9 @@ def run(self): # taosc query: query specified table and query super table os.system("%s -f ./taosbenchmark/json/queryInsertdata.json" % binPath) os.system("%s -f ./taosbenchmark/json/queryTaosc.json" % binPath) + # forbid parallel spec query with super query + os.system("%s -f ./taosbenchmark/json/queryTaosc1.json" % binPath) + os.system("cat query_res0.txt* > all_query_res0_taosc.txt") os.system("cat query_res1.txt* > all_query_res1_taosc.txt") os.system("cat query_res2.txt* > all_query_res2_taosc.txt") From 21f8a9028a6300c9024705fd47810f64882d4cf8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 20:05:44 +0800 Subject: [PATCH 26/49] queryRestful.json split two query json --- tests/taosbenchmark/json/queryRestful.json | 11 ---------- tests/taosbenchmark/json/queryRestful1.json | 24 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 tests/taosbenchmark/json/queryRestful1.json diff --git a/tests/taosbenchmark/json/queryRestful.json b/tests/taosbenchmark/json/queryRestful.json index 5de560fd..6cb83bc2 100644 --- a/tests/taosbenchmark/json/queryRestful.json +++ b/tests/taosbenchmark/json/queryRestful.json @@ -22,17 +22,6 @@ "result": "./query_res1.txt" } ] - }, - "super_table_query": { - "stblname": "stb1", - "query_interval": 1, - "threads": 3, - "sqls": [ - { - "sql": "select last_row(ts) from xxxx", - "result": "./query_res2.txt" - } - ] } } diff --git a/tests/taosbenchmark/json/queryRestful1.json b/tests/taosbenchmark/json/queryRestful1.json new file mode 100644 index 00000000..54d2589c --- /dev/null +++ b/tests/taosbenchmark/json/queryRestful1.json @@ -0,0 +1,24 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "rest", + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } + } + From 001d6b660d2e7cf6bdd80c576b9d726069721c56 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 20:06:59 +0800 Subject: [PATCH 27/49] queryRestful.json split two query json for python edit --- tests/taosbenchmark/taosdemoTestQueryWithJson.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/taosbenchmark/taosdemoTestQueryWithJson.py index e6ec2bf1..47a4be42 100644 --- a/tests/taosbenchmark/taosdemoTestQueryWithJson.py +++ b/tests/taosbenchmark/taosdemoTestQueryWithJson.py @@ -146,6 +146,7 @@ def run(self): # use restful api to query os.system("%s -f ./taosbenchmark/json/queryInsertrestdata.json" % binPath) os.system("%s -f ./taosbenchmark/json/queryRestful.json" % binPath) + os.system("%s -f ./taosbenchmark/json/queryRestful1.json" % binPath) os.system("cat query_res0.txt* > all_query_res0_rest.txt") os.system("cat query_res1.txt* > all_query_res1_rest.txt") os.system("cat query_res2.txt* > all_query_res2_rest.txt") From 01fc7e493a2f69bd023e2e9edd709445274ae3da Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 20:11:08 +0800 Subject: [PATCH 28/49] fix: querySpeciMutisql100.json remove supertable --- tests/taosbenchmark/json/querySpeciMutisql100.json | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/taosbenchmark/json/querySpeciMutisql100.json b/tests/taosbenchmark/json/querySpeciMutisql100.json index 5b523ad3..13010efd 100644 --- a/tests/taosbenchmark/json/querySpeciMutisql100.json +++ b/tests/taosbenchmark/json/querySpeciMutisql100.json @@ -413,17 +413,6 @@ "result": "./query_res0.txt" }] - }, - "super_table_query": { - "stblname": "stb1", - "query_interval": 1, - "threads": 3, - "sqls": [ - { - "sql": "select last_row(ts) from xxxx", - "result": "./query_res2.txt" - } - ] - } + } } \ No newline at end of file From 5781526424bffeed12d49ca1904df2393269c066 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 21:12:46 +0800 Subject: [PATCH 29/49] enh: implement delay avg code --- inc/bench.h | 3 +-- src/benchQuery.c | 34 ++++++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/inc/bench.h b/inc/bench.h index 86d28a04..d47bafba 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -866,12 +866,11 @@ typedef struct SQueryThreadInfo_S { int32_t threadID; BArray* query_delay_list; int32_t sockfd; - int64_t total_delay; + double total_delay; char filePath[MAX_PATH_LEN]; uint64_t start_table_from; uint64_t end_table_to; - double avg_delay; uint64_t ntables; uint64_t querySeq; diff --git a/src/benchQuery.c b/src/benchQuery.c index 7e8fbadb..787d96cb 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -183,7 +183,7 @@ static void *specQueryMixThread(void *sarg) { *delay = et - st; debugPrint("%s() LN%d, delay: %"PRId64"\n", __func__, __LINE__, *delay); - pThreadInfo->total_delay += (et - st); + pThreadInfo->total_delay += *delay; if(benchArrayPush(pThreadInfo->query_delay_list, delay) == NULL){ tmfree(delay); } @@ -264,6 +264,7 @@ static void *specQueryThread(void *sarg) { if (ret == 0) { // only succ add delay list benchArrayPushNoFree(pThreadInfo->query_delay_list, &delay); + pThreadInfo->total_delay += delay; } index++; @@ -353,6 +354,7 @@ static void *stbQueryThread(void *sarg) { if (ret == 0) { // only succ add delay list benchArrayPushNoFree(pThreadInfo->query_delay_list, &delay); + pThreadInfo->total_delay += delay; } // show real QPS @@ -381,7 +383,7 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // statistic BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); - int64_t total_delay = 0; + double total_delays = 0; g_queryInfo.specifiedQueryInfo.totalQueried = 0; g_queryInfo.specifiedQueryInfo.totalFail = 0; @@ -392,7 +394,7 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // append delay benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, pThreadInfo->query_delay_list->size, false); - total_delay += pThreadInfo->total_delay; + total_delays += pThreadInfo->total_delay; // free delay benchArrayDestroy(pThreadInfo->query_delay_list); @@ -400,6 +402,14 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { } + // succ is zero + if (delay_list->size == 0) { + errorPrint("succ queries count is zero.\n"); + benchArrayDestroy(delay_list); + return -1; + } + + // sort qsort(delay_list->pData, delay_list->size, delay_list->elemSize, compare); @@ -417,7 +427,7 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { spend/1E6, threadCnt, (int)delay_list->size, *(int64_t *)(benchArrayGet(delay_list, 0))/1E6, - (double)total_delay/delay_list->size/1E6, + (double)total_delays/delay_list->size/1E6, *(int64_t *)(benchArrayGet(delay_list, (int32_t)(delay_list->size * 0.9)))/1E6, *(int64_t *)(benchArrayGet(delay_list, @@ -635,8 +645,7 @@ static int specQuery(uint16_t iface, char* dbName) { // show QPS and P90 ... // uint64_t n = 0; - double delays = 0; - double avg_delay = 0.0; + double total_delays = 0.0; uint64_t totalQueried = 0; uint64_t totalFail = 0; for (int j = 0; j < threadCnt; j++) { @@ -646,7 +655,7 @@ static int specQuery(uint16_t iface, char* dbName) { for (uint64_t k = 0; k < pThreadInfo->query_delay_list->size; k++) { int64_t * delay = benchArrayGet(pThreadInfo->query_delay_list, k); sql->delay_list[n++] = *delay; - delays += *delay; + total_delays += *delay; } // total queries @@ -665,7 +674,12 @@ static int specQuery(uint16_t iface, char* dbName) { g_queryInfo.specifiedQueryInfo.totalQueried += totalQueried; g_queryInfo.specifiedQueryInfo.totalFail += totalFail; - avg_delay /= threadCnt; + // succ is zero + if(totalQueried == 0 || n == 0) { + errorPrint("succ queries count is zero.\n"); + goto OVER; + } + qsort(sql->delay_list, n, sizeof(uint64_t), compare); int32_t bufLen = strlen(sql->command) + 512; char * buf = benchCalloc(bufLen, sizeof(char), false); @@ -681,7 +695,7 @@ static int specQuery(uint16_t iface, char* dbName) { "SQL command: %s \n", threadCnt, totalQueried, i + 1, spend/1E6, totalQueried / (spend/1E6), - avg_delay / 1E6, /* avg */ + total_delays/n/1E6, /* avg */ sql->delay_list[0] / 1E6, /* min */ sql->delay_list[n - 1] / 1E6, /* max */ /* p90 */ @@ -694,7 +708,7 @@ static int specQuery(uint16_t iface, char* dbName) { infoPrintNoTimestamp("%s", buf); infoPrintNoTimestampToFile("%s", buf); - tmfree(buf); + tmfree(buf); } OVER: From 9093e531dd51bdb7e74e3b88792e7042c36e5a7d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 21:16:23 +0800 Subject: [PATCH 30/49] fix: build error for errorPrint --- src/benchQuery.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 787d96cb..721343aa 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -404,9 +404,9 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // succ is zero if (delay_list->size == 0) { - errorPrint("succ queries count is zero.\n"); + errorPrint("%s", "succ queries count is zero.\n"); benchArrayDestroy(delay_list); - return -1; + return ; } @@ -676,7 +676,7 @@ static int specQuery(uint16_t iface, char* dbName) { // succ is zero if(totalQueried == 0 || n == 0) { - errorPrint("succ queries count is zero.\n"); + errorPrint("%s", "succ queries count is zero.\n"); goto OVER; } From de9db46d97eb7c57c21f2d688c63c8b5d97bb511 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 21:25:20 +0800 Subject: [PATCH 31/49] fix: return value change with code --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 721343aa..633e975d 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -639,7 +639,6 @@ static int specQuery(uint16_t iface, char* dbName) { errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); goto OVER; } - ret = 0; // // show QPS and P90 ... @@ -710,6 +709,7 @@ static int specQuery(uint16_t iface, char* dbName) { infoPrintNoTimestampToFile("%s", buf); tmfree(buf); } + ret = 0; OVER: tmfree((char *)pids); From 3040e6b3f8498fd38047db3932ca7de6d42f37ab Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 20 Jan 2025 21:31:04 +0800 Subject: [PATCH 32/49] fix: query qps case passed --- tests/taosbenchmark/json/queryQps.json | 13 +---------- tests/taosbenchmark/json/queryQps1.json | 22 +++++++++++++++++++ .../taosdemoTestQueryWithJson.py | 2 ++ 3 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 tests/taosbenchmark/json/queryQps1.json diff --git a/tests/taosbenchmark/json/queryQps.json b/tests/taosbenchmark/json/queryQps.json index 02661fe9..21b4cf6f 100644 --- a/tests/taosbenchmark/json/queryQps.json +++ b/tests/taosbenchmark/json/queryQps.json @@ -20,16 +20,5 @@ "sql": "select last_row(*) from db.stb00_9 ", "result": "./query_res1.txt" }] - }, - "super_table_query": { - "stblname": "stb1", - "query_interval":20, - "threads": 4, - "sqls": [ - { - "sql": "select last_row(ts) from xxxx", - "result": "./query_res2.txt" - } - ] - } + } } diff --git a/tests/taosbenchmark/json/queryQps1.json b/tests/taosbenchmark/json/queryQps1.json new file mode 100644 index 00000000..c2c23817 --- /dev/null +++ b/tests/taosbenchmark/json/queryQps1.json @@ -0,0 +1,22 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 1, + "super_table_query": { + "stblname": "stb1", + "query_interval":20, + "threads": 4, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } + } diff --git a/tests/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/taosbenchmark/taosdemoTestQueryWithJson.py index 47a4be42..61b823b2 100644 --- a/tests/taosbenchmark/taosdemoTestQueryWithJson.py +++ b/tests/taosbenchmark/taosdemoTestQueryWithJson.py @@ -195,6 +195,8 @@ def run(self): os.system("%s -f ./taosbenchmark/json/queryInsertdata.json" % binPath) exceptcode = os.system("%s -f ./taosbenchmark/json/queryQps.json" % binPath) assert exceptcode == 0 + exceptcode = os.system("%s -f ./taosbenchmark/json/queryQps1.json" % binPath) + assert exceptcode == 0 # 2021.02.09 need modify taosBenchmakr code # use illegal or out of range parameters query json file From 73638d7850205ce3ae7d1798db9dc96906852ce6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 09:37:29 +0800 Subject: [PATCH 33/49] fix: add mixed query total reset to zero --- src/benchQuery.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 633e975d..0a777dad 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -384,8 +384,6 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // statistic BArray * delay_list = benchArrayInit(1, sizeof(int64_t)); double total_delays = 0; - g_queryInfo.specifiedQueryInfo.totalQueried = 0; - g_queryInfo.specifiedQueryInfo.totalFail = 0; // clear for (int i = 0; i < threadCnt; ++i) { @@ -510,6 +508,10 @@ static int stbQuery(uint16_t iface, char* dbName) { goto OVER; } + // reset total + g_queryInfo.superQueryInfo.totalQueried = 0; + g_queryInfo.superQueryInfo.totalFail = 0; + // real thread count g_queryInfo.superQueryInfo.threadCnt = threadCnt; int64_t start = toolsGetTimestampUs(); @@ -776,6 +778,10 @@ static int specQueryMix(uint16_t iface, char* dbName) { g_arguments->terminate = true; } + // reset total + g_queryInfo.specifiedQueryInfo.totalQueried = 0; + g_queryInfo.specifiedQueryInfo.totalFail = 0; + int64_t start = toolsGetTimestampUs(); for (int i = 0; i < threadCnt; ++i) { pthread_join(pids[i], NULL); @@ -820,18 +826,6 @@ static int specQueryMix(uint16_t iface, char* dbName) { // total query for end void totalQuery(int64_t spends) { - // specifiedQuery - if (g_queryInfo.specifiedQueryInfo.totalQueried) { - infoPrint("Total specified queries: %" PRIu64 "\n", - g_queryInfo.specifiedQueryInfo.totalQueried); - } - - // superQuery - if (g_queryInfo.superQueryInfo.totalQueried) { - infoPrint("Total super queries: %" PRIu64 "\n", - g_queryInfo.superQueryInfo.totalQueried); - } - // total QPS uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried + g_queryInfo.superQueryInfo.totalQueried; @@ -841,7 +835,7 @@ void totalQuery(int64_t spends) { if(g_arguments->continueIfFail == YES_IF_FAILED) { uint16_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; if (totalQueried > 0) { - snprintf(errRate, sizeof(errRate), " Error Rate:%.3f%%", ((float)totalFail * 100)/totalQueried); + snprintf(errRate, sizeof(errRate), " Error %“ PRIu64 ” Rate:%.3f%%", totalFail, ((float)totalFail * 100)/totalQueried); } } From ae6dd19f9027dd6db0fc10f81e20ee5b54b701d3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 09:39:42 +0800 Subject: [PATCH 34/49] fix: code have chinese char --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 0a777dad..fd8f71b6 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -835,7 +835,7 @@ void totalQuery(int64_t spends) { if(g_arguments->continueIfFail == YES_IF_FAILED) { uint16_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; if (totalQueried > 0) { - snprintf(errRate, sizeof(errRate), " Error %“ PRIu64 ” Rate:%.3f%%", totalFail, ((float)totalFail * 100)/totalQueried); + snprintf(errRate, sizeof(errRate), " Error %" PRIu64 " Rate:%.3f%%", totalFail, ((float)totalFail * 100)/totalQueried); } } From f559aee013702b2bdf011e3e1d189d5f2ff914a9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 09:42:24 +0800 Subject: [PATCH 35/49] fix: uint64 mistake write uint16 --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index fd8f71b6..fe48ceeb 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -833,7 +833,7 @@ void totalQuery(int64_t spends) { // error rate char errRate[128] = ""; if(g_arguments->continueIfFail == YES_IF_FAILED) { - uint16_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; + uint64_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; if (totalQueried > 0) { snprintf(errRate, sizeof(errRate), " Error %" PRIu64 " Rate:%.3f%%", totalFail, ((float)totalFail * 100)/totalQueried); } From 72db52f940f38c872eaf2a22b03bdc5ee8f42e07 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 10:06:22 +0800 Subject: [PATCH 36/49] fix: split two query mode in one json --- tests/taosbenchmark/json/TD-32846.json | 4 ++-- tests/taosbenchmark/json/taosc_query.json | 11 ---------- tests/taosbenchmark/json/taosc_query1.json | 24 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 tests/taosbenchmark/json/taosc_query1.json diff --git a/tests/taosbenchmark/json/TD-32846.json b/tests/taosbenchmark/json/TD-32846.json index cd8b87e4..309ba40f 100644 --- a/tests/taosbenchmark/json/TD-32846.json +++ b/tests/taosbenchmark/json/TD-32846.json @@ -29,7 +29,7 @@ "super_tables": [{ "name": "product", "child_table_exists": "no", - "childtable_count": 9, + "childtable_count": 5, "childtable_prefix": "d", "auto_create_table": "yes", "batch_create_tbl_num": 10, @@ -37,7 +37,7 @@ "insert_mode": "stmt2", "non_stop_mode": "no", "line_protocol": "line", - "insert_rows": 10340, + "insert_rows": 1034, "childtable_limit": 0, "childtable_offset": 0, "interlace_rows": 0, diff --git a/tests/taosbenchmark/json/taosc_query.json b/tests/taosbenchmark/json/taosc_query.json index c8ff2e92..2cf8f648 100644 --- a/tests/taosbenchmark/json/taosc_query.json +++ b/tests/taosbenchmark/json/taosc_query.json @@ -18,16 +18,5 @@ "sql": "select count(*) from db.stb", "result": "taosc_query_specified" }] - }, - "super_table_query": { - "stblname": "stb", - "query_interval": 1, - "concurrent": 1, - "sqls": [ - { - "sql": "select count(*) from xxxx", - "result": "taosc_query_super" - } - ] } } \ No newline at end of file diff --git a/tests/taosbenchmark/json/taosc_query1.json b/tests/taosbenchmark/json/taosc_query1.json new file mode 100644 index 00000000..fd650f90 --- /dev/null +++ b/tests/taosbenchmark/json/taosc_query1.json @@ -0,0 +1,24 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 1, + "reset_query_cache": "yes", + "super_table_query": + { + "stblname": "stb", + "query_interval": 1, + "concurrent": 1, + "sqls": [ + { + "sql": "select count(*) from xxxx", + "result": "taosc_query_super" + } + ] + } +} \ No newline at end of file From 6a30aabc18478cb345779b65a27b03a175c32945 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 10:14:55 +0800 Subject: [PATCH 37/49] fix: query json split two json --- tests/taosbenchmark/query_json.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/taosbenchmark/query_json.py b/tests/taosbenchmark/query_json.py index edc43237..64c238e4 100644 --- a/tests/taosbenchmark/query_json.py +++ b/tests/taosbenchmark/query_json.py @@ -74,6 +74,10 @@ def run(self): cmd = "%s -f ./taosbenchmark/json/taosc_query.json" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) + cmd = "%s -f ./taosbenchmark/json/taosc_query1.json" % binPath + tdLog.info("%s" % cmd) + os.system("%s" % cmd) + with open("%s" % "taosc_query_specified-0", "r+") as f1: for line in f1.readlines(): queryTaosc = line.strip().split()[0] From 70af459d8afaee940f8214053b9f42c7ba379561 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 10:23:12 +0800 Subject: [PATCH 38/49] fix: split rest_query.json two --- tests/taosbenchmark/json/rest_query.json | 9 --------- tests/taosbenchmark/json/rest_query1.json | 18 ++++++++++++++++++ tests/taosbenchmark/query_json.py | 4 ++++ 3 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 tests/taosbenchmark/json/rest_query1.json diff --git a/tests/taosbenchmark/json/rest_query.json b/tests/taosbenchmark/json/rest_query.json index 459e496f..817d7332 100644 --- a/tests/taosbenchmark/json/rest_query.json +++ b/tests/taosbenchmark/json/rest_query.json @@ -14,14 +14,5 @@ "sql": "select count(*) from db.stb", "result": "rest_query_specified" }] - }, - "super_table_query": { - "stblname": "stb", - "sqls": [ - { - "sql": "select count(*) from xxxx", - "result": "rest_query_super" - } - ] } } \ No newline at end of file diff --git a/tests/taosbenchmark/json/rest_query1.json b/tests/taosbenchmark/json/rest_query1.json new file mode 100644 index 00000000..e0911273 --- /dev/null +++ b/tests/taosbenchmark/json/rest_query1.json @@ -0,0 +1,18 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_mode": "rest", + "connection_pool_size": 10, + "response_buffer": 10000, + "super_table_query": { + "stblname": "stb", + "sqls": [ + { + "sql": "select count(*) from xxxx", + "result": "rest_query_super" + } + ] + } +} \ No newline at end of file diff --git a/tests/taosbenchmark/query_json.py b/tests/taosbenchmark/query_json.py index 64c238e4..29686927 100644 --- a/tests/taosbenchmark/query_json.py +++ b/tests/taosbenchmark/query_json.py @@ -88,9 +88,13 @@ def run(self): queryTaosc = line.strip().split()[0] assert queryTaosc == "1", "result is %s != expect: 1" % queryTaosc + # split two cmd = "%s -f ./taosbenchmark/json/rest_query.json" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) + cmd = "%s -f ./taosbenchmark/json/rest_query1.json" % binPath + tdLog.info("%s" % cmd) + os.system("%s" % cmd) times = 0 with open("rest_query_super-0", "r+") as f1: From 09a9f001c388d0cff6bd065a43e491c3a2143ef9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 10:43:22 +0800 Subject: [PATCH 39/49] fix: mixed query test case --- .../json/queryTaosc-mixed-query.json | 11 --------- .../json/queryTaosc-mixed-query1.json | 23 +++++++++++++++++++ .../taosdemoTestQueryWithJson-mixed-query.py | 2 ++ 3 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 tests/taosbenchmark/json/queryTaosc-mixed-query1.json diff --git a/tests/taosbenchmark/json/queryTaosc-mixed-query.json b/tests/taosbenchmark/json/queryTaosc-mixed-query.json index 9b2f1b46..9cdc191d 100644 --- a/tests/taosbenchmark/json/queryTaosc-mixed-query.json +++ b/tests/taosbenchmark/json/queryTaosc-mixed-query.json @@ -23,16 +23,5 @@ "result": "./query_res1.txt" } ] - }, - "super_table_query": { - "stblname": "stb1", - "query_interval": 1, - "threads": 3, - "sqls": [ - { - "sql": "select last_row(ts) from xxxx", - "result": "./query_res2.txt" - } - ] } } diff --git a/tests/taosbenchmark/json/queryTaosc-mixed-query1.json b/tests/taosbenchmark/json/queryTaosc-mixed-query1.json new file mode 100644 index 00000000..a3caa1c5 --- /dev/null +++ b/tests/taosbenchmark/json/queryTaosc-mixed-query1.json @@ -0,0 +1,23 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "taosc", + "super_table_query": { + "stblname": "stb1", + "query_interval": 1, + "threads": 3, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/v3/taosdemoTestQueryWithJson-mixed-query.py b/tests/taosbenchmark/v3/taosdemoTestQueryWithJson-mixed-query.py index 0a4a14e1..c7b7fc92 100644 --- a/tests/taosbenchmark/v3/taosdemoTestQueryWithJson-mixed-query.py +++ b/tests/taosbenchmark/v3/taosdemoTestQueryWithJson-mixed-query.py @@ -119,6 +119,7 @@ def run(self): # taosc query: query specified table and query super table os.system("%s -f ./taosbenchmark/json/queryInsertdata.json" % binPath) os.system("%s -f ./taosbenchmark/json/queryTaosc-mixed-query.json" % binPath) + os.system("%s -f ./taosbenchmark/json/queryTaosc-mixed-query1.json" % binPath) os.system("cat query_res2.txt* > all_query_res2_taosc.txt") # correct Times testcases @@ -136,6 +137,7 @@ def run(self): # use restful api to query os.system("%s -f ./taosbenchmark/json/queryInsertrestdata.json" % binPath) os.system("%s -f ./taosbenchmark/json/queryRestful.json" % binPath) + os.system("%s -f ./taosbenchmark/json/queryRestful1.json" % binPath) os.system("cat query_res2.txt* > all_query_res2_rest.txt") # correct Times testcases From abad4de953a5bcc207c360b483a0c510848a073b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Jan 2025 11:35:31 +0800 Subject: [PATCH 40/49] fix: check delay_list pointer --- src/benchQuery.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index fe48ceeb..82db3075 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -388,6 +388,9 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { // clear for (int i = 0; i < threadCnt; ++i) { qThreadInfo * pThreadInfo = infos + i; + if(pThreadInfo->query_delay_list == NULL) { + continue;; + } // append delay benchArrayAddBatch(delay_list, pThreadInfo->query_delay_list->pData, @@ -651,6 +654,9 @@ static int specQuery(uint16_t iface, char* dbName) { uint64_t totalFail = 0; for (int j = 0; j < threadCnt; j++) { qThreadInfo *pThreadInfo = infos + j; + if(pThreadInfo->query_delay_list == NULL) { + continue;; + } // total one sql for (uint64_t k = 0; k < pThreadInfo->query_delay_list->size; k++) { @@ -889,22 +895,29 @@ int queryTestProcess() { // start running // - // specified table + uint64_t startTs = toolsGetTimestampMs(); - if (g_queryInfo.specifiedQueryInfo.mixed_query) { - // mixed - if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { - return -1; + if(g_queryInfo.specifiedQueryInfo.sqls && g_queryInfo.specifiedQueryInfo.sqls->size > 0) { + // specified table + if (g_queryInfo.specifiedQueryInfo.mixed_query) { + // mixed + if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { + return -1; + } + } else { + // no mixied + if (specQuery(g_queryInfo.iface, g_queryInfo.dbName)) { + return -1; + } } - } else { - // no mixied - if (specQuery(g_queryInfo.iface, g_queryInfo.dbName)) { + } else if(g_queryInfo.superQueryInfo.sqlCount > 0) { + // super table + if (stbQuery(g_queryInfo.iface, g_queryInfo.dbName)) { return -1; } - } - - // super table - if (stbQuery(g_queryInfo.iface, g_queryInfo.dbName)) { + } else { + // nothing + errorPrint("%s\n", "Both 'specified_table_query' and 'super_table_query' sqls is empty."); return -1; } From f79883f4d7febd3570ee6151242efda4f8224db5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 Jan 2025 17:10:42 +0800 Subject: [PATCH 41/49] enh: add main case for query and except check case --- src/benchJsonOpt.c | 2 + src/benchQuery.c | 21 +- tests/taosbenchmark/json/queryError1.json | 12 + tests/taosbenchmark/json/queryError2.json | 35 +++ tests/taosbenchmark/json/queryError3.json | 27 ++ tests/taosbenchmark/json/queryMode1.json | 24 ++ tests/taosbenchmark/json/queryMode1Rest.json | 27 ++ tests/taosbenchmark/json/queryMode2.json | 25 ++ tests/taosbenchmark/json/queryMode2Rest.json | 28 ++ tests/taosbenchmark/json/queryMode3.json | 27 ++ tests/taosbenchmark/json/queryMode3Rest.json | 24 ++ tests/taosbenchmark/queryMain.py | 288 +++++++++++++++++++ 12 files changed, 530 insertions(+), 10 deletions(-) create mode 100644 tests/taosbenchmark/json/queryError1.json create mode 100644 tests/taosbenchmark/json/queryError2.json create mode 100644 tests/taosbenchmark/json/queryError3.json create mode 100644 tests/taosbenchmark/json/queryMode1.json create mode 100644 tests/taosbenchmark/json/queryMode1Rest.json create mode 100644 tests/taosbenchmark/json/queryMode2.json create mode 100644 tests/taosbenchmark/json/queryMode2Rest.json create mode 100644 tests/taosbenchmark/json/queryMode3.json create mode 100644 tests/taosbenchmark/json/queryMode3Rest.json create mode 100644 tests/taosbenchmark/queryMain.py diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 174be89e..1adfdb48 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1797,8 +1797,10 @@ int32_t readSpecQueryJson(tools_cJSON * specifiedQuery) { if (tools_cJSON_IsString(mixedQueryObj)) { if (0 == strcasecmp(mixedQueryObj->valuestring, "yes")) { g_queryInfo.specifiedQueryInfo.mixed_query = true; + infoPrint("%s\n","mixed_query is True"); } else if (0 == strcasecmp(mixedQueryObj->valuestring, "no")) { g_queryInfo.specifiedQueryInfo.mixed_query = false; + infoPrint("%s\n","mixed_query is False"); } else { errorPrint("Invalid mixed_query value: %s\n", mixedQueryObj->valuestring); diff --git a/src/benchQuery.c b/src/benchQuery.c index 82db3075..9c60c485 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -13,8 +13,8 @@ #include #include "benchLog.h" -// query and get result other true is no test sql -int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { +// query and get result record is true to total request +int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool record) { int ret = 0; // user cancel @@ -49,7 +49,7 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { ret = -1; } else { // succ query - if (!other) + if (record) rows = fetchResult(res, pThreadInfo->filePath); } @@ -63,11 +63,11 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool other) { // record count if (ret ==0) { // succ - if (!other) + if (record) pThreadInfo->nSucc ++; } else { // fail - if (!other) + if (record) pThreadInfo->nFail ++; // continue option @@ -92,7 +92,7 @@ void autoSleep(uint64_t st, uint64_t et) { // reset int32_t resetQueryCache(qThreadInfo* pThreadInfo) { // execute sql - if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", true)) { + if (selectAndGetResult(pThreadInfo, "RESET QUERY CACHE", false)) { errorPrint("%s() LN%d, reset query cache failed\n", __func__, __LINE__); return -1; } @@ -166,7 +166,7 @@ static void *specQueryMixThread(void *sarg) { // execute sql st = toolsGetTimestampUs(); - int ret = selectAndGetResult(pThreadInfo, sql->command, false); + int ret = selectAndGetResult(pThreadInfo, sql->command, true); if (ret) { g_fail = true; errorPrint("failed call mix selectAndGetResult, i=%d j=%d", i, j); @@ -247,7 +247,7 @@ static void *specQueryThread(void *sarg) { // execute sql st = toolsGetTimestampUs(); - int ret = selectAndGetResult(pThreadInfo, sql->command, false); + int ret = selectAndGetResult(pThreadInfo, sql->command, true); if (ret) { g_fail = true; errorPrint("failed call spec selectAndGetResult, index=%d\n", index); @@ -341,7 +341,7 @@ static void *stbQueryThread(void *sarg) { // execute sql uint64_t s = toolsGetTimestampUs(); - int ret = selectAndGetResult(pThreadInfo, sqlstr, false); + int ret = selectAndGetResult(pThreadInfo, sqlstr, true); if (ret) { // found error errorPrint("failed call stb selectAndGetResult, i=%d j=%d\n", i, j); @@ -745,6 +745,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { if (a < 1) { nConcurrent = total_sql_num; a = 1; + warnPrint("sqls num:%d < concuurent:%d, so set concurrent to %d\n", total_sql_num, nConcurrent, nConcurrent); } int b = 0; if (nConcurrent != 0) { @@ -841,7 +842,7 @@ void totalQuery(int64_t spends) { if(g_arguments->continueIfFail == YES_IF_FAILED) { uint64_t totalFail = g_queryInfo.specifiedQueryInfo.totalFail + g_queryInfo.superQueryInfo.totalFail; if (totalQueried > 0) { - snprintf(errRate, sizeof(errRate), " Error %" PRIu64 " Rate:%.3f%%", totalFail, ((float)totalFail * 100)/totalQueried); + snprintf(errRate, sizeof(errRate), " ,error %" PRIu64 " (rate:%.3f%%)", totalFail, ((float)totalFail * 100)/totalQueried); } } diff --git a/tests/taosbenchmark/json/queryError1.json b/tests/taosbenchmark/json/queryError1.json new file mode 100644 index 00000000..71dc16d5 --- /dev/null +++ b/tests/taosbenchmark/json/queryError1.json @@ -0,0 +1,12 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "taosc" +} diff --git a/tests/taosbenchmark/json/queryError2.json b/tests/taosbenchmark/json/queryError2.json new file mode 100644 index 00000000..48b9de49 --- /dev/null +++ b/tests/taosbenchmark/json/queryError2.json @@ -0,0 +1,35 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 3, + "sqls": [ + { + "sql": "select last_row(*) from meters" + }, + { + "sql": "select count(*) from d0", + "result": "./query_res1.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "concurrent": 3, + "query_interval": 1, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryError3.json b/tests/taosbenchmark/json/queryError3.json new file mode 100644 index 00000000..07bfd6ab --- /dev/null +++ b/tests/taosbenchmark/json/queryError3.json @@ -0,0 +1,27 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "test", + "query_times": 100, + "query_mode": "taosc", + "super_table_query": + "stblname": "meters", + "concurrent": 3, + "query_interval": 1, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + }, + { + "sql": "select count(ts) from xxxx" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode1.json b/tests/taosbenchmark/json/queryMode1.json new file mode 100644 index 00000000..533cb05c --- /dev/null +++ b/tests/taosbenchmark/json/queryMode1.json @@ -0,0 +1,24 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 3, + "sqls": [ + { + "sql": "select last_row(*) from meters" + }, + { + "sql": "select count(*) from d0", + "result": "./query_res1.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode1Rest.json b/tests/taosbenchmark/json/queryMode1Rest.json new file mode 100644 index 00000000..5b00ee64 --- /dev/null +++ b/tests/taosbenchmark/json/queryMode1Rest.json @@ -0,0 +1,27 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "rest", + "specified_table_query": { + "concurrent": 3, + "sqls": [ + { + "sql": "select last_row(*) from meters" + }, + { + "sql": "select count(*) from d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from meters" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode2.json b/tests/taosbenchmark/json/queryMode2.json new file mode 100644 index 00000000..39a9593d --- /dev/null +++ b/tests/taosbenchmark/json/queryMode2.json @@ -0,0 +1,25 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 4, + "mixed_query": "yes", + "sqls": [ + { + "sql": "select last_row(*) from meters" + }, + { + "sql": "select count(*) from d0", + "result": "./query_res1.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode2Rest.json b/tests/taosbenchmark/json/queryMode2Rest.json new file mode 100644 index 00000000..5d962f13 --- /dev/null +++ b/tests/taosbenchmark/json/queryMode2Rest.json @@ -0,0 +1,28 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 100, + "query_mode": "rest", + "specified_table_query": { + "concurrent": 3, + "mixed_query": "yes", + "sqls": [ + { + "sql": "select last_row(*) from meters" + }, + { + "sql": "select count(*) from d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from meters" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode3.json b/tests/taosbenchmark/json/queryMode3.json new file mode 100644 index 00000000..c01c588c --- /dev/null +++ b/tests/taosbenchmark/json/queryMode3.json @@ -0,0 +1,27 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "test", + "query_times": 100, + "query_mode": "taosc", + "super_table_query": { + "stblname": "meters", + "concurrent": 3, + "query_interval": 1, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + }, + { + "sql": "select count(ts) from xxxx" + } + ] + } +} diff --git a/tests/taosbenchmark/json/queryMode3Rest.json b/tests/taosbenchmark/json/queryMode3Rest.json new file mode 100644 index 00000000..59365694 --- /dev/null +++ b/tests/taosbenchmark/json/queryMode3Rest.json @@ -0,0 +1,24 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "test", + "query_times": 100, + "query_mode": "rest", + "super_table_query": { + "stblname": "meters", + "concurrent": 3, + "query_interval": 1, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "./query_res2.txt" + } + ] + } +} diff --git a/tests/taosbenchmark/queryMain.py b/tests/taosbenchmark/queryMain.py new file mode 100644 index 00000000..e5d0a6a3 --- /dev/null +++ b/tests/taosbenchmark/queryMain.py @@ -0,0 +1,288 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import json +import sys +import os +import time +import datetime +import platform +import subprocess + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +from util.dnodes import tdDnodes + + +# reomve single and double quotation +def removeQuotation(origin): + value = "" + for c in origin: + if c != '\'' and c != '"': + value += c + + return value + +class TDTestCase: + def caseDescription(self): + """ + [TD-11510] taosBenchmark test cases + """ + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getPath(self, tool="taosBenchmark"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] + else: + projPath = selfPath[: selfPath.find("tests")] + + paths = [] + for root, dummy, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def runSeconds(self, command, timeout = 180): + tdLog.info(f"runSeconds {command} ...") + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(timeout) + + # get output + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + return output, error + + def getKeyValue(self, content, key, end): + # find key + s = content.find(key) + if s == -1: + return False,"" + + # skip self + s += len(key) + # skip blank + while s < len(content): + if content[s] != " ": + break + s += 1 + + # end check + if s + 1 == len(content): + return False, "" + + # find end + if len(end) == 0: + e = -1 + else: + e = content.find(end, s) + + # get value + if e == -1: + value = content[s : ] + else: + value = content[s : e] + + return True, value + + def getDbRows(self, times): + sql = f"select count(*) from test.meters" + tdSql.waitedQuery(sql, 1, times) + dbRows = tdSql.getData(0, 0) + return dbRows + + def checkItem(self, output, key, end, expect, equal): + ret, value = self.getKeyValue(output, key, end) + if ret == False: + tdLog.exit(f"not found key:{key}. end:{end} output:\n{output}") + + fval = float(value) + # compare + if equal and fval != expect: + tdLog.exit(f"check not expect. expect:{expect} real:{fval}, key:{key} end:{end} output:\n{output}") + elif equal == False and fval <= expect: + tdLog.exit(f"failed because {fval} <= {expect}, key:{key} end:{end} output:\n{output}") + else: + # succ + if equal: + tdLog.info(f"check successfully. key:{key} expect:{expect} real:{fval}") + else: + tdLog.info(f"check successfully. key:{key} {fval} > {expect}") + + + def checkAfterRun(self, benchmark, jsonFile, specMode, tbCnt): + # run + cmd = f"{benchmark} -f {jsonFile}" + output, error = self.runSeconds(cmd) + + if specMode : + label = "specified_table_query" + else: + label = "super_table_query" + + # + # check insert result + # + with open(jsonFile, "r") as file: + data = json.load(file) + + queryTimes = data["query_times"] + # contineIfFail + try: + continueIfFail = data["continue_if_fail"] + except: + continueIfFail = "no" + + concurrent = data[label]["concurrent"] + sqls = data[label]["sqls"] + + + # mix + try: + mixedQuery = data[label]["mixed_query"] + except: + mixedQuery = "no" + + tdLog.info(f"queryTimes={queryTimes} concurrent={concurrent} mixedQuery={mixedQuery} len(sqls)={len(sqls)} label={label}\n") + + totalQueries = 0 + threadQueries = 0 + + if continueIfFail.lower() == "yes": + allEnd = " " + else: + allEnd = "\n" + + if specMode and mixedQuery.lower() != "yes": + # spec + threadQueries = queryTimes * concurrent + totalQueries = queryTimes * concurrent * len(sqls) + threadKey = f"complete query with {concurrent} threads and " + qpsKey = "QPS: " + avgKey = "query delay avg: " + minKey = "min:" + else: + # spec mixed or super + if specMode: + # spec + totalQueries = queryTimes * len(sqls) + else: + # super + totalQueries = queryTimes * len(sqls) * tbCnt + threadQueries = totalQueries + + nSql = len(sqls) + if specMode and nSql < concurrent : + tdLog.info(f"set concurrent = {nSql} because len(sqls) < concurrent") + concurrent = nSql + threadKey = f"using {concurrent} threads complete query " + qpsKey = "" + avgKey = "avg delay:" + minKey = "min delay:" + + items = [ + [threadKey, " ", threadQueries, True], + [qpsKey, " ", 5, False], # qps need > 1 + [avgKey, "s", 0, False], + [minKey, "s", 0, False], + ["max: ", "s", 0, False], + ["p90: ", "s", 0, False], + ["p95: ", "s", 0, False], + ["p99: ", "s", 0, False], + ["INFO: Spend ", " ", 0, False], + ["completed total queries: ", ",", totalQueries, True], + ["the QPS of all threads:", allEnd, 10 , False] # all qps need > 5 + ] + + # check + for item in items: + if len(item[0]) > 0: + self.checkItem(output, item[0], item[1], item[2], item[3]) + + # native + def threeQueryMode(self, benchmark, tbCnt, tbRow): + # json + args = [ + ["taosbenchmark/json/queryMode1", True], + ["taosbenchmark/json/queryMode2", True], + ["taosbenchmark/json/queryMode3", False] + ] + + # native + for arg in args: + self.checkAfterRun(benchmark, arg[0] + ".json", arg[1], tbCnt) + + # rest + for arg in args: + self.checkAfterRun(benchmark, arg[0] + "Rest.json", arg[1], tbCnt) + + def expectFailed(self, command): + ret = os.system(command) + if ret == 0: + tdLog.exit(f" expect failed but success. command={command}") + else: + tdLog.info(f" expect failed is ok. command={command}") + + + # check excption + def exceptTest(self, benchmark, tbCnt, tbRow): + # 'specified_table_query' and 'super_table_query' error + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError1.json") + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError2.json") + # json format error + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError3.json") + + + + def run(self): + tbCnt = 10 + tbRow = 1000 + benchmark = self.getPath() + + # insert + command = f"{benchmark} -d test -t {tbCnt} -n {tbRow} -I stmt2 -r 100 -y" + ret = os.system(command) + if ret !=0 : + tdLog.exit(f"exec failed. command={command}") + + # query mode test + self.threeQueryMode(benchmark, tbCnt, tbRow) + + # exception test + self.exceptTest(benchmark, tbCnt, tbRow); + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 2cfc9a3753a8a2a4451bfaca27f1f12c9e26f351 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 22 Jan 2025 18:22:34 +0800 Subject: [PATCH 42/49] fix: modify position for concurrent change --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 9c60c485..a89758fd 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -743,9 +743,9 @@ static int specQueryMix(uint16_t iface, char* dbName) { int start_sql = 0; int a = total_sql_num / nConcurrent; if (a < 1) { + warnPrint("sqls num:%d < concurent:%d, so set concurrent to %d\n", total_sql_num, nConcurrent, nConcurrent); nConcurrent = total_sql_num; a = 1; - warnPrint("sqls num:%d < concuurent:%d, so set concurrent to %d\n", total_sql_num, nConcurrent, nConcurrent); } int b = 0; if (nConcurrent != 0) { From a5f7d33e5d430bb89ee6b6ccfb59b97ab8cff559 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 19:29:08 +0800 Subject: [PATCH 43/49] fix: change case name with fullname --- ...queryError2.json => queryErrorBothSpecSuper.json} | 0 .../json/{queryError3.json => queryErrorFormat.json} | 0 .../{queryError1.json => queryErrorNoSpecSuper.json} | 0 .../json/{queryMode1.json => queryModeSpec.json} | 0 .../json/{queryMode2.json => queryModeSpecMix.json} | 0 ...queryMode2Rest.json => queryModeSpecMixRest.json} | 0 .../{queryMode1Rest.json => queryModeSpecRest.json} | 0 .../json/{queryMode3.json => queryModeSuper.json} | 0 .../{queryMode3Rest.json => queryModeSuperRest.json} | 0 tests/taosbenchmark/queryMain.py | 12 ++++++------ 10 files changed, 6 insertions(+), 6 deletions(-) rename tests/taosbenchmark/json/{queryError2.json => queryErrorBothSpecSuper.json} (100%) rename tests/taosbenchmark/json/{queryError3.json => queryErrorFormat.json} (100%) rename tests/taosbenchmark/json/{queryError1.json => queryErrorNoSpecSuper.json} (100%) rename tests/taosbenchmark/json/{queryMode1.json => queryModeSpec.json} (100%) rename tests/taosbenchmark/json/{queryMode2.json => queryModeSpecMix.json} (100%) rename tests/taosbenchmark/json/{queryMode2Rest.json => queryModeSpecMixRest.json} (100%) rename tests/taosbenchmark/json/{queryMode1Rest.json => queryModeSpecRest.json} (100%) rename tests/taosbenchmark/json/{queryMode3.json => queryModeSuper.json} (100%) rename tests/taosbenchmark/json/{queryMode3Rest.json => queryModeSuperRest.json} (100%) diff --git a/tests/taosbenchmark/json/queryError2.json b/tests/taosbenchmark/json/queryErrorBothSpecSuper.json similarity index 100% rename from tests/taosbenchmark/json/queryError2.json rename to tests/taosbenchmark/json/queryErrorBothSpecSuper.json diff --git a/tests/taosbenchmark/json/queryError3.json b/tests/taosbenchmark/json/queryErrorFormat.json similarity index 100% rename from tests/taosbenchmark/json/queryError3.json rename to tests/taosbenchmark/json/queryErrorFormat.json diff --git a/tests/taosbenchmark/json/queryError1.json b/tests/taosbenchmark/json/queryErrorNoSpecSuper.json similarity index 100% rename from tests/taosbenchmark/json/queryError1.json rename to tests/taosbenchmark/json/queryErrorNoSpecSuper.json diff --git a/tests/taosbenchmark/json/queryMode1.json b/tests/taosbenchmark/json/queryModeSpec.json similarity index 100% rename from tests/taosbenchmark/json/queryMode1.json rename to tests/taosbenchmark/json/queryModeSpec.json diff --git a/tests/taosbenchmark/json/queryMode2.json b/tests/taosbenchmark/json/queryModeSpecMix.json similarity index 100% rename from tests/taosbenchmark/json/queryMode2.json rename to tests/taosbenchmark/json/queryModeSpecMix.json diff --git a/tests/taosbenchmark/json/queryMode2Rest.json b/tests/taosbenchmark/json/queryModeSpecMixRest.json similarity index 100% rename from tests/taosbenchmark/json/queryMode2Rest.json rename to tests/taosbenchmark/json/queryModeSpecMixRest.json diff --git a/tests/taosbenchmark/json/queryMode1Rest.json b/tests/taosbenchmark/json/queryModeSpecRest.json similarity index 100% rename from tests/taosbenchmark/json/queryMode1Rest.json rename to tests/taosbenchmark/json/queryModeSpecRest.json diff --git a/tests/taosbenchmark/json/queryMode3.json b/tests/taosbenchmark/json/queryModeSuper.json similarity index 100% rename from tests/taosbenchmark/json/queryMode3.json rename to tests/taosbenchmark/json/queryModeSuper.json diff --git a/tests/taosbenchmark/json/queryMode3Rest.json b/tests/taosbenchmark/json/queryModeSuperRest.json similarity index 100% rename from tests/taosbenchmark/json/queryMode3Rest.json rename to tests/taosbenchmark/json/queryModeSuperRest.json diff --git a/tests/taosbenchmark/queryMain.py b/tests/taosbenchmark/queryMain.py index e5d0a6a3..13a6105f 100644 --- a/tests/taosbenchmark/queryMain.py +++ b/tests/taosbenchmark/queryMain.py @@ -231,9 +231,9 @@ def checkAfterRun(self, benchmark, jsonFile, specMode, tbCnt): def threeQueryMode(self, benchmark, tbCnt, tbRow): # json args = [ - ["taosbenchmark/json/queryMode1", True], - ["taosbenchmark/json/queryMode2", True], - ["taosbenchmark/json/queryMode3", False] + ["taosbenchmark/json/queryModeSpec", True], + ["taosbenchmark/json/queryModeSpecMix", True], + ["taosbenchmark/json/queryModeSuper", False] ] # native @@ -255,10 +255,10 @@ def expectFailed(self, command): # check excption def exceptTest(self, benchmark, tbCnt, tbRow): # 'specified_table_query' and 'super_table_query' error - self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError1.json") - self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError2.json") + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryErrorNoSpecSuper.json") + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryErrorBothSpecSuper.json") # json format error - self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryError3.json") + self.expectFailed(f"{benchmark} -f taosbenchmark/json/queryErrorFormat.json") From 7c08a49971cf6ca00198204a7822d5e256f89568 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 20:13:07 +0800 Subject: [PATCH 44/49] fix: solve pr comment --- src/benchJsonOpt.c | 2 +- src/benchQuery.c | 13 ++++++------- src/benchUtil.c | 8 ++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 1adfdb48..526a078a 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -2169,7 +2169,7 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { tools_cJSON *gKillSlowQueryInterval = tools_cJSON_GetObjectItem(json, "kill_slow_query_interval"); if (tools_cJSON_IsNumber(gKillSlowQueryInterval)) { - g_queryInfo.killQueryInterval = gKillSlowQueryInterval ->valueint; + g_queryInfo.killQueryInterval = gKillSlowQueryInterval->valueint; } else { g_queryInfo.killQueryInterval = 1; /* by default, interval 1s */ } diff --git a/src/benchQuery.c b/src/benchQuery.c index a89758fd..bf50d5a0 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -80,12 +80,11 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool record) { } // interlligent sleep -void autoSleep(uint64_t st, uint64_t et) { +void autoSleep(uint64_t delay) { if (g_queryInfo.specifiedQueryInfo.queryInterval && - (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { + delay < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { toolsMsleep((int32_t)( - g_queryInfo.specifiedQueryInfo.queryInterval*1000 - - (et - st))); // ms + g_queryInfo.specifiedQueryInfo.queryInterval*1000 - delay)); // ms } } @@ -175,7 +174,7 @@ static void *specQueryMixThread(void *sarg) { et = toolsGetTimestampUs(); // sleep - autoSleep(st, et); + autoSleep(et - st); // delay if (ret == 0) { @@ -256,7 +255,7 @@ static void *specQueryThread(void *sarg) { et = toolsGetTimestampUs(); // sleep - autoSleep(st, et); + autoSleep(et - st); uint64_t delay = et - st; debugPrint("%s() LN%d, delay: %"PRIu64"\n", __func__, __LINE__, delay); @@ -364,7 +363,7 @@ static void *stbQueryThread(void *sarg) { et = toolsGetTimestampMs(); // sleep - autoSleep(st, et); + autoSleep(et - st); } tmfree(sqlstr); diff --git a/src/benchUtil.c b/src/benchUtil.c index c593d47a..cf59e94c 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -251,11 +251,11 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { char subTblName[TSDB_TABLE_NAME_LEN]; snprintf(subTblName, TSDB_TABLE_NAME_LEN, - "%s.%s", g_queryInfo.dbName, + "`%s`.`%s`", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName[tblIndex]); tstrncpy(outSql, inSql, pos - inSql + 1); - snprintf(outSql + strlen(outSql), TSDB_MAX_ALLOWED_SQL_LEN -1, + snprintf(outSql + pos - inSql + 1, TSDB_MAX_ALLOWED_SQL_LEN - 1, "%s%s", subTblName, pos + strlen(mark)); return 0; } @@ -1709,11 +1709,11 @@ int fetchChildTableName(char *dbName, char *stbName) { char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; if (3 == g_majorVersionOfClient) { snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", + "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM `%s`.`%s`)", dbName, stbName); } else { snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(TBNAME) FROM %s.%s", + "SELECT COUNT(TBNAME) FROM `%s`.`%s`", dbName, stbName); } TAOS_RES *res = taos_query(conn->taos, cmd); From 5b5d1f38709078984866367588673b59f6beb3f8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 20:29:56 +0800 Subject: [PATCH 45/49] fix: build error --- src/benchQuery.c | 1 - src/benchUtil.c | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index bf50d5a0..58e5591a 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -323,7 +323,6 @@ static void *stbQueryThread(void *sarg) { infoPrint("%s\n", "user cancel , so exit testing."); break; } - // get real child name sql if (replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i)) { diff --git a/src/benchUtil.c b/src/benchUtil.c index cf59e94c..f5b0407a 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -254,8 +254,9 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { "`%s`.`%s`", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName[tblIndex]); - tstrncpy(outSql, inSql, pos - inSql + 1); - snprintf(outSql + pos - inSql + 1, TSDB_MAX_ALLOWED_SQL_LEN - 1, + int32_t len = pos - inSql + 1; + tstrncpy(outSql, inSql, len); + snprintf(outSql + len, TSDB_MAX_ALLOWED_SQL_LEN - 1, "%s%s", subTblName, pos + strlen(mark)); return 0; } From 3ac3ee47d70a74b9388c01391353d8adce1af7f0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 20:37:54 +0800 Subject: [PATCH 46/49] fix: strcpy reduce pos -1 --- src/benchUtil.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/benchUtil.c b/src/benchUtil.c index f5b0407a..c2e79144 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -254,9 +254,8 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { "`%s`.`%s`", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName[tblIndex]); - int32_t len = pos - inSql + 1; - tstrncpy(outSql, inSql, len); - snprintf(outSql + len, TSDB_MAX_ALLOWED_SQL_LEN - 1, + tstrncpy(outSql, inSql, pos - inSql + 1); + snprintf(outSql + (pos - inSql), TSDB_MAX_ALLOWED_SQL_LEN - 1, "%s%s", subTblName, pos + strlen(mark)); return 0; } From a7d9348b5672d4bc58a130de09f4af14c2230c90 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 20:45:10 +0800 Subject: [PATCH 47/49] fix: childname no need add blackquota --- src/benchUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchUtil.c b/src/benchUtil.c index c2e79144..900f0ded 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -251,7 +251,7 @@ int32_t replaceChildTblName(char *inSql, char *outSql, int tblIndex) { char subTblName[TSDB_TABLE_NAME_LEN]; snprintf(subTblName, TSDB_TABLE_NAME_LEN, - "`%s`.`%s`", g_queryInfo.dbName, + "`%s`.%s", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName[tblIndex]); tstrncpy(outSql, inSql, pos - inSql + 1); From a29f13452744ccbf0dbda127f49ff00ed14bfe5f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 23 Jan 2025 21:29:34 +0800 Subject: [PATCH 48/49] fix: auto sleep interval super using spec error --- src/benchQuery.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 58e5591a..0de71d8b 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -80,11 +80,9 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool record) { } // interlligent sleep -void autoSleep(uint64_t delay) { - if (g_queryInfo.specifiedQueryInfo.queryInterval && - delay < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { - toolsMsleep((int32_t)( - g_queryInfo.specifiedQueryInfo.queryInterval*1000 - delay)); // ms +void autoSleep(uint64_t interval, uint64_t delay ) { + if (delay < interval * 1000) { + toolsMsleep((int32_t)(interval * 1000 - delay)); // ms } } @@ -145,6 +143,7 @@ static void *specQueryMixThread(void *sarg) { int64_t startTs = toolsGetTimestampMs(); int64_t lastPrintTime = startTs; uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + uint64_t interval = g_queryInfo.specifiedQueryInfo.queryInterval; pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(int64_t)); for (int i = pThreadInfo->start_sql; i <= pThreadInfo->end_sql; ++i) { SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); @@ -174,7 +173,9 @@ static void *specQueryMixThread(void *sarg) { et = toolsGetTimestampUs(); // sleep - autoSleep(et - st); + if (interval > 0) { + autoSleep(interval, et - st); + } // delay if (ret == 0) { @@ -217,6 +218,7 @@ static void *specQueryThread(void *sarg) { } uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + uint64_t interval = g_queryInfo.specifiedQueryInfo.queryInterval; pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(int64_t)); uint64_t startTs = toolsGetTimestampMs(); @@ -255,7 +257,10 @@ static void *specQueryThread(void *sarg) { et = toolsGetTimestampUs(); // sleep - autoSleep(et - st); + if (interval > 0) { + autoSleep(interval, et - st); + } + uint64_t delay = et - st; debugPrint("%s() LN%d, delay: %"PRIu64"\n", __func__, __LINE__, delay); @@ -286,6 +291,7 @@ static void *stbQueryThread(void *sarg) { uint64_t et = 0; uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes; + uint64_t interval = g_queryInfo.superQueryInfo.queryInterval; pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(uint64_t)); uint64_t startTs = toolsGetTimestampMs(); @@ -362,7 +368,10 @@ static void *stbQueryThread(void *sarg) { et = toolsGetTimestampMs(); // sleep - autoSleep(et - st); + if (interval > 0) { + autoSleep(interval, et - st); + } + } tmfree(sqlstr); From 0125f6b41e33597cff95dd478bc6dab92129c74e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 24 Jan 2025 09:15:27 +0800 Subject: [PATCH 49/49] fix: super table adjust interval to 0 --- tests/taosbenchmark/json/queryModeSuper.json | 2 +- tests/taosbenchmark/json/queryModeSuperRest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/taosbenchmark/json/queryModeSuper.json b/tests/taosbenchmark/json/queryModeSuper.json index c01c588c..9d20154d 100644 --- a/tests/taosbenchmark/json/queryModeSuper.json +++ b/tests/taosbenchmark/json/queryModeSuper.json @@ -13,7 +13,7 @@ "super_table_query": { "stblname": "meters", "concurrent": 3, - "query_interval": 1, + "query_interval": 0, "sqls": [ { "sql": "select last_row(ts) from xxxx", diff --git a/tests/taosbenchmark/json/queryModeSuperRest.json b/tests/taosbenchmark/json/queryModeSuperRest.json index 59365694..4ae43062 100644 --- a/tests/taosbenchmark/json/queryModeSuperRest.json +++ b/tests/taosbenchmark/json/queryModeSuperRest.json @@ -13,7 +13,7 @@ "super_table_query": { "stblname": "meters", "concurrent": 3, - "query_interval": 1, + "query_interval": 0, "sqls": [ { "sql": "select last_row(ts) from xxxx",