From 64501165f3939d01e655f041a4943938bca7f9c7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Sep 2024 17:22:57 +0800 Subject: [PATCH 01/76] feat: support run taos/taosd in single process --- source/dnode/mgmt/CMakeLists.txt | 9 +++++- source/dnode/mgmt/exe/dmMain.c | 4 +++ source/os/src/osString.c | 55 ++++++++++++++++++++++++++------ source/util/src/tcompare.c | 2 ++ tools/shell/CMakeLists.txt | 4 +-- tools/shell/inc/shellInt.h | 2 +- tools/shell/src/shellEngine.c | 39 ++++++++++++++++------ tools/shell/src/shellMain.c | 2 +- 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index d72301279ed0..66e0639621da 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -8,11 +8,16 @@ add_subdirectory(mgmt_dnode) add_subdirectory(test) aux_source_directory(exe EXEC_SRC) -add_executable(taosd ${EXEC_SRC}) +add_library(taosd ${EXEC_SRC}) +add_library(taosd_static ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) +target_include_directories( + taosd_static + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" +) IF (TD_ENTERPRISE) IF(${BUILD_WITH_S3}) @@ -36,3 +41,5 @@ ELSE () target_link_libraries(taosd dnode crypt) ENDIF () +target_link_libraries(taosd_static dnode crypt) + diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 89569d69d64c..bd5944eef4b8 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -342,7 +342,11 @@ static void taosCleanupArgs() { if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } +#ifndef TD_ACORE +int dmMain(int argc, char const *argv[]) { +#else int main(int argc, char const *argv[]) { +#endif int32_t code = 0; #ifdef TD_JEMALLOC_ENABLED bool jeBackgroundThread = true; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 68392b50509b..7cfd6f42b8a3 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" +#include "tutil.h" #ifndef DISALLOW_NCHAR_WITHOUT_ICONV #include "iconv.h" @@ -187,6 +188,7 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) return TSDB_CODE_SUCCESS; } +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV typedef struct { iconv_t conv; int8_t inUse; @@ -194,45 +196,72 @@ typedef struct { // 0: Mbs --> Ucs4 // 1: Ucs4--> Mbs -SConv *gConv[2] = {NULL, NULL}; -int32_t convUsed[2] = {0, 0}; -int32_t gConvMaxNum[2] = {0, 0}; +static SConv *gConv[2] = {NULL, NULL}; +static int32_t convUsed[2] = {0, 0}; +static int32_t gConvMaxNum[2] = {0, 0}; +static int8_t convInited = 0; int32_t taosConvInit(void) { + int32_t code = 0, lino = 0; + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&convInited, 0, 2); + if (old != 2) break; + if (++nLoops > 1000) { + TAOS_UNUSED(sched_yield()); + nLoops = 0; + } + } + if (old == 1) { + goto _exit; + } + int8_t M2C = 0; gConvMaxNum[M2C] = 512; gConvMaxNum[1 - M2C] = 512; gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv)); if (gConv[M2C] == NULL) { - return terrno; + atomic_store_8(&convInited, 0); + TAOS_CHECK_EXIT(terrno); } gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv)); if (gConv[1 - M2C] == NULL) { taosMemoryFree(gConv[M2C]); - return terrno; + atomic_store_8(&convInited, 0); + TAOS_CHECK_EXIT(terrno); } for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); if ((iconv_t)-1 == gConv[M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + atomic_store_8(&convInited, 1); + TAOS_CHECK_EXIT(code); } } for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + atomic_store_8(&convInited, 1); + TAOS_CHECK_EXIT(code); } } + atomic_store_8(&convInited, 1); +_exit: + if (code) { + uError("failed to init taosConvInit at line %d since %s", lino, tstrerror(code)); + } return 0; } +#endif void taosConvDestroy() { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV int8_t M2C = 0; for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { (void)iconv_close(gConv[M2C][i].conv); @@ -244,9 +273,11 @@ void taosConvDestroy() { taosMemoryFreeClear(gConv[1 - M2C]); gConvMaxNum[M2C] = -1; gConvMaxNum[1 - M2C] = -1; +#endif } iconv_t taosAcquireConv(int32_t *idx, ConvType type) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (gConvMaxNum[type] <= 0) { *idx = -1; if (type == M2C) { @@ -294,9 +325,14 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } else { return gConv[type][startId].conv; } +#else + terrno = TSDB_CODE_APP_ERROR; + return (iconv_t)-1; +#endif } void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (idx < 0) { (void)iconv_close(conv); return; @@ -304,6 +340,7 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { atomic_store_8(&gConv[type][idx].inUse, 0); (void)atomic_sub_fetch_32(&convUsed[type], 1); + #endif } bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b1f4ed0ed37d..a4ee76f9d226 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1299,7 +1299,9 @@ void DestroyRegexCache(){ taosWLockLatch(&sRegexCache.mutex); sRegexCache.exit = true; taosHashCleanup(sRegexCache.regexHash); + sRegexCache.regexHash = NULL; taosTmrCleanUp(sRegexCache.regexCacheTmr); + sRegexCache.regexCacheTmr = NULL; taosWUnLockLatch(&sRegexCache.mutex); } diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 0ce181808fb8..b32129fbea53 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -40,7 +40,7 @@ endif () target_link_libraries( shell - PRIVATE os common transport geometry util + PRIVATE os common transport geometry util dnode taosd ) target_include_directories( @@ -48,4 +48,4 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) +# SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index ba3dadc64639..379d69a7df1e 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -128,7 +128,7 @@ int32_t shellParseArgs(int32_t argc, char* argv[]); int32_t shellReadCommand(char* command); // shellEngine.c -int32_t shellExecute(); +int32_t shellExecute(int argc, char *argv[]); int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 84da746afdd8..d79f0dab9c04 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,6 +22,9 @@ #include "shellAuto.h" #include "shellInt.h" +extern void dmCleanup(); +extern int dmMain(int argc, char const *argv[]); + typedef struct { const char *sql; bool vertical; @@ -1293,16 +1296,22 @@ void *shellThreadLoop(void *arg) { return NULL; } -int32_t shellExecute() { +int32_t shellExecute(int argc, char *argv[]) { + int32_t code = 0, lino = 0; printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName); fflush(stdout); +#ifndef TD_ACORE + if ((code = dmMain(argc, (char const **)argv)) != 0) { + printf("failed to start dmMain since %s\r\n", tstrerror(code)); + TAOS_CHECK_GOTO(code, &lino, _exit_half); + } +#endif + SShellArgs *pArgs = &shell.args; #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { - if (shell_conn_ws_server(1)) { - return -1; - } + TAOS_CHECK_GOTO((shell_conn_ws_server(1)), &lino, _exit_half); } else { #endif if (shell.args.auth == NULL) { @@ -1314,7 +1323,8 @@ int32_t shellExecute() { if (shell.conn == NULL) { printf("failed to connect to server, reason: %s\n", taos_errstr(NULL)); fflush(stdout); - return -1; + code = terrno == 0 ? TSDB_CODE_INTERNAL_ERROR : terrno; + TAOS_CHECK_GOTO(code, &lino, _exit_half); } #ifdef WEBSOCKET } @@ -1355,12 +1365,12 @@ int32_t shellExecute() { shellWriteHistory(); shellCleanupHistory(); - return 0; + TAOS_CHECK_GOTO(code, &lino, _exit_half); } - if (tsem_init(&shell.cancelSem, 0, 0) != 0) { + if ((code = tsem_init(&shell.cancelSem, 0, 0)) != 0) { printf("failed to create cancel semaphore\r\n"); - return -1; + TAOS_CHECK_GOTO(code, &lino, _exit_half); } TdThread spid = {0}; @@ -1408,12 +1418,21 @@ int32_t shellExecute() { showAD(true); } #endif - taosThreadJoin(spid, NULL); + goto _exit; // normal exit +_exit_half: +#ifndef TD_ACORE + dmCleanup(); +#endif + TAOS_RETURN(code); +_exit: +#ifndef TD_ACORE + dmCleanup(); +#endif shellCleanupHistory(); taos_kill_query(shell.conn); taos_close(shell.conn); - return 0; + TAOS_RETURN(code); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 71acf23e41f7..b9d972be139c 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -111,7 +111,7 @@ int main(int argc, char *argv[]) { // support port feature shellAutoInit(); - int32_t ret = shellExecute(); + int32_t ret = shellExecute(argc, argv); shellAutoExit(); return ret; } From 2069716e8cf6ab61cb9a659ebcba589612656d2c Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 16:06:17 +0800 Subject: [PATCH 02/76] feat: support run taos/taosd in single process --- include/common/tglobal.h | 6 +- include/util/tlog.h | 8 +- source/client/src/clientEnv.c | 5 +- source/common/src/tglobal.c | 32 ++++--- source/dnode/mgmt/exe/dmMain.c | 12 ++- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 2 + source/libs/function/src/udfd.c | 2 +- source/util/src/tcompare.c | 18 ++-- source/util/src/tlog.c | 6 +- tools/shell/inc/shellInt.h | 7 ++ tools/shell/src/shellEngine.c | 109 ++++++++++++++++++++-- tools/shell/src/shellMain.c | 4 +- utils/tsim/src/simSystem.c | 2 +- 13 files changed, 171 insertions(+), 42 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index bece14c17dc0..014d68ff55db 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,7 @@ extern bool tsEnableWhiteList; extern int64_t tsDndStart; extern int64_t tsDndStartOsUptime; extern int64_t tsDndUpTime; +extern int64_t tsDndStarted; // dnode misc extern uint32_t tsEncryptionKeyChksum; @@ -234,6 +235,9 @@ extern int64_t tsmaDataDeleteMark; // wal extern int64_t tsWalFsyncDataSizeLimit; +// misc +extern bool tsAcoreOS; + // internal extern int32_t tsTransPullupInterval; extern int32_t tsCompactPullupInterval; @@ -263,7 +267,7 @@ extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, - const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); + const char *envFile, char *apolloUrl, SArray *pArgs, ELogMode mode); int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs); int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, diff --git a/include/util/tlog.h b/include/util/tlog.h index 832dc7dbc16e..ffb7b4f6ed8d 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -34,6 +34,12 @@ typedef enum { DEBUG_FILE = 128 } ELogLevel; +typedef enum { + LOG_MODE_TAOSC = 1, + LOG_MODE_TAOSD = 2, + LOG_MODE_BOTH = 3, // LOG_MODE_TAOSC | LOG_MODE_TAOSD +} ELogMode; + typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); extern bool tsLogEmbedded; @@ -69,7 +75,7 @@ extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; extern int32_t simDebugFlag; -int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); +int32_t taosInitLog(const char *logName, int32_t maxFiles, ELogMode mode); void taosCloseLog(); void taosResetLog(); void taosDumpData(uint8_t *msg, int32_t len); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3b755c29210d..70f49603a3d6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -958,13 +958,14 @@ void taos_init_imp(void) { #else (void)snprintf(logDirName, 64, "taoslog"); #endif - if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, tsAcoreOS ? LOG_MODE_BOTH : LOG_MODE_TAOSC) != + 0) { (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); tscInitRes = -1; return; } - ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); + ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, tsAcoreOS ? 0 : 1), "failed to init cfg"); initQueryModuleMsgHandle(); ENV_ERR_RET(taosConvInit(), "failed to init conv"); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2aef97ed1b54..9e3345e513f3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -98,6 +98,7 @@ int32_t tsArbSetAssignedTimeoutSec = 30; int64_t tsDndStart = 0; int64_t tsDndStartOsUptime = 0; int64_t tsDndUpTime = 0; +int64_t tsDndStarted = 0; // dnode misc uint32_t tsEncryptionKeyChksum = 0; @@ -273,6 +274,9 @@ int32_t tsTtlFlushThreshold = 100; /* maximum number of dirty items in memory. */ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch +// misc +bool tsAcoreOS = false; + // internal int32_t tsTransPullupInterval = 2; int32_t tsCompactPullupInterval = 10; @@ -1650,24 +1654,26 @@ int32_t taosSetReleaseCfg(SConfig *pCfg); static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); +static int8_t tsLogCreated = 0; + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, - const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + const char *envFile, char *apolloUrl, SArray *pArgs, ELogMode mode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SConfig *pCfg = NULL; + if (atomic_val_compare_exchange_8(&tsLogCreated, 0, 1) != 0) return 0; + if (tsCfg == NULL) { TAOS_CHECK_GOTO(osDefaultInit(), &lino, _exit); } TAOS_CHECK_GOTO(cfgInit(&pCfg), &lino, _exit); - if (tsc) { - tsLogEmbedded = 0; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); - } else { - tsLogEmbedded = 1; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); + tsLogEmbedded = (mode & LOG_MODE_TAOSC) ? 0 : 1; + TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); + + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosAddServerLogCfg(pCfg), &lino, _exit); } @@ -1681,10 +1687,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if (tsc) { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); - } else { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosSetServerLogCfg(pCfg), &lino, _exit); } @@ -1697,7 +1701,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) { + if ((code = taosInitLog(logname, logFileNum, mode)) != 0) { (void)printf("failed to init log file since %s\n", tstrerror(code)); goto _exit; } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index bd5944eef4b8..c2f681cab243 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -99,13 +99,17 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { dInfo("shut down signal is %d", signum); #ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, - taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + if (sigInfo) { + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + } #endif dmStop(); } +void dmStopDaemon() { dmStopDnode(SIGTERM, NULL, NULL); } + void dmLogCrash(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGTERM); // taosIgnSignal(SIGHUP); @@ -335,7 +339,7 @@ static int32_t dmCheckS3() { static int32_t dmInitLog() { return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, - 0); + LOG_MODE_TAOSD); } static void taosCleanupArgs() { @@ -343,7 +347,7 @@ static void taosCleanupArgs() { } #ifndef TD_ACORE -int dmMain(int argc, char const *argv[]) { +int dmStartDaemon(int argc, char const *argv[]) { #else int main(int argc, char const *argv[]) { #endif diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 3cb9030f6071..4db2c789e0a4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -155,6 +155,8 @@ int32_t dmRunDnode(SDnode *pDnode) { return code; } + atomic_store_64(&tsDndStarted, taosGetTimestampMs()); + while (1) { if (pDnode->stop) { dInfo("The daemon is about to stop"); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index e1dfd686d476..4a29440fe5f3 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1478,7 +1478,7 @@ static void udfdPrintVersion() { static int32_t udfdInitLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfd"); - return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSD); } void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index a4ee76f9d226..8ebd590405d0 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1211,11 +1211,12 @@ typedef struct UsingRegex { typedef UsingRegex* HashRegexPtr; typedef struct RegexCache { - SHashObj *regexHash; - void *regexCacheTmr; - void *timer; - SRWLatch mutex; - bool exit; + SHashObj *regexHash; + void *regexCacheTmr; + void *timer; + SRWLatch mutex; + bool exit; + int8_t inited; } RegexCache; static RegexCache sRegexCache; #define MAX_REGEX_CACHE_SIZE 20 @@ -1260,9 +1261,10 @@ void regexCacheFree(void *ppUsingRegex) { } int32_t InitRegexCache() { - #ifdef WINDOWS - return 0; - #endif +#ifdef WINDOWS + return 0; +#endif + if (atomic_val_compare_exchange_8(&sRegexCache.inited, 0, 1) != 0) return TSDB_CODE_SUCCESS; sRegexCache.regexHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (sRegexCache.regexHash == NULL) { uError("failed to create RegexCache"); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6a25c3070473..e246c64a4c18 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -215,7 +215,7 @@ int32_t taosInitSlowLog() { return 0; } -int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { +int32_t taosInitLog(const char *logName, int32_t maxFiles, ELogMode mode) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); if (code != 0) { @@ -223,7 +223,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { } TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); - if (tsc){ + if (mode & LOG_MODE_TAOSC) { TAOS_CHECK_RETURN(taosInitSlowLog()); } TAOS_CHECK_RETURN(taosStartLog()); @@ -656,7 +656,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } - if (dflag & DEBUG_SCREEN) { + if (tsLogEmbedded && (dflag & DEBUG_SCREEN)) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" if (write(1, buffer, (uint32_t)len) < 0) { diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 379d69a7df1e..8375dd2e4028 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -111,6 +111,13 @@ typedef struct { #endif } SShellObj; +#ifndef TD_ACORE +typedef struct { + TdThread pid; + int32_t stat; // < 0: start failed, 0: init(not start), 1: start successfully +} SDaemonObj; +#endif + typedef struct { char *buffer; char *command; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d79f0dab9c04..9fb3f27b66cd 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,9 +22,12 @@ #include "shellAuto.h" #include "shellInt.h" -extern void dmCleanup(); -extern int dmMain(int argc, char const *argv[]); +#ifndef TD_ACORE +extern int dmStartDaemon(int argc, char const *argv[]); +extern void dmStopDaemon(); +SDaemonObj daemonObj = {0}; +#endif typedef struct { const char *sql; bool vertical; @@ -1296,14 +1299,108 @@ void *shellThreadLoop(void *arg) { return NULL; } +#ifndef TD_ACORE +typedef struct { + int32_t argc; + char **argv; +} SExecArgs; + +static void *dmStartDaemonFunc(void *param) { + int32_t code = 0; + SExecArgs *pArgs = (SExecArgs *)param; + int32_t argc = pArgs->argc; + char **argv = pArgs->argv; + + if (argc < 1) { + code = TSDB_CODE_INVALID_PARA; + printf("failed to start taosd since Invalid parameter, argc: %d\r\n", argc); + goto _exit; + } + + code = dmStartDaemon(argc, (const char **)argv); + if (code != 0) { + printf("failed to start taosd since %s\r\n", tstrerror(code)); + goto _exit; + } + +_exit: + if (code != 0) { + atomic_store_32(&daemonObj.stat, code); + } + return NULL; +} + +static int32_t shellStartDaemon(int argc, char *argv[]) { + int32_t code = 0, lino = 0; + int64_t cost = 0; + SExecArgs *pArgs = NULL; + int64_t startMs = taosGetTimestampMs(), endMs = startMs; + + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + pArgs = (SExecArgs *)taosMemoryCalloc(1, sizeof(SExecArgs)); + if (pArgs == NULL) { + code = terrno; + TAOS_CHECK_EXIT(code); + } + pArgs->argc = argc; + pArgs->argv = argv; + + tsLogEmbedded = 1; + + TAOS_CHECK_EXIT(taosThreadCreate(&daemonObj.pid, &thAttr, dmStartDaemonFunc, pArgs)); + + while (true) { + if (atomic_load_64(&tsDndStarted)) { + atomic_store_32(&daemonObj.stat, 1); + break; + } + int32_t daemonstat = atomic_load_32(&daemonObj.stat); + if (daemonstat < 0) { + code = daemonstat; + TAOS_CHECK_EXIT(code); + } + + if (daemonstat > 1) { + code = TSDB_CODE_APP_ERROR; + TAOS_CHECK_EXIT(code); + } + taosMsleep(100); + } + +_exit: + endMs = taosGetTimestampMs(); + (void)taosThreadAttrDestroy(&thAttr); + taosMemoryFreeClear(pArgs); + if (code) { + printf("\r\n The daemon start failed at line %d since %s, cost %" PRIi64 " ms\r\n", lino, tstrerror(code), + endMs - startMs); + } else { + printf("\r\n The daemon started successfully, cost %" PRIi64 " ms\r\n", endMs - startMs); + } + tsLogEmbedded = 0; + return code; +} + +static void shellStopDaemon() { + tsLogEmbedded = 1; + dmStopDaemon(); + if (taosCheckPthreadValid(daemonObj.pid)) { + (void)taosThreadJoin(daemonObj.pid, NULL); + taosThreadClear(&daemonObj.pid); + } +} +#endif + int32_t shellExecute(int argc, char *argv[]) { int32_t code = 0, lino = 0; printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName); fflush(stdout); #ifndef TD_ACORE - if ((code = dmMain(argc, (char const **)argv)) != 0) { - printf("failed to start dmMain since %s\r\n", tstrerror(code)); + if ((code = shellStartDaemon(argc, argv)) != 0) { TAOS_CHECK_GOTO(code, &lino, _exit_half); } #endif @@ -1423,12 +1520,12 @@ int32_t shellExecute(int argc, char *argv[]) { _exit_half: #ifndef TD_ACORE - dmCleanup(); + shellStopDaemon(); #endif TAOS_RETURN(code); _exit: #ifndef TD_ACORE - dmCleanup(); + shellStopDaemon(); #endif shellCleanupHistory(); taos_kill_query(shell.conn); diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index b9d972be139c..0696243a2105 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -19,7 +19,6 @@ SShellObj shell = {0}; - void shellCrashHandler(int signum, void *sigInfo, void *context) { taosIgnSignal(SIGTERM); taosIgnSignal(SIGHUP); @@ -49,6 +48,9 @@ int main(int argc, char *argv[]) { shell.args.cloud = true; shell.args.local = false; #endif +#ifndef TD_ACORE + tsAcoreOS = true; +#endif #if 0 #if !defined(WINDOWS) diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index dcf5d6ab1237..88252a21c557 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -27,7 +27,7 @@ char simScriptDir[PATH_MAX] = {0}; extern bool simExecSuccess; int32_t simInitCfg() { - taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); + taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSC); taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg(); From b51c8f330195b95740359acb005681d91d9c31be Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 16:36:52 +0800 Subject: [PATCH 03/76] feat: support run taos/taosd in single process --- source/dnode/mgmt/CMakeLists.txt | 11 ++++------- source/dnode/mgmt/exe/dmMain.c | 2 +- tools/shell/CMakeLists.txt | 6 +++++- tools/shell/inc/shellInt.h | 2 +- tools/shell/src/shellEngine.c | 10 +++++----- tools/shell/src/shellMain.c | 2 +- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 66e0639621da..93a94698ff48 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -9,15 +9,10 @@ add_subdirectory(test) aux_source_directory(exe EXEC_SRC) add_library(taosd ${EXEC_SRC}) -add_library(taosd_static ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) -target_include_directories( - taosd_static - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" -) IF (TD_ENTERPRISE) IF(${BUILD_WITH_S3}) @@ -27,6 +22,10 @@ IF (TD_ENTERPRISE) ENDIF() ENDIF() +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + IF (TD_LINUX_64 AND JEMALLOC_ENABLED) ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") @@ -41,5 +40,3 @@ ELSE () target_link_libraries(taosd dnode crypt) ENDIF () -target_link_libraries(taosd_static dnode crypt) - diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index c2f681cab243..34b1f2f6b3f5 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -346,7 +346,7 @@ static void taosCleanupArgs() { if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } -#ifndef TD_ACORE +#ifdef TD_ACORE int dmStartDaemon(int argc, char const *argv[]) { #else int main(int argc, char const *argv[]) { diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index b32129fbea53..462bb039ca6b 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,6 +26,10 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + IF (TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE () @@ -48,4 +52,4 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -# SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) +SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 8375dd2e4028..83f2ed3be4f8 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -111,7 +111,7 @@ typedef struct { #endif } SShellObj; -#ifndef TD_ACORE +#ifdef TD_ACORE typedef struct { TdThread pid; int32_t stat; // < 0: start failed, 0: init(not start), 1: start successfully diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9fb3f27b66cd..8655a821f68b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,7 +22,7 @@ #include "shellAuto.h" #include "shellInt.h" -#ifndef TD_ACORE +#ifdef TD_ACORE extern int dmStartDaemon(int argc, char const *argv[]); extern void dmStopDaemon(); @@ -1299,7 +1299,7 @@ void *shellThreadLoop(void *arg) { return NULL; } -#ifndef TD_ACORE +#ifdef TD_ACORE typedef struct { int32_t argc; char **argv; @@ -1399,7 +1399,7 @@ int32_t shellExecute(int argc, char *argv[]) { printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName); fflush(stdout); -#ifndef TD_ACORE +#ifdef TD_ACORE if ((code = shellStartDaemon(argc, argv)) != 0) { TAOS_CHECK_GOTO(code, &lino, _exit_half); } @@ -1519,12 +1519,12 @@ int32_t shellExecute(int argc, char *argv[]) { goto _exit; // normal exit _exit_half: -#ifndef TD_ACORE +#ifdef TD_ACORE shellStopDaemon(); #endif TAOS_RETURN(code); _exit: -#ifndef TD_ACORE +#ifdef TD_ACORE shellStopDaemon(); #endif shellCleanupHistory(); diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 0696243a2105..758e16b9f6d5 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -48,7 +48,7 @@ int main(int argc, char *argv[]) { shell.args.cloud = true; shell.args.local = false; #endif -#ifndef TD_ACORE +#ifdef TD_ACORE tsAcoreOS = true; #endif From 155200f98413fa98019c6d41910956c89af318f0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 16:48:26 +0800 Subject: [PATCH 04/76] feat: support run taos/taosd in single process --- source/client/src/clientMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1b5f95ce1642..967ca5643b7a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -93,7 +93,7 @@ void taos_cleanup(void) { tscInfo("all local resources released"); taosCleanupCfg(); - taosCloseLog(); + if (!tsAcoreOS) taosCloseLog(); } static setConfRet taos_set_config_imp(const char *config) { From cd77254e4f2692ae90ab4eb7acc334ea0b7e3dd1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 17:17:25 +0800 Subject: [PATCH 05/76] enh: code optimization --- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 4db2c789e0a4..c33a9e137bdb 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -155,7 +155,11 @@ int32_t dmRunDnode(SDnode *pDnode) { return code; } - atomic_store_64(&tsDndStarted, taosGetTimestampMs()); + int64_t dndStarted = taosGetTimestampMs(); + if (dndStarted == 0) { + ++dndStarted; + } + atomic_store_64(&tsDndStarted, dndStarted); while (1) { if (pDnode->stop) { From 88d82a5c4de0b142e25422c8a9f578c8ee3dd844 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 17:24:50 +0800 Subject: [PATCH 06/76] feat: support run taos/taosd in single process --- source/dnode/mgmt/CMakeLists.txt | 7 ++++++- tools/shell/CMakeLists.txt | 15 +++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 93a94698ff48..eae085cf2264 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -8,7 +8,12 @@ add_subdirectory(mgmt_dnode) add_subdirectory(test) aux_source_directory(exe EXEC_SRC) -add_library(taosd ${EXEC_SRC}) +if(${TD_ACORE}) + add_library(taosd ${EXEC_SRC}) +else() + add_executable(taosd ${EXEC_SRC}) +endif() + target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 462bb039ca6b..289740302af6 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -42,10 +42,17 @@ else() target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET} ${LINK_JEMALLOC} ${LINK_ARGP}) endif () -target_link_libraries( - shell - PRIVATE os common transport geometry util dnode taosd -) +if(${TD_ACORE}) + target_link_libraries( + shell + PRIVATE os common transport geometry util taosd + ) +else() + target_link_libraries( + shell + PRIVATE os common transport geometry util + ) +endif() target_include_directories( shell From 33e5daebf101e56b29da53061b15f57c636586fa Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 17:51:46 +0800 Subject: [PATCH 07/76] feat: support run taos/taosd in single process --- source/dnode/mgmt/test/sut/src/sut.cpp | 2 +- source/dnode/mnode/impl/test/sdb/sdbTest.cpp | 2 +- source/dnode/mnode/impl/test/stream/stream.cpp | 2 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- source/libs/catalog/test/catalogTests.cpp | 2 +- source/libs/executor/test/joinTests.cpp | 2 +- source/libs/function/test/runUdf.c | 2 +- source/libs/index/test/index_executor_tests.cpp | 2 +- source/libs/parser/test/parTestMain.cpp | 2 +- source/libs/planner/test/planTestMain.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 2 +- source/libs/scalar/test/filter/filterTests.cpp | 2 +- source/libs/scalar/test/scalar/scalarTests.cpp | 2 +- source/libs/scheduler/test/schedulerTests.cpp | 2 +- source/libs/sync/test/syncTest.cpp | 2 +- source/libs/sync/test/syncTestTool.cpp | 2 +- source/libs/transport/test/transUT.cpp | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 13c8c73f445b..26721f020aa9 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -37,7 +37,7 @@ void Testbase::InitLog(const char* path) { taosGetSystemInfo(); tsQueueMemoryAllowed = tsTotalMemoryKB * 0.1; - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index 97b7ccd5f768..a68517ddacae 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -35,7 +35,7 @@ class MndTestSdb : public ::testing::Test { taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index d508cf7390b1..37c2f8fb90fc 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -264,7 +264,7 @@ TEST_F(StreamTest, plan_Test) { SNode * pAst = NULL; SQueryPlan *pPlan = NULL; - if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, 1) != 0) { + if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, LOG_MODE_TAOSC) != 0) { // ignore create log failed, only print (void) printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index b73f07c77886..186a5319cf0e 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -52,7 +52,7 @@ class MndTestTrans2 : public ::testing::Test { taosRemoveDir(logpath); taosMkDir(logpath); tstrncpy(tsLogDir, logpath, PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 25c82b84526c..607380b64adb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -162,7 +162,7 @@ void ctgTestInitLogFile() { (void)ctgdEnableDebug("cache", true); (void)ctgdEnableDebug("lock", true); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, LOG_MODE_TAOSC) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); ASSERT(0); } diff --git a/source/libs/executor/test/joinTests.cpp b/source/libs/executor/test/joinTests.cpp index 6dd42c3a93a6..afdb617fd0e8 100755 --- a/source/libs/executor/test/joinTests.cpp +++ b/source/libs/executor/test/joinTests.cpp @@ -2884,7 +2884,7 @@ void jtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index f28b44d1b8d5..780bdb3585c8 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -34,7 +34,7 @@ static int32_t parseArgs(int32_t argc, char *argv[]) { static int32_t initLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfc"); - return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSD); } int scalarFuncTest() { diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index f65028e56449..7ad9dee4a151 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -57,7 +57,7 @@ void sifInitLogFile() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 3986c96f076b..269330be2f1e 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -72,7 +72,7 @@ class ParserEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS, taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1, false) != 0) { + if (taosInitLog("taoslog", 1, LOG_MODE_TAOSD) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 2a1e169d5f06..040b36470a42 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -67,7 +67,7 @@ class PlannerEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS,taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1, false) != 0) { + if (taosInitLog("taoslog", 1, LOG_MODE_TAOSD) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 42b538b14fe3..5f462ed9e960 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -105,7 +105,7 @@ void qwtInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 70d6f7d0ae93..da6420dff707 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -57,7 +57,7 @@ void flttInitLogFile() { qDebugFlag = 159; (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index e14b772ea83c..f43cca4fe379 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -83,7 +83,7 @@ void scltInitLogFile() { qDebugFlag = 159; (void)strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 44d32b9480e8..0638cfc39955 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,7 +79,7 @@ void schtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 3a9a29f023cd..6c8449fbf6cb 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -48,7 +48,7 @@ void test4() { } int main(int argc, char** argv) { - taosInitLog("/tmp/syncTest.log", 100, false); + taosInitLog("/tmp/syncTest.log", 100, LOG_MODE_TAOSD); tsAsyncLog = 0; sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 6671cebcafaf..63016467ad04 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -351,7 +351,7 @@ int main(int argc, char** argv) { char logFile[256]; snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); - taosInitLog(logFile, 100, false); + taosInitLog(logFile, 100, LOG_MODE_TAOSD); sTrace("logFile : %s", logFile); gSnapshotLastApplyIndex = lastApplyIndex; diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index e57d01bcbc79..7b59cb06390f 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -230,7 +230,7 @@ static void initEnv() { taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } From 32ff35d7ee9e1ad278d680e51013e68799fe2b8a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 18:38:34 +0800 Subject: [PATCH 08/76] feat: support run taos/taosd in single process --- source/libs/index/test/fstUT.cc | 2 +- source/libs/index/test/indexBench.cc | 2 +- source/libs/index/test/indexTests.cc | 2 +- source/libs/index/test/jsonUT.cc | 2 +- source/util/test/trefTest.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 04bc6c9a506b..fcbae0ccf56e 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -28,7 +28,7 @@ static void EnvInit() { taosMkDir(path.c_str()); // init log file tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("tindex.idx", 1, false) != 0) { + if (taosInitLog("tindex.idx", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log"); } // init index file diff --git a/source/libs/index/test/indexBench.cc b/source/libs/index/test/indexBench.cc index 369ed54d0bf0..b574acb33450 100644 --- a/source/libs/index/test/indexBench.cc +++ b/source/libs/index/test/indexBench.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 090ff02ffb4e..f2d785eb78d0 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -283,7 +283,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 493101eb27ab..cdd1e6e478db 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 7abdef7c78e8..6734d5605c44 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 10, false); + taosInitLog("tref.log", 10, LOG_MODE_TAOSD); SRefSpace *pSpaceList = (SRefSpace *)taosMemoryCalloc(sizeof(SRefSpace), threads); TdThread *pThreadList = (TdThread *)taosMemoryCalloc(sizeof(TdThread), threads); From 684f1a0d7eb854c6d45dd60ec89eb6c887054000 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 21:47:09 +0800 Subject: [PATCH 09/76] fix: run taos shell with a file --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8655a821f68b..1f88e3dfd581 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1462,7 +1462,7 @@ int32_t shellExecute(int argc, char *argv[]) { shellWriteHistory(); shellCleanupHistory(); - TAOS_CHECK_GOTO(code, &lino, _exit_half); + goto _exit_half; } if ((code = tsem_init(&shell.cancelSem, 0, 0)) != 0) { @@ -1478,7 +1478,7 @@ int32_t shellExecute(int argc, char *argv[]) { taosSetSignal(SIGINT, shellQueryInterruptHandler); #ifdef WEBSOCKET - if (!shell.args.restful && !shell.args.cloud) { + if (!shell.args.restful && !shell.args.cloud) {g #endif char *buf = taosMemoryMalloc(512); bool community = shellGetGrantInfo(buf); From 3607f819db686c66550a45b63233679022f0b6d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Sep 2024 21:56:33 +0800 Subject: [PATCH 10/76] fix: typo of shell engine --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1f88e3dfd581..ef632029fc66 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1478,7 +1478,7 @@ int32_t shellExecute(int argc, char *argv[]) { taosSetSignal(SIGINT, shellQueryInterruptHandler); #ifdef WEBSOCKET - if (!shell.args.restful && !shell.args.cloud) {g + if (!shell.args.restful && !shell.args.cloud) { #endif char *buf = taosMemoryMalloc(512); bool community = shellGetGrantInfo(buf); From bb75004641135c75c07a841568d720ca1e98c340 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 30 Sep 2024 06:52:46 +0800 Subject: [PATCH 11/76] enh: code optimization --- source/os/src/osString.c | 6 ++++-- tools/shell/src/shellEngine.c | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index cb7a51df6882..374a2ec01367 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -214,8 +214,9 @@ static SConv *gConv[2] = {NULL, NULL}; static int32_t convUsed[2] = {0, 0}; static int32_t gConvMaxNum[2] = {0, 0}; static int8_t convInited = 0; - +#endif int32_t taosConvInit(void) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV int32_t code = 0, lino = 0; int8_t old; int32_t nLoops = 0; @@ -270,9 +271,10 @@ int32_t taosConvInit(void) { if (code) { uError("failed to init taosConvInit at line %d since %s", lino, tstrerror(code)); } +#endif return 0; } -#endif + void taosConvDestroy() { #ifndef DISALLOW_NCHAR_WITHOUT_ICONV diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ef632029fc66..5b3ff646f066 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1332,7 +1332,6 @@ static void *dmStartDaemonFunc(void *param) { static int32_t shellStartDaemon(int argc, char *argv[]) { int32_t code = 0, lino = 0; - int64_t cost = 0; SExecArgs *pArgs = NULL; int64_t startMs = taosGetTimestampMs(), endMs = startMs; From 805c1e0ae0c9c15fc9cb44b63ed03d41552c9ea8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 09:34:49 +0800 Subject: [PATCH 12/76] add rpc demo --- cmake/cmake.platform | 4 + source/libs/transport/CMakeLists.txt | 4 + source/libs/transport/src/trans.c | 223 ++++++++++++++++++++++++++- source/libs/transport/src/transCli.c | 57 +++++-- source/libs/transport/src/transSvr.c | 94 +++++++---- 5 files changed, 333 insertions(+), 49 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 9b96ebe7cb73..2d6843ececf8 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -218,5 +218,9 @@ else () endif() MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR}) +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})") MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})") diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index a48926d2d447..0a9bf324b405 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -24,6 +24,10 @@ if (${BUILD_WITH_UV}) endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV_TRANS}) +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + if (${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 9526c0ee9e20..c33badb26e98 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -15,9 +15,12 @@ #include "transComm.h" +#ifndef TD_ACORE +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer, transInitClient}; +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { transInitServer, transInitClient}; - void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; @@ -238,3 +241,221 @@ void rpcCleanup(void) { return; } + +#else + +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer, transInitClient}; +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; + +void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; +void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; + +int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; + +static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { + return 0; + /// int32_t code = taosGetIpv4FromFqdn(localFqdn, ip); +} +void* rpcOpen(const SRpcInit* pInit) { + int32_t code = rpcInit(); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _end); + } + + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + if (pInit->label) { + int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label); + memcpy(pRpc->label, pInit->label, len); + } + + pRpc->compressSize = pInit->compressSize; + if (pRpc->compressSize < 0) { + pRpc->compressSize = -1; + } + + pRpc->encryption = pInit->encryption; + pRpc->compatibilityVer = pInit->compatibilityVer; + + pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval + pRpc->retryStepFactor = pInit->retryStepFactor; + pRpc->retryMaxInterval = pInit->retryMaxInterval; + pRpc->retryMaxTimeout = pInit->retryMaxTimeout; + + pRpc->failFastThreshold = pInit->failFastThreshold; + pRpc->failFastInterval = pInit->failFastInterval; + + // register callback handle + pRpc->cfp = pInit->cfp; + pRpc->retry = pInit->rfp; + pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; + pRpc->failFastFp = pInit->ffp; + pRpc->noDelayFp = pInit->noDelayFp; + pRpc->connLimitNum = pInit->connLimitNum; + if (pRpc->connLimitNum == 0) { + pRpc->connLimitNum = 20; + } + + pRpc->connLimitLock = pInit->connLimitLock; + pRpc->supportBatch = pInit->supportBatch; + pRpc->batchSize = pInit->batchSize; + + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + if (pRpc->numOfThreads <= 0) { + pRpc->numOfThreads = 1; + } + + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + if ((code = transValidLocalFqdn(pInit->localFqdn, &ip)) != 0) { + tError("invalid fqdn:%s, errmsg:%s", pInit->localFqdn, tstrerror(code)); + TAOS_CHECK_GOTO(code, NULL, _end); + } + } + + pRpc->connType = pInit->connType; + pRpc->idleTime = pInit->idleTime; + pRpc->parent = pInit->parent; + if (pInit->user) { + tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); + } + pRpc->timeToGetConn = pInit->timeToGetConn; + if (pRpc->timeToGetConn == 0) { + pRpc->timeToGetConn = 10 * 1000; + } + pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; + + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + + if (pRpc->tcphandle == NULL) { + tError("failed to init rpc handle"); + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + + int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); + void* tmp = transAcquireExHandle(transGetInstMgt(), refId); + pRpc->refId = refId; + return (void*)refId; +_end: + taosMemoryFree(pRpc); + terrno = code; + + return NULL; +} +void rpcClose(void* arg) { + tInfo("start to close rpc"); + return; +} +void rpcCloseImpl(void* arg) { + if (arg == NULL) return; + return; + // SRpcInfo* pRpc = (SRpcInfo*)arg; + // if (pRpc->tcphandle != NULL) { + // (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + // } + // taosMemoryFree(pRpc); +} + +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; + char* start = taosMemoryCalloc(1, size); + if (start == NULL) { + tError("failed to malloc msg, size:%" PRId64, size); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + tTrace("malloc mem:%p size:%" PRId64, start, size); + } + + return start + sizeof(STransMsgHead); +} + +void rpcFreeCont(void* cont) { transFreeMsg(cont); } + +void* rpcReallocCont(void* ptr, int64_t contLen) { + if (ptr == NULL) return rpcMallocCont(contLen); + + char* st = (char*)ptr - TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; + char* nst = taosMemoryRealloc(st, sz); + if (nst == NULL) { + taosMemoryFree(st); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + st = nst; + } + + return st + TRANS_MSG_OVERHEAD; +} + +int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { + return transSendRequest(shandle, pEpSet, pMsg, NULL); +} +int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); + } +} + +int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +} + +int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { + return transSendRecv(shandle, pEpSet, pMsg, pRsp); +} +int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +} +int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); } + +int32_t rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } + +void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } + +void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } + +int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } +int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } + +// client only +int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { + // later + return transSetDefaultAddr(thandle, ip, fqdn); +} +// server only +int32_t rpcSetIpWhite(void* thandle, void* arg) { return transSetIpWhiteList(thandle, arg, NULL); } + +int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle(refId); } + +int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } +int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { + return transUtilSWhiteListToStr(pWhiteList, ppBuf); +} + +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + +int32_t rpcInit() { return transInit(); } + +void rpcCleanup(void) { + transCleanup(); + transHttpEnvDestroy(); + + return; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c0453c7759d5..e82fb8213123 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,6 +17,7 @@ #include "tmisce.h" // clang-format on +#ifndef TD_ACORE typedef struct { int32_t numOfConn; queue msgQ; @@ -2837,20 +2838,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { return 0; } -void transCloseClient(void* arg) { - int32_t code = 0; - SCliObj* cli = arg; - for (int i = 0; i < cli->numOfThreads; i++) { - code = cliSendQuit(cli->pThreadObj[i]); - if (code != 0) { - tError("failed to send quit to thread:%d, reason:%s", i, tstrerror(code)); - } - - destroyThrdObj(cli->pThreadObj[i]); - } - taosMemoryFree(cli->pThreadObj); - taosMemoryFree(cli); -} void transRefCliHandle(void* handle) { if (handle == NULL) { return; @@ -3076,7 +3063,6 @@ int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* p TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } - int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { @@ -3163,6 +3149,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra pReq->pCont = NULL; return code; } + int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { int32_t code = 0; tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t)); @@ -3412,3 +3399,43 @@ int32_t transFreeConnById(void* shandle, int64_t transpointId) { return code; } +void transCloseClient(void* arg) { + int32_t code = 0; + SCliObj* cli = arg; + for (int i = 0; i < cli->numOfThreads; i++) { + code = cliSendQuit(cli->pThreadObj[i]); + if (code != 0) { + tError("failed to send quit to thread:%d, reason:%s", i, tstrerror(code)); + } + + destroyThrdObj(cli->pThreadObj[i]); + } + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); +} +#else + +void transRefCliHandle(void* handle) { return; } +void transUnrefCliHandle(void* handle) { return; } +int32_t transReleaseCliHandle(void* handle) { return 0; } +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { return 0; } +int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + return 0; +} + +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { return 0; } +int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; } +int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return 0; +} +int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return 0; } +int32_t transAllocHandle(int64_t* refId) { return 0; } +int32_t transFreeConnById(void* shandle, int64_t transpointId) { return 0; } + +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + return NULL; +} +void transCloseClient(void* arg) { return; } + +#endif diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index d8737ef30f4f..ec78eeb8bf7b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -16,6 +16,7 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; +#ifndef TD_ACORE static char* notify = "a"; typedef struct { @@ -1350,6 +1351,13 @@ static void uvPipeListenCb(uv_stream_t* handle, int status) { srv->numOfWorkerReady++; } +#ifdef TD_ACORE +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + int32_t code = 0; + return NULL; +} +#else + void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { int32_t code = 0; @@ -1546,6 +1554,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, terrno = code; return NULL; } +#endif void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) { thrd->quit = true; @@ -1680,39 +1689,6 @@ void sendQuitToWorkThrd(SWorkThrd* pThrd) { TAOS_UNUSED(transAsyncSend(pThrd->asyncPool, &msg->q)); } -void transCloseServer(void* arg) { - // impl later - SServerObj* srv = arg; - - if (srv->inited) { - tDebug("send quit msg to accept thread"); - TAOS_UNUSED(uv_async_send(srv->pAcceptAsync)); - if (taosThreadJoin(srv->thread, NULL) != 0) { - tError("failed to join accept-thread"); - } - - SRV_RELEASE_UV(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { - destroyWorkThrd(srv->pThreadObj[i]); - } - } else { - SRV_RELEASE_UV(srv->loop); - } - - taosMemoryFree(srv->pThreadObj); - taosMemoryFree(srv->pAcceptAsync); - taosMemoryFree(srv->loop); - - for (int i = 0; i < srv->numOfThreads; i++) { - if (srv->pipe[i] != NULL) { - taosMemoryFree(srv->pipe[i]); - } - } - taosMemoryFree(srv->pipe); - - taosMemoryFree(srv); -} - void transRefSrvHandle(void* handle) { if (handle == NULL) { return; @@ -1917,3 +1893,55 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { } int32_t transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } + +void transCloseServer(void* arg) { + // impl later + SServerObj* srv = arg; + + if (srv->inited) { + tDebug("send quit msg to accept thread"); + TAOS_UNUSED(uv_async_send(srv->pAcceptAsync)); + if (taosThreadJoin(srv->thread, NULL) != 0) { + tError("failed to join accept-thread"); + } + + SRV_RELEASE_UV(srv->loop); + for (int i = 0; i < srv->numOfThreads; i++) { + destroyWorkThrd(srv->pThreadObj[i]); + } + } else { + SRV_RELEASE_UV(srv->loop); + } + + taosMemoryFree(srv->pThreadObj); + taosMemoryFree(srv->pAcceptAsync); + taosMemoryFree(srv->loop); + + for (int i = 0; i < srv->numOfThreads; i++) { + if (srv->pipe[i] != NULL) { + taosMemoryFree(srv->pipe[i]); + } + } + taosMemoryFree(srv->pipe); + + taosMemoryFree(srv); +} +#else + +int32_t transReleaseSrvHandle(void *handle) { return 0; } +void transRefSrvHandle(void *handle) { return; } + +void transUnrefSrvHandle(void *handle) { return; } +int32_t transSendResponse(const STransMsg *msg) { return 0; } +int32_t transRegisterMsg(const STransMsg *msg) { return 0; } +int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *shandle) { + int32_t code = 0; + return NULL; +} +void transCloseServer(void *arg) { + // impl later + return; +} + +#endif From 4a28bb0d3c191d3f54b794a8d600a026b08e2106 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 11:54:28 +0800 Subject: [PATCH 13/76] add rpc demo --- include/libs/transport/trpc.h | 172 ++++- source/libs/transport/inc/transComm.h | 5 + source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transComm.c | 805 +++++++++++++++++++++++ 5 files changed, 983 insertions(+), 1 deletion(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 5b860cc23a59..f9a691fca3ed 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -19,6 +19,7 @@ extern "C" { #endif +#ifndef TD_ACORE #include #include #include "taosdef.h" @@ -125,7 +126,7 @@ typedef struct SRpcInit { int32_t timeToGetConn; int8_t supportBatch; // 0: no batch, 1. batch int32_t batchSize; - int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait + int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait void *parent; } SRpcInit; @@ -181,6 +182,175 @@ int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); int32_t rpcCvtErrCode(int32_t code); +#else +#include +#include +#include "taosdef.h" +#include "tmsg.h" +#include "ttrace.h" + +#define TAOS_CONN_SERVER 0 +#define TAOS_CONN_CLIENT 1 +#define IsReq(pMsg) (pMsg->msgType & 1U) + +typedef enum { TD_ACORE_CLIENT = 0, TD_AOCRE_SVER, TD_AOCRE_SVR_INTERNAL } RPC_TYPE; +extern int32_t tsRpcHeadSize; + +typedef struct { + uint32_t clientIp; + uint16_t clientPort; + int64_t applyIndex; + uint64_t applyTerm; + char user[TSDB_USER_LEN]; +} SRpcConnInfo; + +typedef struct SRpcHandleInfo { + // rpc info + void *handle; // rpc handle returned to app + int64_t refId; // refid, used by server + int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) + int8_t persistHandle; // persist handle or not + int8_t hasEpSet; + int32_t cliVer; + + // app info + void *ahandle; // app handle set by client + void *wrapper; // wrapper handle + void *node; // node mgmt handle + + // resp info + void *rsp; + int32_t rspLen; + + STraceId traceId; + + SRpcConnInfo conn; + int8_t forbiddenIp; + int8_t notFreeAhandle; + int8_t compressed; + RPC_TYPE connType; +} SRpcHandleInfo; + +typedef struct SRpcMsg { + tmsg_t msgType; + void *pCont; + int32_t contLen; + int32_t code; + int32_t type; + SRpcHandleInfo info; + +} SRpcMsg; + +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcFFfp)(tmsg_t msgType); +typedef bool (*RpcNoDelayfp)(tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); + +typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; + uint16_t localPort; // local port + char *label; // for debug purpose + int32_t numOfThreads; // number of threads to handle connections + int32_t sessions; // number of sessions allowed + int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS + int32_t idleTime; // milliseconds, 0 means idle timer is disabled + int32_t compatibilityVer; + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int64_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + // the following is for client app ecurity only + char *user; // user name + + // call back to process incoming msg + RpcCfp cfp; + + // retry not not for particular msg + RpcRfp rfp; + + // set up timeout for particular msg + RpcTfp tfp; + + // destroy client ahandle; + RpcDfp dfp; + // fail fast fp + RpcFFfp ffp; + + RpcNoDelayfp noDelayFp; + + int32_t connLimitNum; + int32_t connLimitLock; + int32_t timeToGetConn; + int8_t supportBatch; // 0: no batch, 1. batch + int32_t batchSize; + int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait + void *parent; + +} SRpcInit; + +typedef struct { + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcCtxVal; + +typedef struct { + int32_t msgType; + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcBrokenlinkVal; + +typedef struct { + SHashObj *args; + SRpcBrokenlinkVal brokenVal; + void (*freeFunc)(const void *arg); +} SRpcCtx; + +int32_t rpcInit(); +void rpcCleanup(); + +void *rpcOpen(const SRpcInit *pRpc); +void rpcClose(void *); +void rpcCloseImpl(void *); +void *rpcMallocCont(int64_t contLen); +void rpcFreeCont(void *pCont); +void *rpcReallocCont(void *ptr, int64_t contLen); + +// Because taosd supports multi-process mode +// These functions should not be used on the server side +// Please use tmsg functions, which are defined in tmsgcb.h +int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +int32_t rpcSendResponse(const SRpcMsg *pMsg); +int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); +int32_t rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock + +// These functions will not be called in the child process +int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +int32_t rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int32_t rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); + +int32_t rpcFreeConnById(void *shandle, int64_t connId); + +int32_t rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); +int32_t rpcAllocHandle(int64_t *refId); +int32_t rpcSetIpWhite(void *thandl, void *arg); + +int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); + +int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); + +#endif #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 95ea6944e3d6..d678357fd24e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -465,6 +465,11 @@ int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf); int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf); int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); +#ifdef TD_ACORE +int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)); + +int32_t transGetCb(RPC_TYPE type, void** fp); +#endif #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 703a4dde3e44..02daa9f02bfa 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -77,6 +77,7 @@ typedef struct { void* tcphandle; // returned handle from TCP initialization int64_t refId; TdThreadMutex mutex; + int8_t type; } SRpcInfo; #ifdef __cplusplus diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c33badb26e98..b1da0dd28039 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -336,6 +336,7 @@ void* rpcOpen(const SRpcInit* pInit) { tError("failed to init rpc handle"); TAOS_CHECK_GOTO(terrno, NULL, _end); } + transUpdateCb(pRpc->type, pRpc->cfp); int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); void* tmp = transAcquireExHandle(transGetInstMgt(), refId); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 98b5d907a08e..282e7d38e213 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -14,7 +14,9 @@ */ #include "transComm.h" +#include "tqueue.h" +#ifndef TD_ACORE #define BUFFER_CAP 4096 static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; @@ -886,6 +888,809 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return len; } +#else +#define BUFFER_CAP 4096 + +typedef struct { + int32_t numOfThread; + STaosQueue** qhandle; + STaosQset** qset; + +} MultiThreadQhandle; +typedef struct TThread { + TdThread thread; + int idx; +} TThread; + +TdThreadMutex mutex[2]; +MultiThreadQhandle* multiQ[2] = {NULL, NULL}; +static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; + +static int32_t refMgt; +static int32_t svrRefMgt; +static int32_t instMgt; +static int32_t transSyncMsgMgt; +TdThreadMutex mutex[2]; + +TdThreadMutex tableMutex; +SHashObj* hashTable = NULL; + +void transDestroySyncMsg(void* msg); + +int32_t transCompressMsg(char* msg, int32_t len) { + int32_t ret = 0; + int compHdr = sizeof(STransCompMsg); + STransMsgHead* pHead = transHeadFromCont(msg); + + char* buf = taosMemoryMalloc(len + compHdr + 8); // 8 extra bytes + if (buf == NULL) { + tWarn("failed to allocate memory for rpc msg compression, contLen:%d", len); + ret = len; + return ret; + } + + int32_t clen = LZ4_compress_default(msg, buf, len, len + compHdr); + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (clen > 0 && clen < len - compHdr) { + STransCompMsg* pComp = (STransCompMsg*)msg; + pComp->reserved = 0; + pComp->contLen = htonl(len); + memcpy(msg + compHdr, buf, clen); + + tDebug("compress rpc msg, before:%d, after:%d", len, clen); + ret = clen + compHdr; + pHead->comp = 1; + } else { + ret = len; + pHead->comp = 0; + } + taosMemoryFree(buf); + return ret; +} +int32_t transDecompressMsg(char** msg, int32_t len) { + STransMsgHead* pHead = (STransMsgHead*)(*msg); + if (pHead->comp == 0) return 0; + + char* pCont = transContFromHead(pHead); + + STransCompMsg* pComp = (STransCompMsg*)pCont; + int32_t oriLen = htonl(pComp->contLen); + + char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); + if (buf == NULL) { + return terrno; + } + + STransMsgHead* pNewHead = (STransMsgHead*)buf; + int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); + + pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); + + taosMemoryFree(pHead); + *msg = buf; + if (decompLen != oriLen) { + return TSDB_CODE_INVALID_MSG; + } + return 0; +} + +void transFreeMsg(void* msg) { + if (msg == NULL) { + return; + } + tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); +} +int transSockInfo2Str(struct sockaddr* sockname, char* dst) { + struct sockaddr_in addr = *(struct sockaddr_in*)sockname; + + char buf[20] = {0}; + int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf)); + sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); + return r; +} +int32_t transInitBuffer(SConnBuffer* buf) { + buf->buf = taosMemoryCalloc(1, BUFFER_CAP); + if (buf->buf == NULL) { + return terrno; + } + + buf->cap = BUFFER_CAP; + buf->left = -1; + buf->len = 0; + buf->total = 0; + buf->invalid = 0; + return 0; +} +int32_t transDestroyBuffer(SConnBuffer* p) { + taosMemoryFree(p->buf); + p->buf = NULL; + return 0; +} + +int32_t transClearBuffer(SConnBuffer* buf) { + SConnBuffer* p = buf; + if (p->cap > BUFFER_CAP) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); + if (p->buf == NULL) { + return terrno; + } + } + p->left = -1; + p->len = 0; + p->total = 0; + p->invalid = 0; + return 0; +} + +int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { + static const int HEADSIZE = sizeof(STransMsgHead); + int32_t code = 0; + SConnBuffer* p = connBuf; + if (p->left != 0 || p->total <= 0) { + return TSDB_CODE_INVALID_MSG; + } + int total = p->total; + if (total >= HEADSIZE && !p->invalid) { + *buf = taosMemoryCalloc(1, total); + if (*buf == NULL) { + return terrno; + } + memcpy(*buf, p->buf, total); + if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { + return code; + } + } else { + total = -1; + return TSDB_CODE_INVALID_MSG; + } + return total; +} + +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { + SConnBuffer* p = connBuf; + if (p->total < p->len) { + int left = p->len - p->total; + memmove(p->buf, p->buf + p->total, left); + p->left = -1; + p->total = 0; + p->len = left; + } else if (p->total == p->len) { + p->left = -1; + p->total = 0; + p->len = 0; + if (p->cap > BUFFER_CAP) { + if (resetBuf) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, p->cap); + if (p->buf == NULL) { + return terrno; + } + } + } + } else { + tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); + return TSDB_CODE_INVALID_MSG; + } + return 0; +} +int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { + /* + * formate of data buffer: + * |<--------------------------data from socket------------------------------->| + * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user + * info--->| + */ + SConnBuffer* p = connBuf; + uvBuf->base = p->buf + p->len; + if (p->left == -1) { + uvBuf->len = p->cap - p->len; + } else { + if (p->left < p->cap - p->len) { + uvBuf->len = p->left; + } else { + p->cap = p->left + p->len; + p->buf = taosMemoryRealloc(p->buf, p->cap); + if (p->buf == NULL) { + uvBuf->base = NULL; + uvBuf->len = 0; + return terrno; + } + uvBuf->base = p->buf + p->len; + uvBuf->len = p->left; + } + } + return 0; +} +// check whether already read complete +bool transReadComplete(SConnBuffer* connBuf) { + SConnBuffer* p = connBuf; + if (p->len >= sizeof(STransMsgHead)) { + if (p->left == -1) { + STransMsgHead head; + memcpy((char*)&head, connBuf->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + p->total = msgLen; + p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER; + } + if (p->total >= p->len) { + p->left = p->total - p->len; + } else { + p->left = 0; + } + } + return (p->left == 0 || p->invalid) ? true : false; +} + +int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { +#if defined(WINDOWS) || defined(DARWIN) +#else + return uv_tcp_keepalive(stream, 1, keepalive); +#endif + return uv_tcp_nodelay(stream, 1); + // int ret = uv_tcp_keepalive(stream, 5, 60); +} + +int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { + SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); + if (pool == NULL) { + return terrno; + // return NULL; + } + int32_t code = 0; + + pool->nAsync = sz; + pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); + if (pool->asyncs == NULL) { + taosMemoryFree(pool); + return terrno; + } + + int i = 0, err = 0; + for (i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + + SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); + if (item == NULL) { + code = terrno; + break; + } + item->pThrd = arg; + QUEUE_INIT(&item->qmsg); + code = taosThreadMutexInit(&item->mtx, NULL); + if (code) { + taosMemoryFree(item); + break; + } + + async->data = item; + err = uv_async_init(loop, async, cb); + if (err != 0) { + tError("failed to init async, reason:%s", uv_err_name(err)); + code = TSDB_CODE_THIRDPARTY_ERROR; + break; + } + } + + if (i != pool->nAsync) { + transAsyncPoolDestroy(pool); + pool = NULL; + } + + *pPool = pool; + return 0; + // return pool; +} + +void transAsyncPoolDestroy(SAsyncPool* pool) { + if (pool == NULL) return; + + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + SAsyncItem* item = async->data; + if (item == NULL) continue; + + TAOS_UNUSED(taosThreadMutexDestroy(&item->mtx)); + taosMemoryFree(item); + } + taosMemoryFree(pool->asyncs); + taosMemoryFree(pool); +} +bool transAsyncPoolIsEmpty(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + SAsyncItem* item = async->data; + if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; + } + return true; +} +int transAsyncSend(SAsyncPool* pool, queue* q) { + if (atomic_load_8(&pool->stop) == 1) { + return TSDB_CODE_RPC_ASYNC_MODULE_QUIT; + } + int idx = pool->index % pool->nAsync; + + // no need mutex here + if (pool->index++ > pool->nAsync * 2000) { + pool->index = 0; + } + uv_async_t* async = &(pool->asyncs[idx]); + SAsyncItem* item = async->data; + + if (taosThreadMutexLock(&item->mtx) != 0) { + tError("failed to lock mutex"); + } + + QUEUE_PUSH(&item->qmsg, q); + TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx)); + int ret = uv_async_send(async); + if (ret != 0) { + tError("failed to send async,reason:%s", uv_err_name(ret)); + return TSDB_CODE_THIRDPARTY_ERROR; + } + return 0; +} + +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxCleanup(STransCtx* ctx) { + if (ctx->args == NULL) { + return; + } + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + ctx->freeFunc(iter->val); + iter = taosHashIterate(ctx->args, iter); + } + if (ctx->freeFunc) ctx->freeFunc(ctx->brokenVal.val); + taosHashCleanup(ctx->args); + ctx->args = NULL; +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (src->args == NULL || src->freeFunc == NULL) { + return; + } + if (dst->args == NULL) { + dst->args = src->args; + dst->brokenVal = src->brokenVal; + dst->freeFunc = src->freeFunc; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + if (code != 0) { + tError("failed to put val to hash, reason:%s", tstrerror(code)); + } + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + void* ret = NULL; + TAOS_UNUSED((*cVal->clone)(cVal->val, &ret)); + return ret; +} +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { + void* ret = NULL; + if (ctx->brokenVal.clone == NULL) { + return ret; + } + TAOS_UNUSED((*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret)); + + *msgType = ctx->brokenVal.msgType; + + return ret; +} + +void transReqQueueInit(queue* q) { + // init req queue + QUEUE_INIT(q); +} +void* transReqQueuePush(queue* q) { + STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); + if (req == NULL) { + return NULL; + } + req->wreq.data = req; + QUEUE_PUSH(q, &req->q); + return &req->wreq; +} +void* transReqQueueRemove(void* arg) { + void* ret = NULL; + uv_write_t* wreq = arg; + + STransReq* req = wreq ? wreq->data : NULL; + if (req == NULL) return NULL; + QUEUE_REMOVE(&req->q); + + ret = wreq && wreq->handle ? wreq->handle->data : NULL; + taosMemoryFree(req); + + return ret; +} +void transReqQueueClear(queue* q) { + while (!QUEUE_IS_EMPTY(q)) { + queue* h = QUEUE_HEAD(q); + QUEUE_REMOVE(h); + STransReq* req = QUEUE_DATA(h, STransReq, q); + taosMemoryFree(req); + } +} + +int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { + queue->q = taosArrayInit(2, sizeof(void*)); + if (queue->q == NULL) { + return terrno; + } + queue->freeFunc = (void (*)(const void*))freeFunc; + + return 0; +} +bool transQueuePush(STransQueue* queue, void* arg) { + if (queue->q == NULL) { + return true; + } + if (taosArrayPush(queue->q, &arg) == NULL) { + return false; + } + if (taosArrayGetSize(queue->q) > 1) { + return false; + } + return true; +} +void* transQueuePop(STransQueue* queue) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, 0); + taosArrayRemove(queue->q, 0); + return ptr; +} +int32_t transQueueSize(STransQueue* queue) { + if (queue->q == NULL) { + return 0; + } + return taosArrayGetSize(queue->q); +} +void* transQueueGet(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + + void* ptr = taosArrayGetP(queue->q, i); + return ptr; +} + +void* transQueueRm(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, i); + taosArrayRemove(queue->q, i); + return ptr; +} + +bool transQueueEmpty(STransQueue* queue) { + if (queue->q == NULL) { + return true; + } + return taosArrayGetSize(queue->q) == 0; +} +void transQueueClear(STransQueue* queue) { + if (queue->freeFunc != NULL) { + for (int i = 0; i < taosArrayGetSize(queue->q); i++) { + void* p = taosArrayGetP(queue->q, i); + queue->freeFunc(p); + } + } + taosArrayClear(queue->q); +} +void transQueueDestroy(STransQueue* queue) { + transQueueClear(queue); + taosArrayDestroy(queue->q); +} + +static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { + SDelayTask* arg1 = container_of(a, SDelayTask, node); + SDelayTask* arg2 = container_of(b, SDelayTask, node); + if (arg1->execTime > arg2->execTime) { + return 0; + } else { + return 1; + } +} + +static void transDQTimeout(uv_timer_t* timer) { + SDelayQueue* queue = timer->data; + tTrace("timer %p timeout", timer); + uint64_t timeout = 0; + int64_t current = taosGetTimestampMs(); + do { + HeapNode* minNode = heapMin(queue->heap); + if (minNode == NULL) break; + SDelayTask* task = container_of(minNode, SDelayTask, node); + if (task->execTime <= current) { + heapRemove(queue->heap, minNode); + task->func(task->arg); + taosMemoryFree(task); + timeout = 0; + } else { + timeout = task->execTime - current; + break; + } + } while (1); + if (timeout != 0) { + TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); + } +} +int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { return 0; } + +void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { return; } +void transDQCancel(SDelayQueue* queue, SDelayTask* task) { return; } + +SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { return NULL; } + +void transPrintEpSet(SEpSet* pEpSet) { + if (pEpSet == NULL) { + tTrace("NULL epset"); + return; + } + char buf[512] = {0}; + int len = snprintf(buf, sizeof(buf), "epset:{"); + for (int i = 0; i < pEpSet->numOfEps; i++) { + if (i == pEpSet->numOfEps - 1) { + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } else { + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + } + len += snprintf(buf + len, sizeof(buf) - len, "}"); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); +} +bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} + +void* processSvrMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[1]->qset[idx], qall, &qinfo); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + taosThreadMutexLock(&mutex[1]); + RpcCfp fp = NULL; + transGetCb(pRpcMsg->type, (void**)&fp); + taosThreadMutexUnlock(&mutex[1]); + fp(NULL, pRpcMsg, NULL); + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +void* procClientMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[0]->qset[idx], qall, &qinfo); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + taosThreadMutexLock(&mutex[1]); + + taosThreadMutexUnlock(&mutex[1]); + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +static void transInitEnv() { + refMgt = transOpenRefMgt(50000, transDestroyExHandle); + svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + + taosThreadMutexInit(&tableMutex, NULL); + hashTable = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); + + int32_t numOfAthread = 1; + + multiQ[0] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[0]->numOfThread = numOfAthread; + multiQ[0]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[0]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + + taosThreadMutexInit(&mutex[0], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[0]->qhandle[i])); + taosOpenQset(&multiQ[0]->qset[i]); + taosAddIntoQset(multiQ[0]->qset[i], multiQ[0]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, procClientMsg, (void*)&threads[i]); + } + } + + multiQ[1] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[1]->numOfThread = numOfAthread; + multiQ[1]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[1]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + taosThreadMutexInit(&mutex[1], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[1]->qhandle[i])); + taosOpenQset(&multiQ[1]->qset[i]); + taosAddIntoQset(multiQ[1]->qset[i], multiQ[1]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, processSvrMsg, (void*)&threads[i]); + } + } +} +static void transDestroyEnv() { + transCloseRefMgt(refMgt); + transCloseRefMgt(svrRefMgt); +} +int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)) { + taosThreadMutexLock(&tableMutex); + int32_t code = taosHashPut(hashTable, &type, sizeof(type), fp, sizeof(fp)); + taosThreadMutexUnlock(&tableMutex); + // taosMutexLock(&mutex[0]); + // // update callback + // return; + return 0; +} +int32_t transGetCb(RPC_TYPE type, void** fp) { + taosThreadMutexLock(&tableMutex); + void* p = taosHashGet(hashTable, &type, sizeof(type)); + if (p == NULL) { + taosThreadMutexUnlock(&tableMutex); + return TSDB_CODE_INVALID_MSG; + } + *fp = p; + taosThreadMutexUnlock(&tableMutex); + return 0; +} + +int32_t transInit() { + // init env + int32_t code = taosThreadOnce(&transModuleInit, transInitEnv); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + } + return code; +} + +int32_t transGetRefMgt() { return refMgt; } +int32_t transGetSvrRefMgt() { return svrRefMgt; } +int32_t transGetInstMgt() { return instMgt; } +int32_t transGetSyncMsgMgt() { return transSyncMsgMgt; } + +void transCleanup() { + // clean env + return; +} +int32_t transOpenRefMgt(int size, void (*func)(void*)) { + // added into once later + return 0; +} +void transCloseRefMgt(int32_t mgt) { + // close ref + return; +} +int64_t transAddExHandle(int32_t refMgt, void* p) { + // acquire extern handle + return 0; +} +int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + return 0; +} + +void* transAcquireExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + return NULL; +} + +int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) { + // release extern handle + return 0; +} +void transDestroyExHandle(void* handle) { return; } + +void transDestroySyncMsg(void* msg) { return; } + +uint32_t subnetIpRang2Int(SIpV4Range* pRange) { return 0; } +int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { return 0; } +int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf) { return 0; } +int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { return 0; } + +int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return 0; } + +int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return 0; } + +#endif // int32_t transGenRandomError(int32_t status) { // STUB_RAND_NETWORK_ERR(status) // return status; From 5357fd8f49fb01304ad1ae65dba013e924ccb3e8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 15:40:40 +0800 Subject: [PATCH 14/76] add rpc demo --- include/libs/transport/trpc.h | 9 ++++++- source/libs/transport/inc/transComm.h | 5 ++++ source/libs/transport/inc/transportInt.h | 2 +- source/libs/transport/src/trans.c | 27 +++++++++++++++++++ source/libs/transport/src/transCli.c | 5 +++- source/libs/transport/src/transComm.c | 34 ++++++++++++++++++++++-- source/libs/transport/src/transSvr.c | 5 +++- 7 files changed, 81 insertions(+), 6 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index f9a691fca3ed..7626232166ac 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -193,7 +193,6 @@ int32_t rpcCvtErrCode(int32_t code); #define TAOS_CONN_CLIENT 1 #define IsReq(pMsg) (pMsg->msgType & 1U) -typedef enum { TD_ACORE_CLIENT = 0, TD_AOCRE_SVER, TD_AOCRE_SVR_INTERNAL } RPC_TYPE; extern int32_t tsRpcHeadSize; typedef struct { @@ -204,6 +203,14 @@ typedef struct { char user[TSDB_USER_LEN]; } SRpcConnInfo; +typedef enum { + TD_ACORE_CLIENT = 1, + TD_ACORE_DSVR_CLIENT = 2, + TD_ACORE_DSVR_STA_CLIENT = 4, + TD_ACORE_DSVR_SYNC_CLIENT = 8, + TD_ACORE_DSVR = 16 +} RPC_TYPE; + typedef struct SRpcHandleInfo { // rpc info void *handle; // rpc handle returned to app diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index d678357fd24e..2f5a97bc2be4 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -466,9 +466,14 @@ int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf); int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); #ifdef TD_ACORE + int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)); int32_t transGetCb(RPC_TYPE type, void** fp); + +int32_t transSendReq(SRpcMsg* pMsg, void* pEpSet); + +int32_t transSendResp(const SRpcMsg* pMsg); #endif #ifdef __cplusplus } diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 02daa9f02bfa..c4bc4712552f 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -77,7 +77,7 @@ typedef struct { void* tcphandle; // returned handle from TCP initialization int64_t refId; TdThreadMutex mutex; - int8_t type; + int16_t type; } SRpcInfo; #ifdef __cplusplus diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index b1da0dd28039..87e5790cc2bb 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -257,6 +257,27 @@ static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; /// int32_t code = taosGetIpv4FromFqdn(localFqdn, ip); } + +// typedef enum { +// TD_ACORE_CLIENT = 1, +// TD_ACORE_DSVR_CLIENT = 2, +// TD_ACORE_DSVR_STA_CLIENT = 4, +// TD_ACORE_DSVR_SYNC_CLIENT = 8, +// TD_ACORE_DSVR = 16 +// } RPC_TYPE; + +typedef struct { + char* lablset; + RPC_TYPE type; +} SLableSet; +static SLableSet labelSet[] = { + {"TSC", TD_ACORE_CLIENT | TD_ACORE_DSVR}, + {"DNODE-CLI", TD_ACORE_DSVR_CLIENT | TD_ACORE_DSVR}, + {"DNODE-STA-CLI", TD_ACORE_DSVR_STA_CLIENT | TD_ACORE_DSVR}, + {"DNODE-SYNC-CLI", TD_ACORE_DSVR_SYNC_CLIENT | TD_ACORE_DSVR}, + {"DND-S", TD_ACORE_DSVR}, +}; + void* rpcOpen(const SRpcInit* pInit) { int32_t code = rpcInit(); if (code != 0) { @@ -336,6 +357,12 @@ void* rpcOpen(const SRpcInit* pInit) { tError("failed to init rpc handle"); TAOS_CHECK_GOTO(terrno, NULL, _end); } + for (int8_t i = 0; i < sizeof(labelSet) / sizeof(labelSet[0]); i++) { + if (strcmp(labelSet[i].lablset, pRpc->label) == 0) { + pRpc->type = labelSet[i].type; + break; + } + } transUpdateCb(pRpc->type, pRpc->cfp); int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e82fb8213123..67f0bdd8301d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3418,7 +3418,10 @@ void transCloseClient(void* arg) { void transRefCliHandle(void* handle) { return; } void transUnrefCliHandle(void* handle) { return; } int32_t transReleaseCliHandle(void* handle) { return 0; } -int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { return 0; } +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { + return transSendReq(pReq, NULL); +} + int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { return 0; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 282e7d38e213..5bb2bc141b56 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -895,6 +895,7 @@ typedef struct { int32_t numOfThread; STaosQueue** qhandle; STaosQset** qset; + int64_t idx; } MultiThreadQhandle; typedef struct TThread { @@ -1521,7 +1522,9 @@ void* processSvrMsg(void* arg) { taosGetQitem(qall, (void**)&pRpcMsg); taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; + transGetCb(pRpcMsg->type, (void**)&fp); + taosThreadMutexUnlock(&mutex[1]); fp(NULL, pRpcMsg, NULL); } @@ -1545,14 +1548,16 @@ void* procClientMsg(void* arg) { while (1) { int numOfMsgs = taosReadAllQitemsFromQset(multiQ[0]->qset[idx], qall, &qinfo); - tDebug("%d shell msgs are received", numOfMsgs); + tDebug("%d msgs are received", numOfMsgs); if (numOfMsgs <= 0) break; taosResetQitems(qall); for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); taosThreadMutexLock(&mutex[1]); - + RpcCfp fp = NULL; + transGetCb(pRpcMsg->type, (void**)&fp); taosThreadMutexUnlock(&mutex[1]); + fp(NULL, pRpcMsg, NULL); } taosUpdateItemSize(qinfo.queue, numOfMsgs); } @@ -1633,6 +1638,31 @@ int32_t transGetCb(RPC_TYPE type, void** fp) { return 0; } +int32_t transSendReq(SRpcMsg* pMsg, void* pEpSet) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[0]->idx++; + int32_t idx = cidx % (multiQ[0]->numOfThread); + tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(multiQ[0]->qhandle[idx], pTemp); + return 0; +} +int32_t transSendResp(const SRpcMsg* pMsg) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[1]->idx++; + int32_t idx = cidx % (multiQ[1]->numOfThread); + tDebug("resp is sent to, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(multiQ[1]->qhandle[idx], pTemp); + return 0; +} + int32_t transInit() { // init env int32_t code = taosThreadOnce(&transModuleInit, transInitEnv); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index ec78eeb8bf7b..e3a2be76f2c0 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1932,7 +1932,10 @@ int32_t transReleaseSrvHandle(void *handle) { return 0; } void transRefSrvHandle(void *handle) { return; } void transUnrefSrvHandle(void *handle) { return; } -int32_t transSendResponse(const STransMsg *msg) { return 0; } +int32_t transSendResponse(const STransMsg *msg) { + // + return transSendResp(msg); +} int32_t transRegisterMsg(const STransMsg *msg) { return 0; } int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *shandle) { From dc8e5e42d20cf4ded1159875f7b8d57a2a496614 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 16:24:38 +0800 Subject: [PATCH 15/76] add rpc demo --- source/libs/transport/inc/transComm.h | 4 +- source/libs/transport/src/trans.c | 4 +- source/libs/transport/src/transCli.c | 8 +++- source/libs/transport/src/transComm.c | 64 ++++++++++++++++++++------- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2f5a97bc2be4..851e7f584f04 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -469,9 +469,9 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)); -int32_t transGetCb(RPC_TYPE type, void** fp); +int32_t transGetCb(RPC_TYPE type, RpcCfp* fp); -int32_t transSendReq(SRpcMsg* pMsg, void* pEpSet); +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet); int32_t transSendResp(const SRpcMsg* pMsg); #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 87e5790cc2bb..86f8d85a6b9b 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -354,8 +354,8 @@ void* rpcOpen(const SRpcInit* pInit) { (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { - tError("failed to init rpc handle"); - TAOS_CHECK_GOTO(terrno, NULL, _end); + // tError("failed to init rpc handle"); + // TAOS_CHECK_GOTO(terrno, NULL, _end); } for (int8_t i = 0; i < sizeof(labelSet) / sizeof(labelSet[0]); i++) { if (strcmp(labelSet[i].lablset, pRpc->label) == 0) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 67f0bdd8301d..4d4c1df75ef9 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3419,7 +3419,13 @@ void transRefCliHandle(void* handle) { return; } void transUnrefCliHandle(void* handle) { return; } int32_t transReleaseCliHandle(void* handle) { return 0; } int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - return transSendReq(pReq, NULL); + int32_t code = 0; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + + pReq->type = pTransInst->type; + code = transSendReq(pTransInst, pReq, NULL); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); + return code; } int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5bb2bc141b56..a9e13f6f1af7 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1523,7 +1523,7 @@ void* processSvrMsg(void* arg) { taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; - transGetCb(pRpcMsg->type, (void**)&fp); + transGetCb(pRpcMsg->type, (RpcCfp*)&fp); taosThreadMutexUnlock(&mutex[1]); fp(NULL, pRpcMsg, NULL); @@ -1555,7 +1555,7 @@ void* procClientMsg(void* arg) { taosGetQitem(qall, (void**)&pRpcMsg); taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; - transGetCb(pRpcMsg->type, (void**)&fp); + transGetCb(pRpcMsg->type, (RpcCfp*)&fp); taosThreadMutexUnlock(&mutex[1]); fp(NULL, pRpcMsg, NULL); } @@ -1568,6 +1568,8 @@ void* procClientMsg(void* arg) { static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + instMgt = taosOpenRef(50, rpcCloseImpl); + transSyncMsgMgt = taosOpenRef(50, transDestroySyncMsg); taosThreadMutexInit(&tableMutex, NULL); hashTable = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); @@ -1617,28 +1619,33 @@ static void transDestroyEnv() { transCloseRefMgt(refMgt); transCloseRefMgt(svrRefMgt); } + +typedef struct { + void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + RPC_TYPE type; +} FP_TYPE; int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)) { taosThreadMutexLock(&tableMutex); - int32_t code = taosHashPut(hashTable, &type, sizeof(type), fp, sizeof(fp)); + + FP_TYPE t = {.fp = fp, .type = type}; + int32_t code = taosHashPut(hashTable, &type, sizeof(type), &t, sizeof(t)); taosThreadMutexUnlock(&tableMutex); - // taosMutexLock(&mutex[0]); - // // update callback - // return; return 0; } -int32_t transGetCb(RPC_TYPE type, void** fp) { +int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)) { taosThreadMutexLock(&tableMutex); void* p = taosHashGet(hashTable, &type, sizeof(type)); if (p == NULL) { taosThreadMutexUnlock(&tableMutex); return TSDB_CODE_INVALID_MSG; } - *fp = p; + FP_TYPE* t = p; + *fp = t->fp; taosThreadMutexUnlock(&tableMutex); return 0; } -int32_t transSendReq(SRpcMsg* pMsg, void* pEpSet) { +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet) { SRpcMsg* pTemp; taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); @@ -1679,37 +1686,60 @@ int32_t transGetSyncMsgMgt() { return transSyncMsgMgt; } void transCleanup() { // clean env + transDestroyEnv(); return; } int32_t transOpenRefMgt(int size, void (*func)(void*)) { - // added into once later - return 0; + /// add later + return taosOpenRef(size, func); } void transCloseRefMgt(int32_t mgt) { // close ref + taosCloseRef(mgt); return; } int64_t transAddExHandle(int32_t refMgt, void* p) { + return taosAddRef(refMgt, p); // acquire extern handle - return 0; } int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle - return 0; + return taosRemoveRef(refMgt, refId); } void* transAcquireExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle - return NULL; + return (void*)taosAcquireRef(refMgt, refId); } int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) { // release extern handle - return 0; + return taosReleaseRef(refMgt, refId); } -void transDestroyExHandle(void* handle) { return; } +void transDestroyExHandle(void* handle) { + if (handle == NULL) { + return; + } + SExHandle* eh = handle; + if (!QUEUE_IS_EMPTY(&eh->q)) { + tDebug("handle %p mem leak", handle); + } + tDebug("free exhandle %p", handle); + taosMemoryFree(handle); + return; +} + +void transDestroySyncMsg(void* msg) { + if (msg == NULL) return; -void transDestroySyncMsg(void* msg) { return; } + STransSyncMsg* pSyncMsg = msg; + TAOS_UNUSED(tsem2_destroy(pSyncMsg->pSem)); + taosMemoryFree(pSyncMsg->pSem); + transFreeMsg(pSyncMsg->pRsp->pCont); + taosMemoryFree(pSyncMsg->pRsp); + taosMemoryFree(pSyncMsg); + return; +} uint32_t subnetIpRang2Int(SIpV4Range* pRange) { return 0; } int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { return 0; } From fd7d94ad6200b9f1491db7bd2e31a9fa25a96ee2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 17:23:25 +0800 Subject: [PATCH 16/76] add rpc demo --- include/libs/transport/trpc.h | 4 ++-- source/libs/transport/inc/transComm.h | 2 +- source/libs/transport/src/trans.c | 2 +- source/libs/transport/src/transCli.c | 14 ++++++++++++++ source/libs/transport/src/transComm.c | 12 +++++++++--- source/libs/transport/src/transSvr.c | 8 +++++++- 6 files changed, 34 insertions(+), 8 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 7626232166ac..00e8c32b0ef2 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -235,7 +235,7 @@ typedef struct SRpcHandleInfo { int8_t forbiddenIp; int8_t notFreeAhandle; int8_t compressed; - RPC_TYPE connType; + int16_t connType; } SRpcHandleInfo; typedef struct SRpcMsg { @@ -336,7 +336,7 @@ void *rpcReallocCont(void *ptr, int64_t contLen); // These functions should not be used on the server side // Please use tmsg functions, which are defined in tmsgcb.h int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -int32_t rpcSendResponse(const SRpcMsg *pMsg); +int32_t rpcSendResponse(SRpcMsg *pMsg); int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); int32_t rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 851e7f584f04..0a843f4eb9ec 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -313,7 +313,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); int32_t transFreeConnById(void* shandle, int64_t transpointId); -int32_t transSendResponse(const STransMsg* msg); +int32_t transSendResponse(STransMsg* msg); int32_t transRegisterMsg(const STransMsg* msg); int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 86f8d85a6b9b..51b4fffd6e6b 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -446,7 +446,7 @@ int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRp } int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); } -int32_t rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } +int32_t rpcSendResponse(SRpcMsg* pMsg) { return transSendResponse(pMsg); } void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4d4c1df75ef9..0eb62c6485ab 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -15,6 +15,7 @@ // clang-format off #include "transComm.h" #include "tmisce.h" +#include "tversion.h" // clang-format on #ifndef TD_ACORE @@ -3420,15 +3421,28 @@ void transUnrefCliHandle(void* handle) { return; } int32_t transReleaseCliHandle(void* handle) { return 0; } int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { int32_t code = 0; + int32_t cliVer = 0; + code = taosVersionStrToInt(version, &cliVer); + // if (code != 0) { + // dError("failed to convert version string:%s to int, code:%d", version, code); + // goto _OVER; + // } STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); pReq->type = pTransInst->type; + pReq->info.connType = pReq->type; + pReq->info.cliVer = cliVer; + code = transSendReq(pTransInst, pReq, NULL); TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + int32_t code; + int32_t cliVer = 0; + code = taosVersionStrToInt(version, &cliVer); return 0; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index a9e13f6f1af7..f79defc36314 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1553,11 +1553,17 @@ void* procClientMsg(void* arg) { taosResetQitems(qall); for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); - taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; - transGetCb(pRpcMsg->type, (RpcCfp*)&fp); + taosThreadMutexLock(&mutex[1]); + if ((pRpcMsg->type & TD_ACORE_DSVR) != 0) { + transGetCb(TD_ACORE_DSVR, (RpcCfp*)&fp); + } taosThreadMutexUnlock(&mutex[1]); - fp(NULL, pRpcMsg, NULL); + if (fp != NULL) { + fp(NULL, pRpcMsg, NULL); + } else { + tError("failed to find callback for msg type:%d", pRpcMsg->type); + } } taosUpdateItemSize(qinfo.queue, numOfMsgs); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index e3a2be76f2c0..5a3d7d041d5b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -13,6 +13,7 @@ */ #include "transComm.h" +#include "tversion.h" static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; @@ -1932,8 +1933,13 @@ int32_t transReleaseSrvHandle(void *handle) { return 0; } void transRefSrvHandle(void *handle) { return; } void transUnrefSrvHandle(void *handle) { return; } -int32_t transSendResponse(const STransMsg *msg) { +int32_t transSendResponse(STransMsg *msg) { // + int32_t code = 0; + int32_t svrVer = 0; + code = taosVersionStrToInt(version, &svrVer); + msg->info.cliVer = svrVer; + msg->type = msg->info.connType; return transSendResp(msg); } int32_t transRegisterMsg(const STransMsg *msg) { return 0; } From 14dddc6168053df804dbf87536a661c71ce30fd2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 17 Oct 2024 21:02:19 +0800 Subject: [PATCH 17/76] add rpc demo --- include/libs/transport/trpc.h | 1 + source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/libs/transport/inc/transComm.h | 4 ++-- source/libs/transport/src/trans.c | 2 +- source/libs/transport/src/transCli.c | 3 +++ source/libs/transport/src/transComm.c | 27 +++++++++++++++--------- source/libs/transport/src/transSvr.c | 7 ++++++ 7 files changed, 32 insertions(+), 14 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 00e8c32b0ef2..cf8dffaaa0e2 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -244,6 +244,7 @@ typedef struct SRpcMsg { int32_t contLen; int32_t code; int32_t type; + void *parent; SRpcHandleInfo info; } SRpcMsg; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 4dc2f093e8d0..d5725cba5da3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "mndProfile.h" #include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" -#include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" #include "mndSma.h" diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 0a843f4eb9ec..2b6ecf1736e5 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -467,9 +467,9 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); #ifdef TD_ACORE -int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)); +int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void* arg); -int32_t transGetCb(RPC_TYPE type, RpcCfp* fp); +int32_t transGetCb(RPC_TYPE type, RpcCfp* fp, void** arg); int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 51b4fffd6e6b..2b622abdfeb7 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -363,7 +363,7 @@ void* rpcOpen(const SRpcInit* pInit) { break; } } - transUpdateCb(pRpc->type, pRpc->cfp); + transUpdateCb(pRpc->type, pRpc->cfp, pRpc->parent); int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); void* tmp = transAcquireExHandle(transGetInstMgt(), refId); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0eb62c6485ab..a5c7b25e6bc7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3433,6 +3433,9 @@ int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, S pReq->type = pTransInst->type; pReq->info.connType = pReq->type; pReq->info.cliVer = cliVer; + pReq->info.handle = (void*)0x9537; + pReq->parent = pTransInst; + sprintf(pReq->info.conn.user, "root"); code = transSendReq(pTransInst, pReq, NULL); TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f79defc36314..8c413da0f69a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1522,8 +1522,9 @@ void* processSvrMsg(void* arg) { taosGetQitem(qall, (void**)&pRpcMsg); taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; - - transGetCb(pRpcMsg->type, (RpcCfp*)&fp); + void* parent = NULL; + tDebug("taos %s received from taos", TMSG_INFO(pRpcMsg->msgType)); + transGetCb(pRpcMsg->type, (RpcCfp*)&fp, &parent); taosThreadMutexUnlock(&mutex[1]); fp(NULL, pRpcMsg, NULL); @@ -1553,16 +1554,20 @@ void* procClientMsg(void* arg) { taosResetQitems(qall); for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); + + tDebug("taos %s received from taos", TMSG_INFO(pRpcMsg->msgType)); RpcCfp fp = NULL; + void* parent; taosThreadMutexLock(&mutex[1]); if ((pRpcMsg->type & TD_ACORE_DSVR) != 0) { - transGetCb(TD_ACORE_DSVR, (RpcCfp*)&fp); + transGetCb(TD_ACORE_DSVR, (RpcCfp*)&fp, &parent); } + STrans* pTrans = pRpcMsg->parent; taosThreadMutexUnlock(&mutex[1]); if (fp != NULL) { - fp(NULL, pRpcMsg, NULL); + fp(parent, pRpcMsg, NULL); } else { - tError("failed to find callback for msg type:%d", pRpcMsg->type); + tError("taos failed to find callback for msg type:%s", TMSG_INFO(pRpcMsg->msgType)); } } taosUpdateItemSize(qinfo.queue, numOfMsgs); @@ -1629,16 +1634,17 @@ static void transDestroyEnv() { typedef struct { void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); RPC_TYPE type; + void* parant; } FP_TYPE; -int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)) { +int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void* arg) { taosThreadMutexLock(&tableMutex); - FP_TYPE t = {.fp = fp, .type = type}; + FP_TYPE t = {.fp = fp, .type = type, .parant = arg}; int32_t code = taosHashPut(hashTable, &type, sizeof(type), &t, sizeof(t)); taosThreadMutexUnlock(&tableMutex); return 0; } -int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)) { +int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void** arg) { taosThreadMutexLock(&tableMutex); void* p = taosHashGet(hashTable, &type, sizeof(type)); if (p == NULL) { @@ -1647,6 +1653,7 @@ int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSe } FP_TYPE* t = p; *fp = t->fp; + *arg = t->parant; taosThreadMutexUnlock(&tableMutex); return 0; } @@ -1659,7 +1666,7 @@ int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet) { int64_t cidx = multiQ[0]->idx++; int32_t idx = cidx % (multiQ[0]->numOfThread); - tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + tDebug("taos request is sent , type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); taosWriteQitem(multiQ[0]->qhandle[idx], pTemp); return 0; } @@ -1671,7 +1678,7 @@ int32_t transSendResp(const SRpcMsg* pMsg) { int64_t cidx = multiQ[1]->idx++; int32_t idx = cidx % (multiQ[1]->numOfThread); - tDebug("resp is sent to, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + tDebug("taos resp is sent to, type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); taosWriteQitem(multiQ[1]->qhandle[idx], pTemp); return 0; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5a3d7d041d5b..5af1db495074 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1936,6 +1936,13 @@ void transUnrefSrvHandle(void *handle) { return; } int32_t transSendResponse(STransMsg *msg) { // int32_t code = 0; + if (rpcIsReq(msg->msgType)) { + msg->msgType = msg->msgType + 1; + } + if (msg->info.noResp) { + rpcFreeCont(msg->pCont); + return 0; + } int32_t svrVer = 0; code = taosVersionStrToInt(version, &svrVer); msg->info.cliVer = svrVer; From 954754ca48791b34f5f51240a0c309fe7321cac8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 18 Oct 2024 16:45:15 +0800 Subject: [PATCH 18/76] add rpc demo --- include/libs/transport/trpc.h | 4 ++ source/libs/transport/inc/transComm.h | 4 +- source/libs/transport/inc/transportInt.h | 7 +++ source/libs/transport/src/trans.c | 10 +++- source/libs/transport/src/transCli.c | 36 +++++++++++-- source/libs/transport/src/transComm.c | 64 ++++++++++++++++++------ source/libs/transport/src/transSvr.c | 4 +- 7 files changed, 104 insertions(+), 25 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index cf8dffaaa0e2..d7b98a963a11 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -236,6 +236,10 @@ typedef struct SRpcHandleInfo { int8_t notFreeAhandle; int8_t compressed; int16_t connType; + int64_t seq; + int64_t sidSeq; + int64_t qId; + int32_t msgType; } SRpcHandleInfo; typedef struct SRpcMsg { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2b6ecf1736e5..73d80d1337e2 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -467,9 +467,9 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); #ifdef TD_ACORE -int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void* arg); +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport); -int32_t transGetCb(RPC_TYPE type, RpcCfp* fp, void** arg); +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport); int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c4bc4712552f..473362d378fc 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -78,6 +78,13 @@ typedef struct { int64_t refId; TdThreadMutex mutex; int16_t type; + + TdThreadMutex sidMutx; + SHashObj* sidTable; + + TdThreadMutex seqMutex; + int64_t seq; + SHashObj* seqTable; } SRpcInfo; #ifdef __cplusplus diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 2b622abdfeb7..2f11e1a8c1ff 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -269,6 +269,7 @@ static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { typedef struct { char* lablset; RPC_TYPE type; + } SLableSet; static SLableSet labelSet[] = { {"TSC", TD_ACORE_CLIENT | TD_ACORE_DSVR}, @@ -363,7 +364,14 @@ void* rpcOpen(const SRpcInit* pInit) { break; } } - transUpdateCb(pRpc->type, pRpc->cfp, pRpc->parent); + + taosThreadMutexInit(&pRpc->sidMutx, NULL); + pRpc->sidTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosThreadMutexInit(&pRpc->seqMutex, NULL); + pRpc->seqTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + pRpc->seq = 1; + transUpdateCb(pRpc->type, pRpc); int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); void* tmp = transAcquireExHandle(transGetInstMgt(), refId); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a5c7b25e6bc7..25865255cb39 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3422,19 +3422,44 @@ int32_t transReleaseCliHandle(void* handle) { return 0; } int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { int32_t code = 0; int32_t cliVer = 0; + code = taosVersionStrToInt(version, &cliVer); - // if (code != 0) { - // dError("failed to convert version string:%s to int, code:%d", version, code); - // goto _OVER; - // } STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); pReq->type = pTransInst->type; pReq->info.connType = pReq->type; pReq->info.cliVer = cliVer; - pReq->info.handle = (void*)0x9537; + pReq->info.msgType = pReq->msgType; + // pReq->info.ahandle = pReq->ahandle; + if (pReq->info.handle != NULL) { + pReq->info.qId = (int64_t)pReq->info.handle; + } else { + pReq->info.handle = (void*)-1; + } + + if (pReq->info.qId != 0) { + taosThreadMutexLock(&pTransInst->sidMutx); + /******/ + if (ctx != NULL) { + STransCtx* pUserCtx = taosHashGet(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId)); + if (pUserCtx != NULL) { + transCtxMerge(pUserCtx, ctx); + } else { + taosHashPut(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId), ctx, sizeof(STransCtx)); + } + } + taosThreadMutexUnlock(&pTransInst->sidMutx); + } + taosThreadMutexLock(&pTransInst->seqMutex); + pReq->info.seq = pTransInst->seq++; pReq->parent = pTransInst; + taosHashPut(pTransInst->seqTable, &pReq->info.seq, sizeof(pReq->info.seq), &pReq->msgType, sizeof(pReq->msgType)); + taosThreadMutexUnlock(&pTransInst->seqMutex); + sprintf(pReq->info.conn.user, "root"); code = transSendReq(pTransInst, pReq, NULL); @@ -3446,6 +3471,7 @@ int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* p int32_t code; int32_t cliVer = 0; code = taosVersionStrToInt(version, &cliVer); + return 0; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8c413da0f69a..d783f2fa28d6 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1501,6 +1501,30 @@ bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { return true; } +bool cliMayGetAhandle(STrans* pTrans, SRpcMsg* pMsg) { + int64_t seq = pMsg->info.seq; + int32_t* msgType = NULL; + + taosThreadMutexLock(&pTrans->seqMutex); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + if (msgType == NULL) { + STransCtx* ctx = taosHashGet(pTrans->sidTable, &pMsg->info.qId, sizeof(pMsg->info.qId)); + if (ctx == NULL) { + return false; + } + pMsg->info.ahandle = transCtxDumpVal(ctx, pMsg->msgType); + tError("failed to find msg type for seq:%" PRId64 ", gen ahandle for type %s" PRId64, seq, + TMSG_INFO(pMsg->msgType)); + } else { + taosThreadMutexLock(&pTrans->seqMutex); + taosHashRemove(pTrans->seqTable, &seq, sizeof(seq)); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + } + return true; +} + void* processSvrMsg(void* arg) { TThread* thread = (TThread*)arg; @@ -1523,11 +1547,19 @@ void* processSvrMsg(void* arg) { taosThreadMutexLock(&mutex[1]); RpcCfp fp = NULL; void* parent = NULL; - tDebug("taos %s received from taos", TMSG_INFO(pRpcMsg->msgType)); - transGetCb(pRpcMsg->type, (RpcCfp*)&fp, &parent); + tDebug("taos %s received from taosd", TMSG_INFO(pRpcMsg->msgType)); + STrans* pTrans = NULL; + transGetCb(pRpcMsg->type, &pTrans); taosThreadMutexUnlock(&mutex[1]); - fp(NULL, pRpcMsg, NULL); + + if (pTrans != NULL) { + if (cliMayGetAhandle(pTrans, pRpcMsg)) { + (pTrans->cfp)(NULL, pRpcMsg, NULL); + } else { + tDebug("taosd %s received from taosd, ignore", TMSG_INFO(pRpcMsg->msgType)); + } + } } taosUpdateItemSize(qinfo.queue, numOfMsgs); } @@ -1555,19 +1587,19 @@ void* procClientMsg(void* arg) { for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); - tDebug("taos %s received from taos", TMSG_INFO(pRpcMsg->msgType)); + tDebug("taosc %s received from taosc", TMSG_INFO(pRpcMsg->msgType)); RpcCfp fp = NULL; - void* parent; + // void* parent; + STrans* pTrans = NULL; taosThreadMutexLock(&mutex[1]); if ((pRpcMsg->type & TD_ACORE_DSVR) != 0) { - transGetCb(TD_ACORE_DSVR, (RpcCfp*)&fp, &parent); + transGetCb(TD_ACORE_DSVR, &pTrans); } - STrans* pTrans = pRpcMsg->parent; taosThreadMutexUnlock(&mutex[1]); - if (fp != NULL) { - fp(parent, pRpcMsg, NULL); + if (pTrans->cfp != NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); } else { - tError("taos failed to find callback for msg type:%s", TMSG_INFO(pRpcMsg->msgType)); + tError("taosc failed to find callback for msg type:%s", TMSG_INFO(pRpcMsg->msgType)); } } taosUpdateItemSize(qinfo.queue, numOfMsgs); @@ -1635,16 +1667,17 @@ typedef struct { void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); RPC_TYPE type; void* parant; + STrans* pTransport; } FP_TYPE; -int32_t transUpdateCb(RPC_TYPE type, void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void* arg) { +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport) { taosThreadMutexLock(&tableMutex); - FP_TYPE t = {.fp = fp, .type = type, .parant = arg}; + FP_TYPE t = {.type = type, .pTransport = pTransport}; int32_t code = taosHashPut(hashTable, &type, sizeof(type), &t, sizeof(t)); taosThreadMutexUnlock(&tableMutex); return 0; } -int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet), void** arg) { +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport) { taosThreadMutexLock(&tableMutex); void* p = taosHashGet(hashTable, &type, sizeof(type)); if (p == NULL) { @@ -1652,8 +1685,9 @@ int32_t transGetCb(RPC_TYPE type, void (**fp)(void* parent, SRpcMsg* pMsg, SEpSe return TSDB_CODE_INVALID_MSG; } FP_TYPE* t = p; - *fp = t->fp; - *arg = t->parant; + *ppTransport = t->pTransport; + // *fp = t->fp; + // *arg = t->parant; taosThreadMutexUnlock(&tableMutex); return 0; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5af1db495074..fdf7ca1c30e5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1936,8 +1936,8 @@ void transUnrefSrvHandle(void *handle) { return; } int32_t transSendResponse(STransMsg *msg) { // int32_t code = 0; - if (rpcIsReq(msg->msgType)) { - msg->msgType = msg->msgType + 1; + if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { + msg->msgType = msg->info.msgType + 1; } if (msg->info.noResp) { rpcFreeCont(msg->pCont); From 03f39e3fd6adbc501eaaa3d7a87028fd60f975e3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 18 Oct 2024 19:31:02 +0800 Subject: [PATCH 19/76] add rpc demo --- include/libs/transport/trpc.h | 1 + source/libs/transport/inc/transComm.h | 5 ++++ source/libs/transport/src/transCli.c | 35 ++++++++++++++++++++++++--- source/libs/transport/src/transComm.c | 17 +++++++++---- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index d7b98a963a11..d73b41cdbe1b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -240,6 +240,7 @@ typedef struct SRpcHandleInfo { int64_t sidSeq; int64_t qId; int32_t msgType; + void *reqWithSem; } SRpcHandleInfo; typedef struct SRpcMsg { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 73d80d1337e2..cfbbeb7d78cd 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -109,6 +109,11 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +typedef struct { + tsem_t* sem; + STransMsg pMsg; +} STransReqWithSem; + // ref mgt handle typedef struct SExHandle { void* handle; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 25865255cb39..822ef9969b35 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3469,16 +3469,45 @@ int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, S int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { int32_t code; - int32_t cliVer = 0; - code = taosVersionStrToInt(version, &cliVer); + // int32_t cliVer = 0; + // code = taosVersionStrToInt(version, &cliVer); + return transSendRequest(shandle, pEpSet, pReq, NULL); return 0; } -int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { return 0; } +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { + return transSendRecvWithTimeout(shandle, (SEpSet*)pEpSet, pReq, pRsp, NULL, 0); +} int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; } int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs) { + int32_t code = 0; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + STransReqWithSem* tmsg = taosMemoryCalloc(1, sizeof(STransReqWithSem)); + tmsg->sem = taosMemoryCalloc(1, sizeof(tsem_t)); + code = tsem_init(tmsg->sem, 0, 0); + if (code != 0) { + taosMemoryFree(tmsg->sem); + return code; + } + pReq->info.reqWithSem = tmsg; + transSendRequest(shandle, NULL, pReq, NULL); + TAOS_UNUSED(tsem_wait(tmsg->sem)); + tsem_destroy(tmsg->sem); + + memcpy(pRsp, &tmsg->pMsg, sizeof(STransMsg)); + taosMemoryFree(tmsg); + + taosReleaseRef(transGetInstMgt(), (int64_t)shandle); return 0; } int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return 0; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index d783f2fa28d6..2c8d808cf64f 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1539,15 +1539,15 @@ void* processSvrMsg(void* arg) { while (1) { int numOfMsgs = taosReadAllQitemsFromQset(multiQ[1]->qset[idx], qall, &qinfo); - tDebug("%d shell msgs are received", numOfMsgs); if (numOfMsgs <= 0) break; taosResetQitems(qall); for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); taosThreadMutexLock(&mutex[1]); - RpcCfp fp = NULL; - void* parent = NULL; - tDebug("taos %s received from taosd", TMSG_INFO(pRpcMsg->msgType)); + RpcCfp fp = NULL; + void* parent = NULL; + STraceId* trace = &pRpcMsg->info.traceId; + tGDebug("taos %s received from taosd", TMSG_INFO(pRpcMsg->msgType)); STrans* pTrans = NULL; transGetCb(pRpcMsg->type, &pTrans); @@ -1555,7 +1555,13 @@ void* processSvrMsg(void* arg) { if (pTrans != NULL) { if (cliMayGetAhandle(pTrans, pRpcMsg)) { - (pTrans->cfp)(NULL, pRpcMsg, NULL); + if (pRpcMsg->info.reqWithSem == NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); + } else { + STransReqWithSem* reqWithSem = pRpcMsg->info.reqWithSem; + memcpy(&reqWithSem->pMsg, pRpcMsg, sizeof(SRpcMsg)); + tsem_post(reqWithSem->sem); + } } else { tDebug("taosd %s received from taosd, ignore", TMSG_INFO(pRpcMsg->msgType)); } @@ -1587,6 +1593,7 @@ void* procClientMsg(void* arg) { for (int i = 0; i < numOfMsgs; i++) { taosGetQitem(qall, (void**)&pRpcMsg); + STraceId* trace = &pRpcMsg->info.traceId; tDebug("taosc %s received from taosc", TMSG_INFO(pRpcMsg->msgType)); RpcCfp fp = NULL; // void* parent; From 4637496b0358293cb35c59dfba90f39b242a1a2c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Oct 2024 09:12:58 +0800 Subject: [PATCH 20/76] add rpc demo --- source/libs/transport/inc/transComm.h | 21 +- source/libs/transport/inc/transportInt.h | 70 ++ source/libs/transport/src/thttp.c | 17 + source/libs/transport/src/trans.c | 14 +- source/libs/transport/src/transComm.c | 861 ++++++++++++----------- 5 files changed, 539 insertions(+), 444 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index cfbbeb7d78cd..4f3954f9eca8 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -18,6 +18,14 @@ extern "C" { #endif +#ifdef TD_ACORE +#include "tmsg.h" +#include "transLog.h" +#include "transportInt.h" +#include "trpc.h" +#include "ttrace.h" + +#else #include #include "theap.h" #include "tmsg.h" @@ -25,6 +33,7 @@ extern "C" { #include "transportInt.h" #include "trpc.h" #include "ttrace.h" +#endif typedef bool (*FilteFunc)(void* arg); @@ -229,7 +238,12 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transIsReq(type) (type & 1U) #define transLabel(trans) ((STrans*)trans)->label +#ifdef TD_ACORE +typedef struct SConnBuffer { + char* buf; +} SConnBuffer; +#else typedef struct SConnBuffer { char* buf; int len; @@ -301,6 +315,7 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); int32_t transSetConnOption(uv_tcp_t* stream, int keepalive); +#endif void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); @@ -342,8 +357,8 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); // request list typedef struct STransReq { - queue q; - uv_write_t wreq; + queue q; + // uv_write_t wreq; } STransReq; void transReqQueueInit(queue* q); @@ -405,6 +420,7 @@ typedef struct STaskArg { void* param2; } STaskArg; +#ifndef TD_ACORE typedef struct SDelayTask { void (*func)(void* arg); void* arg; @@ -423,6 +439,7 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); void transDQCancel(SDelayQueue* queue, SDelayTask* task); +#endif bool transEpSetIsEqual(SEpSet* a, SEpSet* b); bool transEpSetIsEqual2(SEpSet* a, SEpSet* b); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 473362d378fc..0f6cabd4c44d 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,6 +16,7 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#ifndef TD_ACORE #include #include "lz4.h" #include "os.h" @@ -27,11 +28,79 @@ #include "tref.h" #include "trpc.h" #include "tutil.h" +#else +#include "lz4.h" +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tmsg.h" +#include "transLog.h" +#include "tref.h" +#include "trpc.h" +#include "tutil.h" +#endif #ifdef __cplusplus extern "C" { #endif +#ifndef TD_ACORE +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void taosCloseServer(void* arg); +void taosCloseClient(void* arg); + +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + char label[TSDB_LABEL_LEN]; + char user[TSDB_UNI_LEN]; // meter ID + int32_t compatibilityVer; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int8_t notWaitAvaliableConn; // 1: no delay, 0: delay + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + bool (*retry)(int32_t code, tmsg_t msgType); + bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); + bool (*failFastFp)(tmsg_t msgType); + bool (*noDelayFp)(tmsg_t msgType); + + int32_t connLimitNum; + int8_t connLimitLock; // 0: no lock. 1. lock + int8_t supportBatch; // 0: no batch, 1: support batch + int32_t batchSize; + int32_t timeToGetConn; + int index; + void* parent; + void* tcphandle; // returned handle from TCP initialization + int64_t refId; + TdThreadMutex mutex; + int16_t type; + + TdThreadMutex sidMutx; + SHashObj* sidTable; + + TdThreadMutex seqMutex; + int64_t seq; + SHashObj* seqTable; +} SRpcInfo; +#else void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); @@ -86,6 +155,7 @@ typedef struct { int64_t seq; SHashObj* seqTable; } SRpcInfo; +#endif #ifdef __cplusplus } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 7d7868f3cd06..2d10e9bbe3fc 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE // clang-format off +#ifndef TD_ACORE #include #include "zlib.h" #include "thttp.h" @@ -868,3 +869,19 @@ void taosDestroyHttpChan(int64_t chanId) { TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); TAOS_UNUSED(taosRemoveRef(httpRefMgt, chanId)); } +#else +#include "thttp.h" +#include "taoserror.h" +#include "transComm.h" +#include "zlib.h" +void transHttpEnvDestroy() { return; } +int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag) { + return 0; +} +int32_t taosSendHttpReportWithQID(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag, const char* qid) { + return 0; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 2f11e1a8c1ff..babd5e751f4d 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -253,19 +253,7 @@ void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHan int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; -static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { - return 0; - /// int32_t code = taosGetIpv4FromFqdn(localFqdn, ip); -} - -// typedef enum { -// TD_ACORE_CLIENT = 1, -// TD_ACORE_DSVR_CLIENT = 2, -// TD_ACORE_DSVR_STA_CLIENT = 4, -// TD_ACORE_DSVR_SYNC_CLIENT = 8, -// TD_ACORE_DSVR = 16 -// } RPC_TYPE; - +static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; } typedef struct { char* lablset; RPC_TYPE type; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 2c8d808cf64f..3e918048a29a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -987,256 +987,259 @@ void transFreeMsg(void* msg) { tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD); taosMemoryFree((char*)msg - sizeof(STransMsgHead)); } -int transSockInfo2Str(struct sockaddr* sockname, char* dst) { - struct sockaddr_in addr = *(struct sockaddr_in*)sockname; - - char buf[20] = {0}; - int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf)); - sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); - return r; -} -int32_t transInitBuffer(SConnBuffer* buf) { - buf->buf = taosMemoryCalloc(1, BUFFER_CAP); - if (buf->buf == NULL) { - return terrno; - } - - buf->cap = BUFFER_CAP; - buf->left = -1; - buf->len = 0; - buf->total = 0; - buf->invalid = 0; - return 0; -} -int32_t transDestroyBuffer(SConnBuffer* p) { - taosMemoryFree(p->buf); - p->buf = NULL; - return 0; -} - -int32_t transClearBuffer(SConnBuffer* buf) { - SConnBuffer* p = buf; - if (p->cap > BUFFER_CAP) { - p->cap = BUFFER_CAP; - p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); - if (p->buf == NULL) { - return terrno; - } - } - p->left = -1; - p->len = 0; - p->total = 0; - p->invalid = 0; - return 0; -} - -int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { - static const int HEADSIZE = sizeof(STransMsgHead); - int32_t code = 0; - SConnBuffer* p = connBuf; - if (p->left != 0 || p->total <= 0) { - return TSDB_CODE_INVALID_MSG; - } - int total = p->total; - if (total >= HEADSIZE && !p->invalid) { - *buf = taosMemoryCalloc(1, total); - if (*buf == NULL) { - return terrno; - } - memcpy(*buf, p->buf, total); - if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { - return code; - } - } else { - total = -1; - return TSDB_CODE_INVALID_MSG; - } - return total; -} - -int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { - SConnBuffer* p = connBuf; - if (p->total < p->len) { - int left = p->len - p->total; - memmove(p->buf, p->buf + p->total, left); - p->left = -1; - p->total = 0; - p->len = left; - } else if (p->total == p->len) { - p->left = -1; - p->total = 0; - p->len = 0; - if (p->cap > BUFFER_CAP) { - if (resetBuf) { - p->cap = BUFFER_CAP; - p->buf = taosMemoryRealloc(p->buf, p->cap); - if (p->buf == NULL) { - return terrno; - } - } - } - } else { - tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); - return TSDB_CODE_INVALID_MSG; - } - return 0; -} -int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { - /* - * formate of data buffer: - * |<--------------------------data from socket------------------------------->| - * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user - * info--->| - */ - SConnBuffer* p = connBuf; - uvBuf->base = p->buf + p->len; - if (p->left == -1) { - uvBuf->len = p->cap - p->len; - } else { - if (p->left < p->cap - p->len) { - uvBuf->len = p->left; - } else { - p->cap = p->left + p->len; - p->buf = taosMemoryRealloc(p->buf, p->cap); - if (p->buf == NULL) { - uvBuf->base = NULL; - uvBuf->len = 0; - return terrno; - } - uvBuf->base = p->buf + p->len; - uvBuf->len = p->left; - } - } - return 0; -} +// int transSockInfo2Str(struct sockaddr* sockname, char* dst) { +// return 0; +// struct sockaddr_in addr = *(struct sockaddr_in*)sockname; + +// char buf[20] = {0}; +// int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf)); +// sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); +// return r; +//} +// int32_t transInitBuffer(SConnBuffer* buf) { +// buf->buf = taosMemoryCalloc(1, BUFFER_CAP); +// if (buf->buf == NULL) { +// return terrno; +// } + +// buf->cap = BUFFER_CAP; +// buf->left = -1; +// buf->len = 0; +// buf->total = 0; +// buf->invalid = 0; +// return 0; +//} +// int32_t transDestroyBuffer(SConnBuffer* p) { +// taosMemoryFree(p->buf); +// p->buf = NULL; +// return 0; +//} + +// int32_t transClearBuffer(SConnBuffer* buf) { +// SConnBuffer* p = buf; +// if (p->cap > BUFFER_CAP) { +// p->cap = BUFFER_CAP; +// p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); +// if (p->buf == NULL) { +// return terrno; +// } +// } +// p->left = -1; +// p->len = 0; +// p->total = 0; +// p->invalid = 0; +// return 0; +//} + +// int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { +// static const int HEADSIZE = sizeof(STransMsgHead); +// return 0; +// int32_t code = 0; +// SConnBuffer* p = connBuf; +// if (p->left != 0 || p->total <= 0) { +// return TSDB_CODE_INVALID_MSG; +// } +// int total = p->total; +// if (total >= HEADSIZE && !p->invalid) { +// *buf = taosMemoryCalloc(1, total); +// if (*buf == NULL) { +// return terrno; +// } +// memcpy(*buf, p->buf, total); +// if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { +// return code; +// } +// } else { +// total = -1; +// return TSDB_CODE_INVALID_MSG; +// } +// return total; +//} + +// int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { +// // SConnBuffer* p = connBuf; +// // if (p->total < p->len) { +// // int left = p->len - p->total; +// // memmove(p->buf, p->buf + p->total, left); +// // p->left = -1; +// // p->total = 0; +// // p->len = left; +// // } else if (p->total == p->len) { +// // p->left = -1; +// // p->total = 0; +// // p->len = 0; +// // if (p->cap > BUFFER_CAP) { +// // if (resetBuf) { +// // p->cap = BUFFER_CAP; +// // p->buf = taosMemoryRealloc(p->buf, p->cap); +// // if (p->buf == NULL) { +// // return terrno; +// // } +// // } +// // } +// // } else { +// // tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, +// // tstrerror(TSDB_CODE_INVALID_MSG)); return TSDB_CODE_INVALID_MSG; +// // } +// return 0; +// } +// int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { +// // /* +// // * formate of data buffer: +// // * |<--------------------------data from socket------------------------------->| +// // * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user +// // * info--->| +// // */ +// // SConnBuffer* p = connBuf; +// // uvBuf->base = p->buf + p->len; +// // if (p->left == -1) { +// // uvBuf->len = p->cap - p->len; +// // } else { +// // if (p->left < p->cap - p->len) { +// // uvBuf->len = p->left; +// // } else { +// // p->cap = p->left + p->len; +// // p->buf = taosMemoryRealloc(p->buf, p->cap); +// // if (p->buf == NULL) { +// // uvBuf->base = NULL; +// // uvBuf->len = 0; +// // return terrno; +// // } +// // uvBuf->base = p->buf + p->len; +// // uvBuf->len = p->left; +// // } +// // } +// return 0; +// } // check whether already read complete -bool transReadComplete(SConnBuffer* connBuf) { - SConnBuffer* p = connBuf; - if (p->len >= sizeof(STransMsgHead)) { - if (p->left == -1) { - STransMsgHead head; - memcpy((char*)&head, connBuf->buf, sizeof(head)); - int32_t msgLen = (int32_t)htonl(head.msgLen); - p->total = msgLen; - p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER; - } - if (p->total >= p->len) { - p->left = p->total - p->len; - } else { - p->left = 0; - } - } - return (p->left == 0 || p->invalid) ? true : false; -} - -int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { -#if defined(WINDOWS) || defined(DARWIN) -#else - return uv_tcp_keepalive(stream, 1, keepalive); -#endif - return uv_tcp_nodelay(stream, 1); - // int ret = uv_tcp_keepalive(stream, 5, 60); -} - -int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { - SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); - if (pool == NULL) { - return terrno; - // return NULL; - } - int32_t code = 0; - - pool->nAsync = sz; - pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); - if (pool->asyncs == NULL) { - taosMemoryFree(pool); - return terrno; - } - - int i = 0, err = 0; - for (i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - - SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); - if (item == NULL) { - code = terrno; - break; - } - item->pThrd = arg; - QUEUE_INIT(&item->qmsg); - code = taosThreadMutexInit(&item->mtx, NULL); - if (code) { - taosMemoryFree(item); - break; - } - - async->data = item; - err = uv_async_init(loop, async, cb); - if (err != 0) { - tError("failed to init async, reason:%s", uv_err_name(err)); - code = TSDB_CODE_THIRDPARTY_ERROR; - break; - } - } - - if (i != pool->nAsync) { - transAsyncPoolDestroy(pool); - pool = NULL; - } - - *pPool = pool; - return 0; - // return pool; -} - -void transAsyncPoolDestroy(SAsyncPool* pool) { - if (pool == NULL) return; - - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - if (item == NULL) continue; +// bool transReadComplete(SConnBuffer* connBuf) { +// SConnBuffer* p = connBuf; +// return false; +// if (p->len >= sizeof(STransMsgHead)) { +// if (p->left == -1) { +// STransMsgHead head; +// memcpy((char*)&head, connBuf->buf, sizeof(head)); +// int32_t msgLen = (int32_t)htonl(head.msgLen); +// p->total = msgLen; +// p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER; +// } +// if (p->total >= p->len) { +// p->left = p->total - p->len; +// } else { +// p->left = 0; +// } +// } +// return (p->left == 0 || p->invalid) ? true : false; +//} + +// int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { +// #if defined(WINDOWS) || defined(DARWIN) +// #else +// return uv_tcp_keepalive(stream, 1, keepalive); +// #endif +// return uv_tcp_nodelay(stream, 1); +// // int ret = uv_tcp_keepalive(stream, 5, 60); +// } - TAOS_UNUSED(taosThreadMutexDestroy(&item->mtx)); - taosMemoryFree(item); - } - taosMemoryFree(pool->asyncs); - taosMemoryFree(pool); -} -bool transAsyncPoolIsEmpty(SAsyncPool* pool) { - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; - } - return true; -} -int transAsyncSend(SAsyncPool* pool, queue* q) { - if (atomic_load_8(&pool->stop) == 1) { - return TSDB_CODE_RPC_ASYNC_MODULE_QUIT; - } - int idx = pool->index % pool->nAsync; +// int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { +// SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); +// if (pool == NULL) { +// return terrno; +// // return NULL; +// } +// int32_t code = 0; + +// pool->nAsync = sz; +// pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); +// if (pool->asyncs == NULL) { +// taosMemoryFree(pool); +// return terrno; +// } + +// int i = 0, err = 0; +// for (i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); + +// SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); +// if (item == NULL) { +// code = terrno; +// break; +// } +// item->pThrd = arg; +// QUEUE_INIT(&item->qmsg); +// code = taosThreadMutexInit(&item->mtx, NULL); +// if (code) { +// taosMemoryFree(item); +// break; +// } + +// async->data = item; +// err = uv_async_init(loop, async, cb); +// if (err != 0) { +// tError("failed to init async, reason:%s", uv_err_name(err)); +// code = TSDB_CODE_THIRDPARTY_ERROR; +// break; +// } +// } + +// if (i != pool->nAsync) { +// transAsyncPoolDestroy(pool); +// pool = NULL; +// } + +// *pPool = pool; +// return 0; +// // return pool; +// } - // no need mutex here - if (pool->index++ > pool->nAsync * 2000) { - pool->index = 0; - } - uv_async_t* async = &(pool->asyncs[idx]); - SAsyncItem* item = async->data; +// void transAsyncPoolDestroy(SAsyncPool* pool) { +// if (pool == NULL) return; - if (taosThreadMutexLock(&item->mtx) != 0) { - tError("failed to lock mutex"); - } +// for (int i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); +// SAsyncItem* item = async->data; +// if (item == NULL) continue; - QUEUE_PUSH(&item->qmsg, q); - TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx)); - int ret = uv_async_send(async); - if (ret != 0) { - tError("failed to send async,reason:%s", uv_err_name(ret)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - return 0; -} +// TAOS_UNUSED(taosThreadMutexDestroy(&item->mtx)); +// taosMemoryFree(item); +// } +// taosMemoryFree(pool->asyncs); +// taosMemoryFree(pool); +// } +// bool transAsyncPoolIsEmpty(SAsyncPool* pool) { +// for (int i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); +// SAsyncItem* item = async->data; +// if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; +// } +// return true; +// } +// int transAsyncSend(SAsyncPool* pool, queue* q) { +// if (atomic_load_8(&pool->stop) == 1) { +// return TSDB_CODE_RPC_ASYNC_MODULE_QUIT; +// } +// int idx = pool->index % pool->nAsync; + +// // no need mutex here +// if (pool->index++ > pool->nAsync * 2000) { +// pool->index = 0; +// } +// uv_async_t* async = &(pool->asyncs[idx]); +// SAsyncItem* item = async->data; + +// if (taosThreadMutexLock(&item->mtx) != 0) { +// tError("failed to lock mutex"); +// } + +// QUEUE_PUSH(&item->qmsg, q); +// TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx)); +// int ret = uv_async_send(async); +// if (ret != 0) { +// tError("failed to send async,reason:%s", uv_err_name(ret)); +// return TSDB_CODE_THIRDPARTY_ERROR; +// } +// return 0; +// } void transCtxInit(STransCtx* ctx) { // init transCtx @@ -1307,199 +1310,199 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { return ret; } -void transReqQueueInit(queue* q) { - // init req queue - QUEUE_INIT(q); -} -void* transReqQueuePush(queue* q) { - STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); - if (req == NULL) { - return NULL; - } - req->wreq.data = req; - QUEUE_PUSH(q, &req->q); - return &req->wreq; -} -void* transReqQueueRemove(void* arg) { - void* ret = NULL; - uv_write_t* wreq = arg; - - STransReq* req = wreq ? wreq->data : NULL; - if (req == NULL) return NULL; - QUEUE_REMOVE(&req->q); - - ret = wreq && wreq->handle ? wreq->handle->data : NULL; - taosMemoryFree(req); - - return ret; -} -void transReqQueueClear(queue* q) { - while (!QUEUE_IS_EMPTY(q)) { - queue* h = QUEUE_HEAD(q); - QUEUE_REMOVE(h); - STransReq* req = QUEUE_DATA(h, STransReq, q); - taosMemoryFree(req); - } -} - -int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { - queue->q = taosArrayInit(2, sizeof(void*)); - if (queue->q == NULL) { - return terrno; - } - queue->freeFunc = (void (*)(const void*))freeFunc; +// void transReqQueueInit(queue* q) { +// // init req queue +// QUEUE_INIT(q); +// } +// void* transReqQueuePush(queue* q) { +// STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); +// if (req == NULL) { +// return NULL; +// } +// req->wreq.data = req; +// QUEUE_PUSH(q, &req->q); +// return &req->wreq; +// } +// void* transReqQueueRemove(void* arg) { +// void* ret = NULL; +// uv_write_t* wreq = arg; - return 0; -} -bool transQueuePush(STransQueue* queue, void* arg) { - if (queue->q == NULL) { - return true; - } - if (taosArrayPush(queue->q, &arg) == NULL) { - return false; - } - if (taosArrayGetSize(queue->q) > 1) { - return false; - } - return true; -} -void* transQueuePop(STransQueue* queue) { - if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { - return NULL; - } - void* ptr = taosArrayGetP(queue->q, 0); - taosArrayRemove(queue->q, 0); - return ptr; -} -int32_t transQueueSize(STransQueue* queue) { - if (queue->q == NULL) { - return 0; - } - return taosArrayGetSize(queue->q); -} -void* transQueueGet(STransQueue* queue, int i) { - if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { - return NULL; - } - if (i >= taosArrayGetSize(queue->q)) { - return NULL; - } +// STransReq* req = wreq ? wreq->data : NULL; +// if (req == NULL) return NULL; +// QUEUE_REMOVE(&req->q); - void* ptr = taosArrayGetP(queue->q, i); - return ptr; -} +// ret = wreq && wreq->handle ? wreq->handle->data : NULL; +// taosMemoryFree(req); -void* transQueueRm(STransQueue* queue, int i) { - if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { - return NULL; - } - if (i >= taosArrayGetSize(queue->q)) { - return NULL; - } - void* ptr = taosArrayGetP(queue->q, i); - taosArrayRemove(queue->q, i); - return ptr; -} +// return ret; +// } +// void transReqQueueClear(queue* q) { +// while (!QUEUE_IS_EMPTY(q)) { +// queue* h = QUEUE_HEAD(q); +// QUEUE_REMOVE(h); +// STransReq* req = QUEUE_DATA(h, STransReq, q); +// taosMemoryFree(req); +// } +// } -bool transQueueEmpty(STransQueue* queue) { - if (queue->q == NULL) { - return true; - } - return taosArrayGetSize(queue->q) == 0; -} -void transQueueClear(STransQueue* queue) { - if (queue->freeFunc != NULL) { - for (int i = 0; i < taosArrayGetSize(queue->q); i++) { - void* p = taosArrayGetP(queue->q, i); - queue->freeFunc(p); - } - } - taosArrayClear(queue->q); -} -void transQueueDestroy(STransQueue* queue) { - transQueueClear(queue); - taosArrayDestroy(queue->q); -} +// int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { +// queue->q = taosArrayInit(2, sizeof(void*)); +// if (queue->q == NULL) { +// return terrno; +// } +// queue->freeFunc = (void (*)(const void*))freeFunc; -static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { - SDelayTask* arg1 = container_of(a, SDelayTask, node); - SDelayTask* arg2 = container_of(b, SDelayTask, node); - if (arg1->execTime > arg2->execTime) { - return 0; - } else { - return 1; - } -} +// return 0; +// } +// bool transQueuePush(STransQueue* queue, void* arg) { +// if (queue->q == NULL) { +// return true; +// } +// if (taosArrayPush(queue->q, &arg) == NULL) { +// return false; +// } +// if (taosArrayGetSize(queue->q) > 1) { +// return false; +// } +// return true; +// } +// void* transQueuePop(STransQueue* queue) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// void* ptr = taosArrayGetP(queue->q, 0); +// taosArrayRemove(queue->q, 0); +// return ptr; +// } +// int32_t transQueueSize(STransQueue* queue) { +// if (queue->q == NULL) { +// return 0; +// } +// return taosArrayGetSize(queue->q); +// } +// void* transQueueGet(STransQueue* queue, int i) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// if (i >= taosArrayGetSize(queue->q)) { +// return NULL; +// } + +// void* ptr = taosArrayGetP(queue->q, i); +// return ptr; +// } -static void transDQTimeout(uv_timer_t* timer) { - SDelayQueue* queue = timer->data; - tTrace("timer %p timeout", timer); - uint64_t timeout = 0; - int64_t current = taosGetTimestampMs(); - do { - HeapNode* minNode = heapMin(queue->heap); - if (minNode == NULL) break; - SDelayTask* task = container_of(minNode, SDelayTask, node); - if (task->execTime <= current) { - heapRemove(queue->heap, minNode); - task->func(task->arg); - taosMemoryFree(task); - timeout = 0; - } else { - timeout = task->execTime - current; - break; - } - } while (1); - if (timeout != 0) { - TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); - } -} -int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { return 0; } +// void* transQueueRm(STransQueue* queue, int i) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// if (i >= taosArrayGetSize(queue->q)) { +// return NULL; +// } +// void* ptr = taosArrayGetP(queue->q, i); +// taosArrayRemove(queue->q, i); +// return ptr; +// } -void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { return; } -void transDQCancel(SDelayQueue* queue, SDelayTask* task) { return; } +// bool transQueueEmpty(STransQueue* queue) { +// if (queue->q == NULL) { +// return true; +// } +// return taosArrayGetSize(queue->q) == 0; +// } +// void transQueueClear(STransQueue* queue) { +// if (queue->freeFunc != NULL) { +// for (int i = 0; i < taosArrayGetSize(queue->q); i++) { +// void* p = taosArrayGetP(queue->q, i); +// queue->freeFunc(p); +// } +// } +// taosArrayClear(queue->q); +// } +// void transQueueDestroy(STransQueue* queue) { +// transQueueClear(queue); +// taosArrayDestroy(queue->q); +// } -SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { return NULL; } +// static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { +// SDelayTask* arg1 = container_of(a, SDelayTask, node); +// SDelayTask* arg2 = container_of(b, SDelayTask, node); +// if (arg1->execTime > arg2->execTime) { +// return 0; +// } else { +// return 1; +// } +// } -void transPrintEpSet(SEpSet* pEpSet) { - if (pEpSet == NULL) { - tTrace("NULL epset"); - return; - } - char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset:{"); - for (int i = 0; i < pEpSet->numOfEps; i++) { - if (i == pEpSet->numOfEps - 1) { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } else { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - } - len += snprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse:%d", buf, pEpSet->inUse); -} -bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { - if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} -bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { - if (a->numOfEps != b->numOfEps) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} +// static void transDQTimeout(uv_timer_t* timer) { +// SDelayQueue* queue = timer->data; +// tTrace("timer %p timeout", timer); +// uint64_t timeout = 0; +// int64_t current = taosGetTimestampMs(); +// do { +// HeapNode* minNode = heapMin(queue->heap); +// if (minNode == NULL) break; +// SDelayTask* task = container_of(minNode, SDelayTask, node); +// if (task->execTime <= current) { +// heapRemove(queue->heap, minNode); +// task->func(task->arg); +// taosMemoryFree(task); +// timeout = 0; +// } else { +// timeout = task->execTime - current; +// break; +// } +// } while (1); +// if (timeout != 0) { +// TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); +// } +// } +// int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { return 0; } + +// void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { return; } +// void transDQCancel(SDelayQueue* queue, SDelayTask* task) { return; } + +// SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { return NULL; } + +// void transPrintEpSet(SEpSet* pEpSet) { +// if (pEpSet == NULL) { +// tTrace("NULL epset"); +// return; +// } +// char buf[512] = {0}; +// int len = snprintf(buf, sizeof(buf), "epset:{"); +// for (int i = 0; i < pEpSet->numOfEps; i++) { +// if (i == pEpSet->numOfEps - 1) { +// len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); +// } else { +// len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); +// } +// } +// len += snprintf(buf + len, sizeof(buf) - len, "}"); +// tTrace("%s, inUse:%d", buf, pEpSet->inUse); +// } +// bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { +// if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { +// return false; +// } +// for (int i = 0; i < a->numOfEps; i++) { +// if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { +// return false; +// } +// } +// return true; +// } +// bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { +// if (a->numOfEps != b->numOfEps) { +// return false; +// } +// for (int i = 0; i < a->numOfEps; i++) { +// if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { +// return false; +// } +// } +// return true; +// } bool cliMayGetAhandle(STrans* pTrans, SRpcMsg* pMsg) { int64_t seq = pMsg->info.seq; From 938fae1d642a3975857ab13161cf6197de66f9a9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Dec 2024 09:58:17 +0800 Subject: [PATCH 21/76] support acore system --- include/libs/transport/trpc.h | 4 + source/libs/function/src/tudf.c | 7 +- source/libs/transport/inc/transComm.h | 32 ++-- source/libs/transport/src/transComm.c | 187 +++++++++++----------- source/libs/transport/src/transSvr.c | 217 +++++++++++++++++++++++++- 5 files changed, 331 insertions(+), 116 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3378e4f02614..0a0e6bae5fdd 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -316,6 +316,10 @@ typedef struct SRpcInit { int8_t supportBatch; // 0: no batch, 1. batch int32_t batchSize; int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait + int32_t shareConnLimit; + int8_t shareConn; // 0: no share, 1. share + int8_t startReadTimer; + int64_t readTimeout; // s void *parent; } SRpcInit; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 4efa8764e56a..41250e3124a2 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -65,7 +65,7 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) TAOS_UDF_CHECK_PTR_RVOID(process); fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); SUdfdData *pData = process->data; - if(pData == NULL) { + if (pData == NULL) { fnError("udfd process data is NULL"); return; } @@ -410,7 +410,7 @@ typedef void *QUEUE[2]; #define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) /* Public macros. */ -#define QUEUE_DATA(ptr, type, field) ((type *)((char *)(ptr) - offsetof(type, field))) +#define QUEUE_DATA(ptr, type, field) ((type *)((char *)(ptr)-offsetof(type, field))) /* Important note: mutating the list while QUEUE_FOREACH is * iterating over its elements results in undefined behavior. @@ -1934,7 +1934,8 @@ void udfcAsyncTaskCb(uv_async_t *async) { } void cleanUpUvTasks(SUdfcProxy *udfc) { - fnDebug("clean up uv tasks") QUEUE wq; + fnDebug("clean up uv tasks"); + QUEUE wq; uv_mutex_lock(&udfc->taskQueueMutex); QUEUE_MOVE(&udfc->taskQueue, &wq); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index dd360bcc31f0..ce8955cbc625 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -328,7 +328,7 @@ void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); -int32_t transUnrefCliHandle(void* handle); +void transUnrefCliHandle(void* handle); int32_t transGetRefCount(void* handle); int32_t transReleaseCliHandle(void* handle); @@ -459,6 +459,21 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); void transDQCancel(SDelayQueue* queue, SDelayTask* task); +typedef struct { + queue node; // queue for write + queue q; // queue for reqs + uv_write_t wreq; + void* arg; +} SWReqsWrapper; + +int32_t initWQ(queue* wq); +void destroyWQ(queue* wq); +uv_write_t* allocWReqFromWQ(queue* wq, void* arg); + +void freeWReqToWQ(queue* wq, SWReqsWrapper* w); + +int32_t transSetReadOption(uv_handle_t* handle); + #endif bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b); @@ -532,21 +547,6 @@ enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING }; #define HEAP_MISS_HIT_LIMIT 100000 #define READ_TIMEOUT 100000 -typedef struct { - queue node; // queue for write - queue q; // queue for reqs - uv_write_t wreq; - void* arg; -} SWReqsWrapper; - -int32_t initWQ(queue* wq); -void destroyWQ(queue* wq); -uv_write_t* allocWReqFromWQ(queue* wq, void* arg); - -void freeWReqToWQ(queue* wq, SWReqsWrapper* w); - -int32_t transSetReadOption(uv_handle_t* handle); - #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f371a8873b7d..48decac516e2 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -907,6 +907,71 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { *ppBuf = pBuf; return len; } +int32_t initWQ(queue* wq) { + int32_t code = 0; + QUEUE_INIT(wq); + for (int i = 0; i < 4; i++) { + SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper)); + if (w == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + w->wreq.data = w; + w->arg = NULL; + QUEUE_INIT(&w->node); + QUEUE_PUSH(wq, &w->q); + } + return 0; +_exception: + destroyWQ(wq); + return code; +} +void destroyWQ(queue* wq) { + while (!QUEUE_IS_EMPTY(wq)) { + queue* h = QUEUE_HEAD(wq); + QUEUE_REMOVE(h); + SWReqsWrapper* w = QUEUE_DATA(h, SWReqsWrapper, q); + taosMemoryFree(w); + } +} + +uv_write_t* allocWReqFromWQ(queue* wq, void* arg) { + if (!QUEUE_IS_EMPTY(wq)) { + queue* node = QUEUE_HEAD(wq); + QUEUE_REMOVE(node); + SWReqsWrapper* w = QUEUE_DATA(node, SWReqsWrapper, q); + w->arg = arg; + QUEUE_INIT(&w->node); + + return &w->wreq; + } else { + SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper)); + if (w == NULL) { + return NULL; + } + w->wreq.data = w; + w->arg = arg; + QUEUE_INIT(&w->node); + return &w->wreq; + } + return NULL; +} + +void freeWReqToWQ(queue* wq, SWReqsWrapper* w) { + QUEUE_INIT(&w->node); + QUEUE_PUSH(wq, &w->q); +} + +int32_t transSetReadOption(uv_handle_t* handle) { + int32_t code = 0; + int32_t fd; + int ret = uv_fileno((uv_handle_t*)handle, &fd); + if (ret != 0) { + tWarn("failed to get fd since %s", uv_err_name(ret)); + return TSDB_CODE_THIRDPARTY_ERROR; + } + code = taosSetSockOpt2(fd); + return code; +} #else #define BUFFER_CAP 4096 @@ -971,33 +1036,34 @@ int32_t transCompressMsg(char* msg, int32_t len) { taosMemoryFree(buf); return ret; } -int32_t transDecompressMsg(char** msg, int32_t len) { - STransMsgHead* pHead = (STransMsgHead*)(*msg); - if (pHead->comp == 0) return 0; +int32_t transDecompressMsg(char** msg, int32_t* len) { + return 0; + // STransMsgHead* pHead = (STransMsgHead*)(*msg); + // if (pHead->comp == 0) return 0; - char* pCont = transContFromHead(pHead); + // char* pCont = transContFromHead(pHead); - STransCompMsg* pComp = (STransCompMsg*)pCont; - int32_t oriLen = htonl(pComp->contLen); + // STransCompMsg* pComp = (STransCompMsg*)pCont; + // int32_t oriLen = htonl(pComp->contLen); - char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); - if (buf == NULL) { - return terrno; - } + // char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); + // if (buf == NULL) { + // return terrno; + // } - STransMsgHead* pNewHead = (STransMsgHead*)buf; - int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); - memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); + // STransMsgHead* pNewHead = (STransMsgHead*)buf; + // int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, + // len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + // memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); - pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); + // pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); - taosMemoryFree(pHead); - *msg = buf; - if (decompLen != oriLen) { - return TSDB_CODE_INVALID_MSG; - } - return 0; + // taosMemoryFree(pHead); + // *msg = buf; + // if (decompLen != oriLen) { + // return TSDB_CODE_INVALID_MSG; + // } + // return 0; } void transFreeMsg(void* msg) { @@ -1779,9 +1845,10 @@ int64_t transAddExHandle(int32_t refMgt, void* p) { return taosAddRef(refMgt, p); // acquire extern handle } -int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) { +void transRemoveExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle - return taosRemoveRef(refMgt, refId); + int32_t code = taosRemoveRef(refMgt, refId); + return; } void* transAcquireExHandle(int32_t refMgt, int64_t refId) { @@ -1789,9 +1856,10 @@ void* transAcquireExHandle(int32_t refMgt, int64_t refId) { return (void*)taosAcquireRef(refMgt, refId); } -int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) { +void transReleaseExHandle(int32_t refMgt, int64_t refId) { // release extern handle - return taosReleaseRef(refMgt, refId); + int32_t code = taosReleaseRef(refMgt, refId); + return; } void transDestroyExHandle(void* handle) { if (handle == NULL) { @@ -1828,75 +1896,6 @@ int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return 0; } int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return 0; } #endif -// int32_t transGenRandomError(int32_t status) { -// STUB_RAND_NETWORK_ERR(status) -// return status; -// } - -int32_t initWQ(queue* wq) { - int32_t code = 0; - QUEUE_INIT(wq); - for (int i = 0; i < 4; i++) { - SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper)); - if (w == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _exception); - } - w->wreq.data = w; - w->arg = NULL; - QUEUE_INIT(&w->node); - QUEUE_PUSH(wq, &w->q); - } - return 0; -_exception: - destroyWQ(wq); - return code; -} -void destroyWQ(queue* wq) { - while (!QUEUE_IS_EMPTY(wq)) { - queue* h = QUEUE_HEAD(wq); - QUEUE_REMOVE(h); - SWReqsWrapper* w = QUEUE_DATA(h, SWReqsWrapper, q); - taosMemoryFree(w); - } -} - -uv_write_t* allocWReqFromWQ(queue* wq, void* arg) { - if (!QUEUE_IS_EMPTY(wq)) { - queue* node = QUEUE_HEAD(wq); - QUEUE_REMOVE(node); - SWReqsWrapper* w = QUEUE_DATA(node, SWReqsWrapper, q); - w->arg = arg; - QUEUE_INIT(&w->node); - - return &w->wreq; - } else { - SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper)); - if (w == NULL) { - return NULL; - } - w->wreq.data = w; - w->arg = arg; - QUEUE_INIT(&w->node); - return &w->wreq; - } -} - -void freeWReqToWQ(queue* wq, SWReqsWrapper* w) { - QUEUE_INIT(&w->node); - QUEUE_PUSH(wq, &w->q); -} - -int32_t transSetReadOption(uv_handle_t* handle) { - int32_t code = 0; - int32_t fd; - int ret = uv_fileno((uv_handle_t*)handle, &fd); - if (ret != 0) { - tWarn("failed to get fd since %s", uv_err_name(ret)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - code = taosSetSockOpt2(fd); - return code; -} int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) { if (pEpset == NULL) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index ee2b6451c29c..ebb1c99e5777 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -15,6 +15,18 @@ #include "transComm.h" #include "tversion.h" +#ifdef TD_ACORE +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; #ifndef TD_ACORE @@ -2073,6 +2085,56 @@ void transCloseServer(void* arg) { } #else +typedef struct { + queue q; + int32_t acceptFd; +} SFdArg; +typedef struct { + queue q; +} SFdQueue; +typedef struct SWorkThrd { + TdThread thread; + // //uv_connect_t connect_req; + // uv_pipe_t* pipe; + // uv_os_fd_t fd; + // uv_loop_t* loop; + // SAsyncPool* asyncPool; + queue msg; + + queue conn; + void *pInst; + bool quit; + + // SIpWhiteListTab* pWhiteList; + int64_t whiteListVer; + int8_t enableIpWhiteList; + + int32_t connRefMgt; + + int32_t pipe_fd[2]; // + int32_t pipe_queue_fd[2]; + int32_t client_count; + int8_t inited; + TdThreadMutex mutex; + SFdQueue fdQueue; + +} SWorkThrd; +typedef struct SServerObj { + TdThread thread; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd **pThreadObj; + + // uv_pipe_t pipeListen; + // uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; + // uv_async_t* pAcceptAsync; // just to quit from from accept thread + int32_t serverFd; + bool inited; +} SServerObj; + int32_t transReleaseSrvHandle(void *handle) { return 0; } void transRefSrvHandle(void *handle) { return; } @@ -2088,16 +2150,165 @@ int32_t transSendResponse(STransMsg *msg) { return 0; } int32_t svrVer = 0; - code = taosVersionStrToInt(version, &svrVer); + code = taosVersionStrToInt(td_version, &svrVer); msg->info.cliVer = svrVer; msg->type = msg->info.connType; return transSendResp(msg); } int32_t transRegisterMsg(const STransMsg *msg) { return 0; } int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } -void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *shandle) { - int32_t code = 0; + +void *transAcceptThread(void *arg) { + int32_t code = 0; + + setThreadName("trans-accept-work"); + SServerObj *srv = arg; + struct sockaddr_in client_addr; + socklen_t client_addr_len = sizeof(client_addr); + int32_t workerIdx = 0; + + while (1) { + int32_t client_fd = accept(srv->serverFd, (struct sockaddr *)&client_addr, &client_addr_len); + if (client_fd < 0) { + tError("failed to accept since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + continue; + } + workerIdx = (workerIdx + 1) % srv->numOfThreads; + SWorkThrd *pThrd = srv->pThreadObj[workerIdx]; + taosThreadMutexLock(&pThrd->mutex); + + SFdArg *arg = taosMemoryCalloc(1, sizeof(SFdArg)); + QUEUE_PUSH(&pThrd->fdQueue.q, &arg->q); + code = write(pThrd->pipe_fd[1], "1", 1); + + taosThreadMutexUnlock(&pThrd->mutex); + + if (1 != sizeof(client_fd)) { + tError("failed to write to pipe since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + continue; + } + } + return NULL; +} +void *transWorkerThread(void *arg) { + int32_t code = 0; + setThreadName("trans-svr-work"); + SWorkThrd *pThrd = (SWorkThrd *)arg; + while (1) { + fd_set read_fds; + FD_ZERO(&read_fds); + + FD_SET(pThrd->pipe_fd[0], &read_fds); + int32_t max_fd = pThrd->pipe_fd[0]; + int32_t nActivitys = select(max_fd + 1, &read_fds, NULL, NULL, NULL); + if (nActivitys < 0 && errno != EINTR) { + tError("failed to select since %s", tstrerror(errno)); + } + if (FD_ISSET(pThrd->pipe_fd[0], &read_fds)) { + char buf[2] = {0}; + int32_t nBytes = read(pThrd->pipe_fd[0], buf, sizeof(buf)); + + taosThreadMutexLock(&pThrd->mutex); + queue rq; + QUEUE_INIT(&rq); + QUEUE_MOVE(&pThrd->fdQueue.q, &rq); + while (!QUEUE_IS_EMPTY(rq)) { + queue *el = QUEUE_HEAD(rq); + QUEUE_REMOVE(el); + SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); + // TODO create new socket + taosMemoryFree(pArg); + } + taosThreadMutexUnlock(&pThrd->mutex); + } + } + + return NULL; +} +static int32_t addHandleToAcceptloop(void *arg) { + // impl later + int32_t code = 0; + SServerObj *srv = arg; + int32_t server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server_fd < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + srv->serverFd = server_fd; + + int32_t opt = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = inet_addr("0.0.0.0"); + server_addr.sin_port = htons(srv->port); + + if (bind(server_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + if (listen(server_fd, 128) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); + return code; +} +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { + int32_t code = 0; + + SServerObj *srv = taosMemoryCalloc(1, sizeof(SServerObj)); + if (srv == NULL) { + code = terrno; + tError("failed to init server since: %s", tstrerror(code)); return NULL; + } + + srv->ip = ip; + srv->port = port; + srv->numOfThreads = numOfThreads; + srv->workerIdx = 0; + srv->numOfWorkerReady = 0; + // srv->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); + srv->pThreadObj = (SWorkThrd **)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd *)); + // srv->pipe = (uv_pipe_t **)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t *)); + if (srv->pThreadObj == NULL) { + code = terrno; + return NULL; + } + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrd *thrd = (SWorkThrd *)taosMemoryCalloc(1, sizeof(SWorkThrd)); + thrd->pInst = pInit; + thrd->quit = false; + thrd->pInst = pInit; + thrd->connRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + if (thrd->connRefMgt < 0) { + code = thrd->connRefMgt; + goto End; + } + + QUEUE_INIT(&thrd->fdQueue.q); + if (pipe(thrd->pipe_fd) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto End; + } + + int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); + if (err == 0) { + tDebug("success to create worker-thread:%d", i); + } else { + // TODO: clear all other resource later + tError("failed to create worker-thread:%d", i); + goto End; + } + thrd->inited = 1; + } + code = addHandleToAcceptloop(srv); + if (code != 0) { + goto End; + } + return NULL; +End: + return NULL; } void transCloseServer(void *arg) { // impl later From e2f94235541cc6e8ebf78ae952a0925d8f66d066 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 10:35:08 +0800 Subject: [PATCH 22/76] refactor code --- source/libs/transport/src/transImpl2.c | 624 +++++++++++++++++++++++++ source/libs/transport/src/transSvr.c | 220 +-------- 2 files changed, 627 insertions(+), 217 deletions(-) create mode 100644 source/libs/transport/src/transImpl2.c diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c new file mode 100644 index 000000000000..322ecfe914d0 --- /dev/null +++ b/source/libs/transport/src/transImpl2.c @@ -0,0 +1,624 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "transComm.h" +#include "tversion.h" + +#ifdef TD_ACORE +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#define DIV_ROUNDUP(x, y) (((x) + ((y)-1)) / (y)) + +#define EVT_TIMEOUT 0x01 +#define EVT_READ 0x02 +#define EVT_WRITE 0x04 +#define EVT_SIGNAL 0x08 +#define EVT_ASYNC 0x10 + +/* How many bytes to allocate for N fds? */ +#define SELECT_ALLOC_SIZE(n) (DIV_ROUNDUP(n, NFDBITS) * sizeof(fd_mask)) + +typedef void (*AsyncCb)(void *async, int32_t status); + +typedef struct { + void *data; + int32_t sendFd; + AsyncCb cb; + queue q; + TdThreadMutex mutex; +} SAsyncHandle; +typedef struct { + queue q; + int32_t acceptFd; +} SFdArg; +typedef struct { + queue q; +} SFdQueue; +typedef struct SWorkThrd { + TdThread thread; + // //uv_connect_t connect_req; + // uv_pipe_t* pipe; + // uv_os_fd_t fd; + // uv_loop_t* loop; + // SAsyncPool* asyncPool; + queue msg; + + queue conn; + void *pInst; + bool quit; + + // SIpWhiteListTab* pWhiteList; + int64_t whiteListVer; + int8_t enableIpWhiteList; + + int32_t connRefMgt; + + int32_t pipe_fd[2]; // + int32_t pipe_queue_fd[2]; + int32_t client_count; + int8_t inited; + TdThreadMutex mutex; + SFdQueue fdQueue; + + SAsyncHandle *notifyNewConnHandle; + SAsyncHandle *handle; +} SWorkThrd2; +typedef struct SServerObj { + TdThread thread; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd2 **pThreadObj; + + // uv_pipe_t pipeListen; + // uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; + // uv_async_t* pAcceptAsync; // just to quit from from accept thread + int32_t serverFd; + bool inited; +} SServerObj2; + +typedef void (*__sendCb)(SFdArg *arg, int32_t status); +typedef void (*__readCb)(SFdArg *arg, int32_t status); +typedef void (*__asyncCb)(SFdArg *arg, int32_t status); + +enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T }; + +typedef struct { + int32_t fd; + void *data; + int32_t event; + + __sendCb sentFn; + __readCb readFn; + __asyncCb asyncFn; + + int8_t evtType; + void *arg; +} SFdCbArg; +typedef struct { + int32_t evtFds; + int32_t evtFdsSize; + int32_t resizeOutSets; + fd_set *evtReadSetIn; + fd_set *evtWriteSetIn; + fd_set *evtReadSetOut; + fd_set *evtWriteSetOut; + SHashObj *pFdTable; +} SEvtMgt; + +static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); +static int32_t evtMgtCreate(SEvtMgt **pOpt); +static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv); +static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); +static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); +static void evtMgtDestroy(SEvtMgt *pOpt); + +int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd); + +static int32_t evtMgtCreate(SEvtMgt **pOpt) { + int32_t code = 0; + SEvtMgt *pRes = taosMemoryCalloc(1, sizeof(SEvtMgt)); + if (pRes == NULL) { + *pOpt = NULL; + return terrno; + } + pRes->evtFds = 0; + pRes->evtFdsSize = 0; + pRes->resizeOutSets = 0; + pRes->evtReadSetIn = NULL; + pRes->evtWriteSetIn = NULL; + pRes->evtReadSetOut = NULL; + pRes->evtWriteSetOut = NULL; + + code = evtMgtResize(pRes, (32 + 1)); + if (code != 0) { + evtMgtDestroy(pRes); + } else { + *pOpt = pRes; + } + + return code; +} + +// int32_t selectUtilRange() + +int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg) { + int32_t code = 0; + if (pArg->evtType == EVT_CONN_T) { + SAsyncHandle *handle = pArg->arg; + handle->cb(handle, 0); + } else if (pArg->evtType == EVT_ASYNC_T) { + SAsyncHandle *handle = pArg->arg; + handle->cb(handle, 0); + } else { + } + return code; +} +int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { + int32_t code = 0; + if (res & EVT_READ) { + // handle read + } + if (res & EVT_WRITE) { + // handle write + } + SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (pArg == NULL) { + return TAOS_SYSTEM_ERROR(EBADF); + } + code = evtMgtHandleImpl(pOpt, pArg); + return code; +} +static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { + int32_t code = 0, res = 0, i, j, nfds = 0; + if (pOpt->resizeOutSets) { + fd_set *readSetOut = NULL, *writeSetOut = NULL; + int32_t sz = pOpt->evtFdsSize; + readSetOut = taosMemoryRealloc(pOpt->evtReadSetOut, sz); + if (readSetOut == NULL) { + return terrno; + } + pOpt->evtReadSetOut = readSetOut; + + writeSetOut = taosMemoryRealloc(pOpt->evtWriteSetOut, sz); + if (writeSetOut == NULL) { + return terrno; + } + + pOpt->evtWriteSetOut = writeSetOut; + pOpt->resizeOutSets = 0; + } + + memcpy(pOpt->evtReadSetOut, pOpt->evtReadSetIn, pOpt->evtFdsSize); + memcpy(pOpt->evtWriteSetOut, pOpt->evtWriteSetIn, pOpt->evtFdsSize); + + nfds = pOpt->evtFds + 1; + // TODO lock or not + code = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); + if (code < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + + for (j = 0; j < nfds; j++) { + res = 0; + if (FD_ISSET(j, pOpt->evtReadSetOut)) { + res |= EVT_READ; + } + if (FD_ISSET(j, pOpt->evtWriteSetOut)) { + res |= EVT_WRITE; + } + code = evtMgtHandle(pOpt, res, i); + if (code != 0) { + tError("failed to handle fd %d since %s", i, tstrerror(code)); + } + } + + return code; +} + +static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap) { + int32_t code = 0; + + fd_set *readSetIn = NULL; + fd_set *writeSetIn = NULL; + + readSetIn = taosMemoryRealloc(pOpt->evtReadSetIn, cap); + if (readSetIn == NULL) { + return terrno; + } + pOpt->evtReadSetIn = readSetIn; + + writeSetIn = taosMemoryRealloc(pOpt->evtWriteSetIn, cap); + if (writeSetIn == NULL) { + return terrno; + } + + pOpt->evtWriteSetIn = writeSetIn; + pOpt->resizeOutSets = 1; + + memset((char *)pOpt->evtReadSetIn + pOpt->evtFdsSize, 0, cap - pOpt->evtFdsSize); + memset((char *)pOpt->evtWriteSetIn + pOpt->evtFdsSize, 0, cap - pOpt->evtFdsSize); + + pOpt->evtFdsSize = cap; + return code; +} + +static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { + // add new fd to the set + int32_t code = 0; + if (pOpt->evtFds < fd) { + int32_t fdSize = pOpt->evtFdsSize; + + if (fdSize < (int32_t)sizeof(fd_mask)) { + fdSize = (int32_t)sizeof(fd_mask); + } + while (fdSize < (int32_t)SELECT_ALLOC_SIZE(fd + 1)) { + fdSize *= 2; + } + if (fdSize != pOpt->evtFdsSize) { + if (evtMgtResize(pOpt, fdSize)) { + return -1; + } + pOpt->evtFds = fd; + } + } + if (events & EVT_READ) { + FD_SET(fd, pOpt->evtReadSetIn); + } + + if (events & EVT_WRITE) { + FD_SET(fd, pOpt->evtWriteSetIn); + } + code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); + return 0; +} + +static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { + int32_t code = 0; + ASSERT((events & EVT_SIGNAL) == 0); + + if (pOpt->evtFds < fd) { + return TAOS_SYSTEM_ERROR(EBADF); + } + + if (events & EVT_READ) { + FD_CLR(fd, pOpt->evtReadSetIn); + } + + if (events & EVT_WRITE) { + FD_CLR(fd, pOpt->evtWriteSetIn); + } + SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (pArg == NULL) { + // TODO, destroy pArg + } else { + code = taosHashRemove(pOpt->pFdTable, &fd, sizeof(fd)); + } + return code; +} +static void evtMgtDestroy(SEvtMgt *pOpt) { + if (pOpt == NULL) return; + + if (pOpt->evtReadSetIn) { + taosMemoryFree(pOpt->evtReadSetIn); + } + + if (pOpt->evtWriteSetIn) { + taosMemoryFree(pOpt->evtWriteSetIn); + } + + if (pOpt->evtReadSetOut) { + taosMemoryFree(pOpt->evtReadSetOut); + } + + if (pOpt->evtWriteSetOut) { + taosMemoryFree(pOpt->evtWriteSetOut); + } + + taosHashCleanup(pOpt->pFdTable); + taosMemoryFree(pOpt); +} + +static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, AsyncCb cb, int8_t evtType) { + int32_t code = 0; + SAsyncHandle *pAsync = taosMemoryCalloc(1, sizeof(SAsyncHandle)); + if (pAsync == NULL) { + return terrno; + } + pAsync->data = pOpt; + + taosThreadMutexInit(&pAsync->mutex, NULL); + pAsync->sendFd = fd[1]; + pAsync->cb = cb; + + SFdCbArg arg = {.evtType = evtType, .arg = pAsync, .fd = fd[0]}; + arg.arg = pAsync; + + code = evtMgtAdd(pOpt, fd[0], EVT_READ, &arg); + if (code != 0) { + taosMemoryFree(pAsync); + return code; + } + QUEUE_INIT(&pAsync->q); + *async = pAsync; + return code; +} + +static int32_t evtAsyncSend(SAsyncHandle *async, queue *q) { + int32_t code = 0; + taosThreadMutexLock(&async->mutex); + QUEUE_PUSH(&async->q, q); + taosThreadMutexUnlock(&async->mutex); + int32_t nBytes = write(async->sendFd, "1", 1); + if (nBytes != 1) { + return TAOS_SYSTEM_ERROR(errno); + } + + return code; +} + +void *transAcceptThread(void *arg) { + int32_t code = 0; + + setThreadName("trans-accept-work"); + SServerObj2 *srv = arg; + struct sockaddr_in client_addr; + socklen_t client_addr_len = sizeof(client_addr); + int32_t workerIdx = 0; + + while (1) { + int32_t client_fd = accept(srv->serverFd, (struct sockaddr *)&client_addr, &client_addr_len); + if (client_fd < 0) { + tError("failed to accept since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + continue; + } + workerIdx = (workerIdx + 1) % srv->numOfThreads; + SWorkThrd2 *pThrd = srv->pThreadObj[workerIdx]; + + SFdArg *arg = taosMemoryCalloc(1, sizeof(SFdArg)); + if (arg == NULL) { + tError("failed to create fd arg since %s", tstrerror(terrno)); + taosThreadMutexUnlock(&pThrd->mutex); + continue; + } + arg->acceptFd = client_fd; + QUEUE_INIT(&arg->q); + + code = evtAsyncSend(pThrd->notifyNewConnHandle, &arg->q); + if (code != 0) { + tError("failed to send async msg since %s", tstrerror(code)); + } + } + return NULL; +} + +void evtNewConnNotify(void *async, int32_t status) { + int32_t code = 0; + + SAsyncHandle *handle = async; + SEvtMgt *pEvtMgt = handle->data; + + queue wq; + QUEUE_INIT(&wq); + + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); + + SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pArg, .fd = pArg->acceptFd}; + code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); + taosMemoryFree(pArg); + } + + return; +} +void evtHandleReq(void *async, int32_t status) { + int32_t code = 0; + + SAsyncHandle *handle = async; + SEvtMgt *pEvtMgt = handle->data; + + queue wq; + QUEUE_INIT(&wq); + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); + // code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, NULL); + taosMemoryFree(pArg); + } + + return; +} + +void *transWorkerThread(void *arg) { + int32_t code = 0; + int32_t line = 0; + + struct timeval tv = {5, 0}; + setThreadName("trans-svr-work"); + SWorkThrd2 *pThrd = (SWorkThrd2 *)arg; + + SEvtMgt *pOpt = NULL; + + code = evtMgtCreate(&pOpt); + if (code != 0) { + tError("failed to create select op since %s", tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotify, EVT_CONN_T); + if (code != 0) { + tError("failed to create select op since %s", tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + + code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->handle, evtHandleReq, EVT_ASYNC_T); + if (code != 0) { + tError("failed to create select op since %s", tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + + while (!pThrd->quit) { + code = evtMgtDispath(pOpt, &tv); + if (code != 0) { + tError("failed to dispatch since %s", tstrerror(code)); + continue; + } + } +_end: + if (code != 0) { + tError("failed to do work %s", tstrerror(code)); + } + evtMgtDestroy(pOpt); + return NULL; +} +static int32_t addHandleToAcceptloop(void *arg) { + // impl later + int32_t code = 0; + SServerObj2 *srv = arg; + int32_t server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server_fd < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + srv->serverFd = server_fd; + + int32_t opt = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = inet_addr("0.0.0.0"); + server_addr.sin_port = htons(srv->port); + + if (bind(server_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + if (listen(server_fd, 128) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); + return code; +} +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { + int32_t code = 0; + + SServerObj2 *srv = taosMemoryCalloc(1, sizeof(SServerObj2)); + if (srv == NULL) { + code = terrno; + tError("failed to init server since: %s", tstrerror(code)); + return NULL; + } + + srv->ip = ip; + srv->port = port; + srv->numOfThreads = numOfThreads; + srv->workerIdx = 0; + srv->numOfWorkerReady = 0; + srv->pThreadObj = (SWorkThrd2 **)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd2 *)); + if (srv->pThreadObj == NULL) { + code = terrno; + return NULL; + } + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrd2 *thrd = (SWorkThrd2 *)taosMemoryCalloc(1, sizeof(SWorkThrd2)); + thrd->pInst = pInit; + thrd->quit = false; + thrd->pInst = pInit; + thrd->connRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + if (thrd->connRefMgt < 0) { + code = thrd->connRefMgt; + goto End; + } + + QUEUE_INIT(&thrd->fdQueue.q); + if (pipe(thrd->pipe_fd) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto End; + } + + int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); + if (err == 0) { + tDebug("success to create worker-thread:%d", i); + } else { + // TODO: clear all other resource later + tError("failed to create worker-thread:%d", i); + goto End; + } + thrd->inited = 1; + } + code = addHandleToAcceptloop(srv); + if (code != 0) { + goto End; + } + return NULL; +End: + return NULL; +} + +void transCloseServer(void *arg) { + // impl later + return; +} + +// int32_t transReleaseSrvHandle(void *handle) { return 0; } +// void transRefSrvHandle(void *handle) { return; } + +// void transUnrefSrvHandle(void *handle) { return; } +// int32_t transSendResponse(STransMsg *msg) { +// // +// int32_t code = 0; +// if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { +// msg->msgType = msg->info.msgType + 1; +// } +// if (msg->info.noResp) { +// rpcFreeCont(msg->pCont); +// return 0; +// } +// int32_t svrVer = 0; +// code = taosVersionStrToInt(td_version, &svrVer); +// msg->info.cliVer = svrVer; +// msg->type = msg->info.connType; +// return transSendResp(msg); +// } +// int32_t transRegisterMsg(const STransMsg *msg) { return 0; } +// int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index ebb1c99e5777..456b9a61d3a0 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -15,18 +15,6 @@ #include "transComm.h" #include "tversion.h" -#ifdef TD_ACORE -#include -#include -#include -#include -#include -#include -#include -#include - -#endif - static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; #ifndef TD_ACORE @@ -2085,62 +2073,11 @@ void transCloseServer(void* arg) { } #else -typedef struct { - queue q; - int32_t acceptFd; -} SFdArg; -typedef struct { - queue q; -} SFdQueue; -typedef struct SWorkThrd { - TdThread thread; - // //uv_connect_t connect_req; - // uv_pipe_t* pipe; - // uv_os_fd_t fd; - // uv_loop_t* loop; - // SAsyncPool* asyncPool; - queue msg; - - queue conn; - void *pInst; - bool quit; - - // SIpWhiteListTab* pWhiteList; - int64_t whiteListVer; - int8_t enableIpWhiteList; - - int32_t connRefMgt; - - int32_t pipe_fd[2]; // - int32_t pipe_queue_fd[2]; - int32_t client_count; - int8_t inited; - TdThreadMutex mutex; - SFdQueue fdQueue; - -} SWorkThrd; -typedef struct SServerObj { - TdThread thread; - int workerIdx; - int numOfThreads; - int numOfWorkerReady; - SWorkThrd **pThreadObj; - - // uv_pipe_t pipeListen; - // uv_pipe_t** pipe; - uint32_t ip; - uint32_t port; - // uv_async_t* pAcceptAsync; // just to quit from from accept thread - int32_t serverFd; - bool inited; -} SServerObj; - int32_t transReleaseSrvHandle(void *handle) { return 0; } void transRefSrvHandle(void *handle) { return; } void transUnrefSrvHandle(void *handle) { return; } int32_t transSendResponse(STransMsg *msg) { - // int32_t code = 0; if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { msg->msgType = msg->info.msgType + 1; @@ -2158,160 +2095,9 @@ int32_t transSendResponse(STransMsg *msg) { int32_t transRegisterMsg(const STransMsg *msg) { return 0; } int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } -void *transAcceptThread(void *arg) { - int32_t code = 0; - - setThreadName("trans-accept-work"); - SServerObj *srv = arg; - struct sockaddr_in client_addr; - socklen_t client_addr_len = sizeof(client_addr); - int32_t workerIdx = 0; - - while (1) { - int32_t client_fd = accept(srv->serverFd, (struct sockaddr *)&client_addr, &client_addr_len); - if (client_fd < 0) { - tError("failed to accept since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); - continue; - } - workerIdx = (workerIdx + 1) % srv->numOfThreads; - SWorkThrd *pThrd = srv->pThreadObj[workerIdx]; - taosThreadMutexLock(&pThrd->mutex); - - SFdArg *arg = taosMemoryCalloc(1, sizeof(SFdArg)); - QUEUE_PUSH(&pThrd->fdQueue.q, &arg->q); - code = write(pThrd->pipe_fd[1], "1", 1); - - taosThreadMutexUnlock(&pThrd->mutex); - - if (1 != sizeof(client_fd)) { - tError("failed to write to pipe since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); - continue; - } - } - return NULL; -} -void *transWorkerThread(void *arg) { - int32_t code = 0; - setThreadName("trans-svr-work"); - SWorkThrd *pThrd = (SWorkThrd *)arg; - while (1) { - fd_set read_fds; - FD_ZERO(&read_fds); - - FD_SET(pThrd->pipe_fd[0], &read_fds); - int32_t max_fd = pThrd->pipe_fd[0]; - int32_t nActivitys = select(max_fd + 1, &read_fds, NULL, NULL, NULL); - if (nActivitys < 0 && errno != EINTR) { - tError("failed to select since %s", tstrerror(errno)); - } - if (FD_ISSET(pThrd->pipe_fd[0], &read_fds)) { - char buf[2] = {0}; - int32_t nBytes = read(pThrd->pipe_fd[0], buf, sizeof(buf)); - - taosThreadMutexLock(&pThrd->mutex); - queue rq; - QUEUE_INIT(&rq); - QUEUE_MOVE(&pThrd->fdQueue.q, &rq); - while (!QUEUE_IS_EMPTY(rq)) { - queue *el = QUEUE_HEAD(rq); - QUEUE_REMOVE(el); - SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); - // TODO create new socket - taosMemoryFree(pArg); - } - taosThreadMutexUnlock(&pThrd->mutex); - } - } - - return NULL; -} -static int32_t addHandleToAcceptloop(void *arg) { - // impl later - int32_t code = 0; - SServerObj *srv = arg; - int32_t server_fd = socket(AF_INET, SOCK_STREAM, 0); - if (server_fd < 0) { - return TAOS_SYSTEM_ERROR(errno); - } - srv->serverFd = server_fd; - - int32_t opt = 1; - setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); - - struct sockaddr_in server_addr; - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_addr.s_addr = inet_addr("0.0.0.0"); - server_addr.sin_port = htons(srv->port); - - if (bind(server_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - return TAOS_SYSTEM_ERROR(errno); - } - if (listen(server_fd, 128) < 0) { - return TAOS_SYSTEM_ERROR(errno); - } - code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); - return code; -} -void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { - int32_t code = 0; - - SServerObj *srv = taosMemoryCalloc(1, sizeof(SServerObj)); - if (srv == NULL) { - code = terrno; - tError("failed to init server since: %s", tstrerror(code)); - return NULL; - } - - srv->ip = ip; - srv->port = port; - srv->numOfThreads = numOfThreads; - srv->workerIdx = 0; - srv->numOfWorkerReady = 0; - // srv->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); - srv->pThreadObj = (SWorkThrd **)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd *)); - // srv->pipe = (uv_pipe_t **)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t *)); - if (srv->pThreadObj == NULL) { - code = terrno; - return NULL; - } - for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrd *thrd = (SWorkThrd *)taosMemoryCalloc(1, sizeof(SWorkThrd)); - thrd->pInst = pInit; - thrd->quit = false; - thrd->pInst = pInit; - thrd->connRefMgt = transOpenRefMgt(50000, transDestroyExHandle); - if (thrd->connRefMgt < 0) { - code = thrd->connRefMgt; - goto End; - } - - QUEUE_INIT(&thrd->fdQueue.q); - if (pipe(thrd->pipe_fd) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto End; - } - - int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); - if (err == 0) { - tDebug("success to create worker-thread:%d", i); - } else { - // TODO: clear all other resource later - tError("failed to create worker-thread:%d", i); - goto End; - } - thrd->inited = 1; - } - code = addHandleToAcceptloop(srv); - if (code != 0) { - goto End; - } - return NULL; -End: - return NULL; -} -void transCloseServer(void *arg) { - // impl later +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { return NULL; } +void transCloseServer(void *arg) { + // impl later return; } From f9502f9732cad4ed5fc28569f4218ade10153859 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 12:02:45 +0800 Subject: [PATCH 23/76] refactor code --- source/libs/transport/src/transImpl2.c | 430 ++++++++++++++++++++++++- 1 file changed, 427 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 322ecfe914d0..8d39567789a7 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -400,7 +400,6 @@ void *transAcceptThread(void *arg) { SFdArg *arg = taosMemoryCalloc(1, sizeof(SFdArg)); if (arg == NULL) { tError("failed to create fd arg since %s", tstrerror(terrno)); - taosThreadMutexUnlock(&pThrd->mutex); continue; } arg->acceptFd = client_fd; @@ -539,7 +538,8 @@ static int32_t addHandleToAcceptloop(void *arg) { code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); return code; } -void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { + +void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { int32_t code = 0; SServerObj2 *srv = taosMemoryCalloc(1, sizeof(SServerObj2)); @@ -595,10 +595,434 @@ void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, return NULL; } -void transCloseServer(void *arg) { +void transCloseServer2(void *arg) { // impl later return; } +// impl client with poll +typedef struct SCliConn { + int32_t ref; + // uv_connect_t connReq; + // uv_stream_t *stream; + + // uv_timer_t *timer; // read timer, forbidden + + void *hostThrd; + + SConnBuffer readBuf; + STransQueue reqsToSend; + STransQueue reqsSentOut; + + queue q; + // SConnList *list; + + STransCtx ctx; + bool broken; // link broken or not + ConnStatus status; // + + // SCliBatch *pBatch; + + // SDelayTask *task; + + // HeapNode node; // for heap + int8_t inHeap; + int32_t reqRefCnt; + uint32_t clientIp; + uint32_t serverIp; + + char *dstAddr; + char src[32]; + char dst[32]; + + char *ipStr; + int32_t port; + + int64_t seq; + + int8_t registered; + int8_t connnected; + SHashObj *pQTable; + int8_t userInited; + void *pInitUserReq; + + void *heap; // point to req conn heap + int32_t heapMissHit; + int64_t lastAddHeapTime; + int8_t forceDelFromHeap; + + // uv_buf_t *buf; + int32_t bufSize; + int32_t readerStart; + + queue wq; // uv_write_t queue + + queue batchSendq; + int8_t inThreadSendq; + +} SCliConn; + +typedef struct { + SCliConn *conn; + void *arg; +} SReqState; + +typedef struct { + int64_t seq; + int32_t msgType; +} SFiterArg; +typedef struct SCliReq { + SReqCtx *ctx; + queue q; + queue sendQ; + STransMsgType type; + uint64_t st; + int64_t seq; + int32_t sent; //(0: no send, 1: alread sent) + int8_t inSendQ; + STransMsg msg; + int8_t inRetry; + +} SCliReq; + +#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps >= 0 && (epSet)->inUse >= 0) +#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + if ((epSet)->numOfEps != 0) { \ + ++((epSet)->inUse); \ + (epSet)->inUse = ((epSet)->inUse) % ((epSet)->numOfEps); \ + } \ + } while (0) +typedef struct SCliThrd2 { + TdThread thread; // tid + int64_t pid; // pid + + // uv_loop_t *loop; + // SAsyncPool *asyncPool; + void *pool; // conn pool + // timer handles + SArray *timerList; + // msg queue + queue msg; + TdThreadMutex msgMtx; + // SDelayQueue *delayQueue; + // SDelayQueue *timeoutQueue; + // SDelayQueue *waitConnQueue; + uint64_t nextTimeout; // next timeout + STrans *pInst; // + + void (*destroyAhandleFp)(void *ahandle); + SHashObj *fqdn2ipCache; + SCvtAddr *pCvtAddr; + + SHashObj *failFastCache; + SHashObj *batchCache; + SHashObj *connHeapCache; + + SCliReq *stopMsg; + bool quit; + + int32_t (*initCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + int32_t (*notifyCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + int32_t (*notifyExceptCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + + SHashObj *pIdConnTable; // + + SArray *pQIdBuf; // tmp buf to avoid alloc buf; + queue batchSendSet; + int8_t thrdInited; + int32_t pipe_queue_fd[2]; + SEvtMgt *pEvtMgt; + SAsyncHandle *asyncHandle; +} SCliThrd2; + +typedef struct SCliObj2 { + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrd2 **pThreadObj; +} SCliObj2; + +static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { + if (ctx) { + taosMemoryFree(ctx->epSet); + taosMemoryFree(ctx->origEpSet); + taosMemoryFree(ctx); + } +} +static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq) { + if (pReq == NULL || pReq->inSendQ == 0) { + return; + } + QUEUE_REMOVE(&pReq->sendQ); + pReq->inSendQ = 0; +} +static void destroyThrdObj(SCliThrd2 *pThrd) { + if (pThrd == NULL) { + return; + } + if (pThrd->pEvtMgt) { + evtMgtDestroy(pThrd->pEvtMgt); + } + taosMemoryFree(pThrd); +} +static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { + int32_t line = 0; + int32_t code = 0; + + STrans *pInst = trans; + + SCliThrd2 *pThrd = (SCliThrd2 *)taosMemoryCalloc(1, sizeof(SCliThrd2)); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + code = evtMgtCreate(&pThrd->pEvtMgt); + if (code != 0) { + TAOS_CHECK_GOTO(code, &line, _end); + } + return code; +_end: + if (code != 0) { + destroyThrdObj(pThrd); + tError("%s failed to init rpc client at line %d since %s,", __func__, line, tstrerror(code)); + } + return code; +} + +FORCE_INLINE int cliRBChoseIdx(STrans *pInst) { + int32_t index = pInst->index; + if (pInst->numOfThreads == 0) { + return -1; + } + /* + * no lock, and to avoid CPU load imbalance, set limit pInst->numOfThreads * 2000; + */ + if (pInst->index++ >= pInst->numOfThreads * 2000) { + pInst->index = 0; + } + return index % pInst->numOfThreads; +} +static FORCE_INLINE SCliThrd2 *transGetWorkThrdFromHandle(STrans *trans, int64_t handle) { + SCliThrd2 *pThrd = NULL; + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), handle); + if (exh == NULL) { + return NULL; + } + taosWLockLatch(&exh->latch); + if (exh->pThrd == NULL && trans != NULL) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + exh->pThrd = ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; + } + + pThrd = exh->pThrd; + taosWUnLockLatch(&exh->latch); + transReleaseExHandle(transGetRefMgt(), handle); + + return pThrd; +} +SCliThrd2 *transGetWorkThrd(STrans *trans, int64_t handle) { + if (handle == 0) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + return ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; + } + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(trans, handle); + return pThrd; +} +static int32_t transInitMsg(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx, SCliReq **pCliMsg) { + int32_t code = 0; + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + SCliReq *pCliReq = NULL; + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } + + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + + if (ctx != NULL) pCtx->userCtx = *ctx; + + pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCliReq == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + *pCliMsg = pCliReq; + return code; +_exception: + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + taosMemoryFree(pCliReq); + return code; +} + +static FORCE_INLINE void destroyReq(void *arg) { + SCliReq *pReq = arg; + if (pReq == NULL) { + return; + } + + removeReqFromSendQ(pReq); + STraceId *trace = &pReq->msg.info.traceId; + tGDebug("free memory:%p, free ctx: %p", pReq, pReq->ctx); + + if (pReq->ctx) { + destroyReqCtx(pReq->ctx); + } + transFreeMsg(pReq->msg.pCont); + taosMemoryFree(pReq); +} +int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx) { + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + int32_t code = 0; + int64_t handle = (int64_t)pReq->info.handle; + SCliThrd2 *pThrd = transGetWorkThrd(pInst, handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception); + } + + pReq->info.qId = handle; + + SCliReq *pCliMsg = NULL; + TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, ctx, &pCliMsg), NULL, _exception); + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliMsg->q); + if (code != 0) { + destroyReq(pCliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + } + { + // TODO(weizhong): add trace log + // if ((code = transAsyncSend(pThrd->asyncPool, &(pCliMsg->q))) != 0) { + // } + } + + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return 0; + +_exception: + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (code != 0) { + tError("failed to send request since %s", tstrerror(code)); + } + return code; +} +static void *cliWorkThread2(void *arg) { + int32_t line = 0; + int32_t code = 0; + char threadName[TSDB_LABEL_LEN] = {0}; + struct timeval tv = {30, 0}; + SCliThrd2 *pThrd = (SCliThrd2 *)arg; + + pThrd->pid = taosGetSelfPthreadId(); + + tsEnableRandErr = true; + TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); + setThreadName(threadName); + + code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleReq, EVT_ASYNC_T); + TAOS_CHECK_GOTO(code, &line, _end); + + while (!pThrd->quit) { + code = evtMgtDispath(pThrd->pEvtMgt, &tv); + if (code != 0) { + tError("failed to dispatch since %s", tstrerror(code)); + continue; + } + } + + tDebug("thread quit-thread:%08" PRId64 "", pThrd->pid); + return NULL; +_end: + if (code != 0) { + tError("failed to do work %s", tstrerror(code)); + } + return NULL; +} +void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInstRef) { + int32_t code = 0; + SCliObj2 *cli = taosMemoryCalloc(1, sizeof(SCliObj2)); + if (cli == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + + STrans *pInst = pInstRef; + memcpy(cli->label, label, TSDB_LABEL_LEN); + cli->numOfThreads = numOfThreads; + cli->pThreadObj = (SCliThrd2 **)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd2 *)); + if (cli->pThreadObj == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + for (int i = 0; i < cli->numOfThreads; i++) { + SCliThrd2 *pThrd = NULL; + code = createThrdObj(pInstRef, &pThrd); + if (code != 0) { + goto _err; + } + + int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); + if (err != 0) { + destroyThrdObj(pThrd); + code = TAOS_SYSTEM_ERROR(errno); + TAOS_CHECK_GOTO(code, NULL, _err); + } else { + tDebug("success to create tranport-cli thread:%d", i); + } + pThrd->thrdInited = 1; + cli->pThreadObj[i] = pThrd; + } + +_err: + if (cli) { + for (int i = 0; i < cli->numOfThreads; i++) { + // send quit msg + destroyThrdObj(cli->pThreadObj[i]); + } + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); + } + terrno = code; + return NULL; +} +void transCloseClient(void *arg) { + int32_t code = 0; + return; +} // int32_t transReleaseSrvHandle(void *handle) { return 0; } // void transRefSrvHandle(void *handle) { return; } From 3ec8f607bebc90032d82fcdb0c8705e28691153e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 14:17:48 +0800 Subject: [PATCH 24/76] refactor code --- include/os/osSocket.h | 22 +++++++++++----------- source/libs/transport/inc/transComm.h | 9 ++++++++- source/libs/transport/src/trans.c | 9 +++++++++ source/libs/transport/src/transImpl2.c | 7 +------ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 49acf285ee62..965f15a0ddf4 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -18,17 +18,17 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. -#ifndef ALLOW_FORBID_FUNC -#define socket SOCKET_FUNC_TAOS_FORBID -#define bind BIND_FUNC_TAOS_FORBID -#define listen LISTEN_FUNC_TAOS_FORBID -#define accept ACCEPT_FUNC_TAOS_FORBID -#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID -#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID -#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID -#define inet_addr INET_ADDR_FUNC_TAOS_FORBID -#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID -#endif +// #ifndef ALLOW_FORBID_FUNC +// #define socket SOCKET_FUNC_TAOS_FORBID +// #define bind BIND_FUNC_TAOS_FORBID +// #define listen LISTEN_FUNC_TAOS_FORBID +// #define accept ACCEPT_FUNC_TAOS_FORBID +// #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID +// #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID +// #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID +// #define inet_addr INET_ADDR_FUNC_TAOS_FORBID +// #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID +// #endif #if defined(WINDOWS) #if BYTE_ORDER == LITTLE_ENDIAN diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index ce8955cbc625..218b1092173e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -335,6 +335,7 @@ int32_t transReleaseCliHandle(void* handle); int32_t transReleaseSrvHandle(void* handle); int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); +int32_t transSendRequest2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs); @@ -350,11 +351,17 @@ void transSockInfo2Str(struct sockaddr* sockname, char* dst); int32_t transAllocHandle(int64_t* refId); +#ifndef TD_ACORE void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); - void transCloseClient(void* arg); void transCloseServer(void* arg); +#else +void* transInitClient2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void* transInitServer2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void transCloseClient2(void* arg); +void transCloseServer2(void* arg); +#endif void transCtxInit(STransCtx* ctx); void transCtxCleanup(STransCtx* ctx); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d965cb96e0d1..91bebf276116 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -252,10 +252,19 @@ void rpcCleanup(void) { #else +#ifdef TD_ACORE +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer2, transInitClient2}; + +void (*taosCloseHandle[])(void* arg) = {transCloseServer2, transCloseClient2}; +#else + void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { transInitServer, transInitClient}; void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; +#endif + void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 8d39567789a7..a3a29e83a4c2 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -924,11 +924,6 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); } - { - // TODO(weizhong): add trace log - // if ((code = transAsyncSend(pThrd->asyncPool, &(pCliMsg->q))) != 0) { - // } - } transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); return 0; @@ -1019,7 +1014,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads terrno = code; return NULL; } -void transCloseClient(void *arg) { +void transCloseClient2(void *arg) { int32_t code = 0; return; } From 80a05460a2576053606a0037185ee4e4677452e5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 14:45:16 +0800 Subject: [PATCH 25/76] refactor code --- source/libs/executor/test/queryPlanTests.cpp | 2 +- source/libs/transport/test/CMakeLists.txt | 22 + source/libs/transport/test/svrBench.c | 24 +- .../libs/transport/test/transImpl2_test.cpp | 517 ++++++++++++++++++ source/libs/transport/test/transUT2.cpp | 2 +- source/util/test/log.cpp | 2 +- 6 files changed, 554 insertions(+), 15 deletions(-) create mode 100644 source/libs/transport/test/transImpl2_test.cpp diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 69097ce755d2..fe822385ee07 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -3005,7 +3005,7 @@ void qptInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_BOTH ) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index f30dfc56a2de..34c906295e44 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(transUT2 "") add_executable(svrBench "") add_executable(cliBench "") add_executable(httpBench "") +add_executable(transImpl2 "") target_sources(transUT PRIVATE @@ -32,6 +33,13 @@ target_sources(httpBench "http_test.c" ) +target_sources(transImpl2 + PRIVATE + "transImpl2_test.cpp" +) + + + target_include_directories(transportTest PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" @@ -100,6 +108,20 @@ target_include_directories(httpBench "${TD_SOURCE_DIR}/include/libs/transport" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(transImpl2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(transImpl2 + os + util + common + gtest_main + transport + function +) target_link_libraries(cliBench os diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 44299d86a3af..760bd8b33ef5 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -41,7 +41,7 @@ typedef struct TThread { int idx; } TThread; -MultiThreadQhandle *multiQ = NULL; +MultiThreadQhandle *tMultiQ = NULL; void initLogEnv() { const char *logDir = "/tmp/trans_svr"; @@ -70,7 +70,7 @@ void *processShellMsg(void *arg) { taosAllocateQall(&qall); while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(multiQ->qset[idx], qall, &qinfo); + int numOfMsgs = taosReadAllQitemsFromQset(tMultiQ->qset[idx], qall, &qinfo); tDebug("%d shell msgs are received", numOfMsgs); if (numOfMsgs <= 0) break; @@ -106,11 +106,11 @@ void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void **)&pTemp); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - int32_t idx = balance % multiQ->numOfThread; + int32_t idx = balance % tMultiQ->numOfThread; tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(multiQ->qhandle[idx], pTemp); + taosWriteQitem(tMultiQ->qhandle[idx], pTemp); balance++; - if (balance >= multiQ->numOfThread) balance = 0; + if (balance >= tMultiQ->numOfThread) balance = 0; } int main(int argc, char *argv[]) { @@ -181,15 +181,15 @@ int main(int argc, char *argv[]) { } int32_t numOfAthread = 1; - multiQ = taosMemoryMalloc(sizeof(MultiThreadQhandle)); - multiQ->numOfThread = numOfAthread; - multiQ->qhandle = (STaosQueue **)taosMemoryMalloc(sizeof(STaosQueue *) * numOfAthread); - multiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); + tMultiQ = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + tMultiQ->numOfThread = numOfAthread; + tMultiQ->qhandle = (STaosQueue **)taosMemoryMalloc(sizeof(STaosQueue *) * numOfAthread); + tMultiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); for (int i = 0; i < numOfAthread; i++) { - taosOpenQueue(&multiQ->qhandle[i]); - taosOpenQset(&multiQ->qset[i]); - taosAddIntoQset(multiQ->qset[i], multiQ->qhandle[i], NULL); + taosOpenQueue(&tMultiQ->qhandle[i]); + taosOpenQset(&tMultiQ->qset[i]); + taosAddIntoQset(tMultiQ->qset[i], tMultiQ->qhandle[i], NULL); } TThread *threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); for (int i = 0; i < numOfAthread; i++) { diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp new file mode 100644 index 000000000000..5bae39367028 --- /dev/null +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include "tdatablock.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmisce.h" +#include "transLog.h" +#include "trpc.h" +#include "tversion.h" +using namespace std; + +const char *label = "APP"; +const char *secret = "secret"; +const char *user = "user"; +const char *ckey = "ckey"; + +class Server; +int port = 7000; +// server process +// server except + +typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +class Client { + public: + void Init(int nThread) { + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); + memset(&rpcInit_, 0, sizeof(rpcInit_)); + rpcInit_.localPort = 0; + rpcInit_.label = (char *)label; + rpcInit_.numOfThreads = nThread; + rpcInit_.cfp = processResp; + rpcInit_.user = (char *)user; + rpcInit_.parent = this; + rpcInit_.connType = TAOS_CONN_CLIENT; + rpcInit_.shareConnLimit = 200; + + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + this->transCli = rpcOpen(&rpcInit_); + tsem_init(&this->sem, 0, 0); + } + void SetResp(SRpcMsg *pMsg) { + // set up resp; + this->resp = *pMsg; + } + SRpcMsg *Resp() { return &this->resp; } + + void Restart(CB cb) { + rpcClose(this->transCli); + rpcInit_.cfp = cb; + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + this->transCli = rpcOpen(&rpcInit_); + } + void Stop() { + rpcClose(this->transCli); + this->transCli = NULL; + } + + void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { + SEpSet epSet = {0}; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); + + rpcSendRequest(this->transCli, &epSet, req, NULL); + SemWait(); + *resp = this->resp; + } + void sendReq(SRpcMsg *req) { + SEpSet epSet = {0}; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); + + rpcSendRequest(this->transCli, &epSet, req, NULL); + } + void SendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { + if (req->info.handle != NULL) { + rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT); + req->info.handle = NULL; + } + SendAndRecv(req, resp); + } + + void SemWait() { tsem_wait(&this->sem); } + void SemPost() { tsem_post(&this->sem); } + void Reset() {} + + ~Client() { + if (this->transCli) rpcClose(this->transCli); + } + + private: + tsem_t sem; + SRpcInit rpcInit_; + void *transCli; + SRpcMsg resp; +}; +class Server { + public: + Server() { + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); + memset(&rpcInit_, 0, sizeof(rpcInit_)); + + memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); + rpcInit_.localPort = port; + rpcInit_.label = (char *)label; + rpcInit_.numOfThreads = 5; + rpcInit_.cfp = processReq; + rpcInit_.user = (char *)user; + rpcInit_.connType = TAOS_CONN_SERVER; + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + } + void Start() { + this->transSrv = rpcOpen(&this->rpcInit_); + taosMsleep(1000); + } + void SetSrvContinueSend(CB cb) { + this->Stop(); + rpcInit_.cfp = cb; + this->Start(); + } + void Stop() { + if (this->transSrv == NULL) return; + rpcClose(this->transSrv); + this->transSrv = NULL; + } + void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + this->Stop(); + rpcInit_.cfp = cfp; + this->Start(); + } + void Restart() { + this->Stop(); + this->Start(); + } + ~Server() { + if (this->transSrv) rpcClose(this->transSrv); + this->transSrv = NULL; + } + + private: + SRpcInit rpcInit_; + void *transSrv; +}; +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcFreeCont(pMsg->pCont); + rpcSendResponse(&rpcMsg); +} + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + for (int i = 0; i < 10; i++) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } +} +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); +} +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + { + SRpcMsg rpcMsg1 = {0}; + rpcMsg1.pCont = rpcMallocCont(100); + rpcMsg1.contLen = 100; + rpcMsg1.info = pMsg->info; + rpcMsg1.code = 0; + rpcRegisterBrokenLinkArg(&rpcMsg1); + } + taosMsleep(10); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + Client *client = (Client *)parent; + client->SetResp(pMsg); + client->SemPost(); + tDebug("received resp"); +} + +static void initEnv() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + std::string path = TD_TMP_DIR_PATH "transport"; + // taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { + printf("failed to init log file\n"); + } +} + +class TransObj { + public: + TransObj() { + initEnv(); + cli = new Client; + cli->Init(1); + srv = new Server; + srv->Start(); + } + + void RestartCli(CB cb) { + // + cli->Restart(cb); + } + void StopSrv() { + // + srv->Stop(); + } + // call when link broken, and notify query or fetch stop + void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + /////// + srv->SetSrvContinueSend(cfp); + } + void RestartSrv() { srv->Restart(); } + void StopCli() { + /////// + cli->Stop(); + } + void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } + void cliSendReq(SRpcMsg *req) { cli->sendReq(req); } + void cliSendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecvNoHandle(req, resp); } + + ~TransObj() { + delete cli; + delete srv; + } + + private: + Client *cli; + Server *srv; +}; +class TransEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // set up trans obj + tr = new TransObj(); + } + virtual void TearDown() { + // tear down + delete tr; + } + + TransObj *tr = NULL; +}; + +TEST_F(TransEnv, 01sendAndRec) { + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } +} + +TEST_F(TransEnv, 02StopServer) { + for (int i = 0; i < 1; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.info.ahandle = (void *)0x35; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } + SRpcMsg req = {0}, resp = {0}; + req.info.ahandle = (void *)0x35; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->StopSrv(); + // tr->RestartSrv(); + tr->cliSendAndRecv(&req, &resp); + assert(resp.code != 0); +} +TEST_F(TransEnv, clientUserDefined) { + tr->RestartSrv(); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } + + ////////////////// +} + +TEST_F(TransEnv, cliPersistHandle) { + SRpcMsg resp = {0}; + void *handle = NULL; + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}; + req.info = resp.info; + req.info.persistHandle = 1; + + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // if (i == 5) { + // std::cout << "stop server" << std::endl; + // tr->StopSrv(); + //} + // if (i >= 6) { + // EXPECT_TRUE(resp.code != 0); + //} + handle = resp.info.handle; + } + rpcReleaseHandle(handle, TAOS_CONN_CLIENT); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + } + + taosMsleep(1000); + ////////////////// +} + +TEST_F(TransEnv, srvReleaseHandle) { + SRpcMsg resp = {0}; + tr->SetSrvContinueSend(processReleaseHandleCb); + // tr->Restart(processReleaseHandleCb); + void *handle = NULL; + SRpcMsg req = {0}; + for (int i = 0; i < 1; i++) { + memset(&req, 0, sizeof(req)); + req.info = resp.info; + req.info.persistHandle = 1; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // tr->cliSendAndRecvNoHandle(&req, &resp); + EXPECT_TRUE(resp.code == 0); + } + ////////////////// +} +// reopen later +// TEST_F(TransEnv, cliReleaseHandleExcept) { +// SRpcMsg resp = {0}; +// SRpcMsg req = {0}; +// for (int i = 0; i < 3; i++) { +// memset(&req, 0, sizeof(req)); +// req.info = resp.info; +// req.info.persistHandle = 1; +// req.info.ahandle = (void *)1234; +// req.msgType = 1; +// req.pCont = rpcMallocCont(10); +// req.contLen = 10; +// tr->cliSendAndRecv(&req, &resp); +// if (i == 1) { +// std::cout << "stop server" << std::endl; +// tr->StopSrv(); +// } +// if (i > 1) { +// EXPECT_TRUE(resp.code != 0); +// } +// } +// ////////////////// +//} +TEST_F(TransEnv, srvContinueSend) { + tr->SetSrvContinueSend(processContinueSend); + SRpcMsg req = {0}, resp = {0}; + for (int i = 0; i < 10; i++) { + // memset(&req, 0, sizeof(req)); + // memset(&resp, 0, sizeof(resp)); + // req.msgType = 1; + // req.pCont = rpcMallocCont(10); + // req.contLen = 10; + // tr->cliSendAndRecv(&req, &resp); + } + taosMsleep(1000); +} + +TEST_F(TransEnv, srvPersistHandleExcept) { + tr->SetSrvContinueSend(processContinueSend); + // tr->SetCliPersistFp(cliPersistHandle); + SRpcMsg resp = {0}; + SRpcMsg req = {0}; + for (int i = 0; i < 5; i++) { + // memset(&req, 0, sizeof(req)); + // req.info = resp.info; + // req.msgType = 1; + // req.pCont = rpcMallocCont(10); + // req.contLen = 10; + // tr->cliSendAndRecv(&req, &resp); + // if (i > 2) { + // tr->StopCli(); + // break; + //} + } + taosMsleep(2000); + // conn broken + // +} +TEST_F(TransEnv, cliPersistHandleExcept) { + tr->SetSrvContinueSend(processContinueSend); + SRpcMsg resp = {0}; + SRpcMsg req = {0}; + for (int i = 0; i < 5; i++) { + // memset(&req, 0, sizeof(req)); + // req.info = resp.info; + // req.msgType = 1; + // req.pCont = rpcMallocCont(10); + // req.contLen = 10; + // tr->cliSendAndRecv(&req, &resp); + // if (i > 2) { + // tr->StopSrv(); + // break; + //} + } + taosMsleep(2000); + // conn broken + // +} + +TEST_F(TransEnv, multiCliPersistHandleExcept) { + // conn broken +} +TEST_F(TransEnv, queryExcept) { + tr->SetSrvContinueSend(processRegisterFailure); + SRpcMsg resp = {0}; + SRpcMsg req = {0}; + // for (int i = 0; i < 5; i++) { + // memset(&req, 0, sizeof(req)); + // req.info = resp.info; + // req.info.persistHandle = 1; + // req.msgType = 1; + // req.pCont = rpcMallocCont(10); + // req.contLen = 10; + // tr->cliSendAndRecv(&req, &resp); + // if (i == 2) { + // rpcReleaseHandle(resp.info.handle, TAOS_CONN_CLIENT); + // tr->StopCli(); + // break; + // } + //} + taosMsleep(4 * 1000); +} +TEST_F(TransEnv, noResp) { + SRpcMsg resp = {0}; + SRpcMsg req = {0}; + for (int i = 0; i < 500000; i++) { + memset(&req, 0, sizeof(req)); + req.info.noResp = 1; + req.msgType = 3; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendReq(&req); + // tr->cliSendAndRecv(&req, &resp); + } + taosMsleep(2000); + + // no resp +} \ No newline at end of file diff --git a/source/libs/transport/test/transUT2.cpp b/source/libs/transport/test/transUT2.cpp index 54d23b1f64fa..262cd1089bc1 100644 --- a/source/libs/transport/test/transUT2.cpp +++ b/source/libs/transport/test/transUT2.cpp @@ -250,7 +250,7 @@ static void initEnv() { taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_BOTH ) != 0) { printf("failed to init log file\n"); } } diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp index ba32d2d63949..4f254d20dcd2 100644 --- a/source/util/test/log.cpp +++ b/source/util/test/log.cpp @@ -15,7 +15,7 @@ TEST(log, check_log_refactor) { tsAsyncLog = 0; // idxDebugFlag = 143; strcpy(tsLogDir, (char *)logDir); - taosInitLog(tsLogDir, 10, false); + taosInitLog(tsLogDir, 10, LOG_MODE_BOTH); tsAsyncLog = 0; uDebugFlag = 143; From 4546dbeffe507e0b8264000657fc9fb17a2fa93f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 15:03:37 +0800 Subject: [PATCH 26/76] refactor code --- source/libs/transport/src/trans.c | 4 + source/libs/transport/src/transImpl2.c | 1 + .../libs/transport/test/transImpl2_test.cpp | 210 +----------------- 3 files changed, 7 insertions(+), 208 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 91bebf276116..1d9bd9a1d9ec 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -436,7 +436,11 @@ void* rpcReallocCont(void* ptr, int64_t contLen) { } int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { +#ifndef TD_ACORE return transSendRequest(shandle, pEpSet, pMsg, NULL); +#else + return transSendRequest2(shandle, pEpSet, pMsg, NULL); +#endif } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index a3a29e83a4c2..ee7fd19059c2 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -782,6 +782,7 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { if (code != 0) { TAOS_CHECK_GOTO(code, &line, _end); } + *ppThrd = pThrd; return code; _end: if (code != 0) { diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 5bae39367028..106e5ed1006d 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -299,219 +299,13 @@ class TransEnv : public ::testing::Test { TransObj *tr = NULL; }; -TEST_F(TransEnv, 01sendAndRec) { +TEST_F(TransEnv, 01sendAndReq) { for (int i = 0; i < 10; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 0; req.pCont = rpcMallocCont(10); req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); - assert(resp.code == 0); - } -} - -TEST_F(TransEnv, 02StopServer) { - for (int i = 0; i < 1; i++) { - SRpcMsg req = {0}, resp = {0}; - req.msgType = 0; - req.info.ahandle = (void *)0x35; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); - assert(resp.code == 0); - } - SRpcMsg req = {0}, resp = {0}; - req.info.ahandle = (void *)0x35; - req.msgType = 1; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->StopSrv(); - // tr->RestartSrv(); - tr->cliSendAndRecv(&req, &resp); - assert(resp.code != 0); -} -TEST_F(TransEnv, clientUserDefined) { - tr->RestartSrv(); - for (int i = 0; i < 10; i++) { - SRpcMsg req = {0}, resp = {0}; - req.msgType = 0; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); + tr->cliSendReq(&req); assert(resp.code == 0); } - - ////////////////// } - -TEST_F(TransEnv, cliPersistHandle) { - SRpcMsg resp = {0}; - void *handle = NULL; - for (int i = 0; i < 10; i++) { - SRpcMsg req = {0}; - req.info = resp.info; - req.info.persistHandle = 1; - - req.msgType = 1; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); - // if (i == 5) { - // std::cout << "stop server" << std::endl; - // tr->StopSrv(); - //} - // if (i >= 6) { - // EXPECT_TRUE(resp.code != 0); - //} - handle = resp.info.handle; - } - rpcReleaseHandle(handle, TAOS_CONN_CLIENT); - for (int i = 0; i < 10; i++) { - SRpcMsg req = {0}; - req.msgType = 1; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); - } - - taosMsleep(1000); - ////////////////// -} - -TEST_F(TransEnv, srvReleaseHandle) { - SRpcMsg resp = {0}; - tr->SetSrvContinueSend(processReleaseHandleCb); - // tr->Restart(processReleaseHandleCb); - void *handle = NULL; - SRpcMsg req = {0}; - for (int i = 0; i < 1; i++) { - memset(&req, 0, sizeof(req)); - req.info = resp.info; - req.info.persistHandle = 1; - req.msgType = 1; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendAndRecv(&req, &resp); - // tr->cliSendAndRecvNoHandle(&req, &resp); - EXPECT_TRUE(resp.code == 0); - } - ////////////////// -} -// reopen later -// TEST_F(TransEnv, cliReleaseHandleExcept) { -// SRpcMsg resp = {0}; -// SRpcMsg req = {0}; -// for (int i = 0; i < 3; i++) { -// memset(&req, 0, sizeof(req)); -// req.info = resp.info; -// req.info.persistHandle = 1; -// req.info.ahandle = (void *)1234; -// req.msgType = 1; -// req.pCont = rpcMallocCont(10); -// req.contLen = 10; -// tr->cliSendAndRecv(&req, &resp); -// if (i == 1) { -// std::cout << "stop server" << std::endl; -// tr->StopSrv(); -// } -// if (i > 1) { -// EXPECT_TRUE(resp.code != 0); -// } -// } -// ////////////////// -//} -TEST_F(TransEnv, srvContinueSend) { - tr->SetSrvContinueSend(processContinueSend); - SRpcMsg req = {0}, resp = {0}; - for (int i = 0; i < 10; i++) { - // memset(&req, 0, sizeof(req)); - // memset(&resp, 0, sizeof(resp)); - // req.msgType = 1; - // req.pCont = rpcMallocCont(10); - // req.contLen = 10; - // tr->cliSendAndRecv(&req, &resp); - } - taosMsleep(1000); -} - -TEST_F(TransEnv, srvPersistHandleExcept) { - tr->SetSrvContinueSend(processContinueSend); - // tr->SetCliPersistFp(cliPersistHandle); - SRpcMsg resp = {0}; - SRpcMsg req = {0}; - for (int i = 0; i < 5; i++) { - // memset(&req, 0, sizeof(req)); - // req.info = resp.info; - // req.msgType = 1; - // req.pCont = rpcMallocCont(10); - // req.contLen = 10; - // tr->cliSendAndRecv(&req, &resp); - // if (i > 2) { - // tr->StopCli(); - // break; - //} - } - taosMsleep(2000); - // conn broken - // -} -TEST_F(TransEnv, cliPersistHandleExcept) { - tr->SetSrvContinueSend(processContinueSend); - SRpcMsg resp = {0}; - SRpcMsg req = {0}; - for (int i = 0; i < 5; i++) { - // memset(&req, 0, sizeof(req)); - // req.info = resp.info; - // req.msgType = 1; - // req.pCont = rpcMallocCont(10); - // req.contLen = 10; - // tr->cliSendAndRecv(&req, &resp); - // if (i > 2) { - // tr->StopSrv(); - // break; - //} - } - taosMsleep(2000); - // conn broken - // -} - -TEST_F(TransEnv, multiCliPersistHandleExcept) { - // conn broken -} -TEST_F(TransEnv, queryExcept) { - tr->SetSrvContinueSend(processRegisterFailure); - SRpcMsg resp = {0}; - SRpcMsg req = {0}; - // for (int i = 0; i < 5; i++) { - // memset(&req, 0, sizeof(req)); - // req.info = resp.info; - // req.info.persistHandle = 1; - // req.msgType = 1; - // req.pCont = rpcMallocCont(10); - // req.contLen = 10; - // tr->cliSendAndRecv(&req, &resp); - // if (i == 2) { - // rpcReleaseHandle(resp.info.handle, TAOS_CONN_CLIENT); - // tr->StopCli(); - // break; - // } - //} - taosMsleep(4 * 1000); -} -TEST_F(TransEnv, noResp) { - SRpcMsg resp = {0}; - SRpcMsg req = {0}; - for (int i = 0; i < 500000; i++) { - memset(&req, 0, sizeof(req)); - req.info.noResp = 1; - req.msgType = 3; - req.pCont = rpcMallocCont(10); - req.contLen = 10; - tr->cliSendReq(&req); - // tr->cliSendAndRecv(&req, &resp); - } - taosMsleep(2000); - - // no resp -} \ No newline at end of file From 9ff8faaf47f93625ea207f435d81e722594a098e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 15:38:49 +0800 Subject: [PATCH 27/76] refactor code --- source/libs/transport/src/transImpl2.c | 63 +++++++++++++++++-- .../libs/transport/test/transImpl2_test.cpp | 1 + 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index ee7fd19059c2..6da609e3d807 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -126,6 +126,7 @@ typedef struct { fd_set *evtReadSetOut; fd_set *evtWriteSetOut; SHashObj *pFdTable; + void *arg; } SEvtMgt; static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); @@ -341,7 +342,8 @@ static void evtMgtDestroy(SEvtMgt *pOpt) { taosMemoryFree(pOpt); } -static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, AsyncCb cb, int8_t evtType) { +static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, AsyncCb cb, int8_t evtType, + void *pThrd) { int32_t code = 0; SAsyncHandle *pAsync = taosMemoryCalloc(1, sizeof(SAsyncHandle)); if (pAsync == NULL) { @@ -484,13 +486,16 @@ void *transWorkerThread(void *arg) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } - code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotify, EVT_CONN_T); + + pOpt->arg = pThrd; + + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotify, EVT_CONN_T, (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } - code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->handle, evtHandleReq, EVT_ASYNC_T); + code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->handle, evtHandleReq, EVT_ASYNC_T, (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); @@ -938,6 +943,44 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, } return code; } + +static void evtHandleCliReq(void *arg, int32_t status) { + int32_t code = 0; + SAsyncHandle *handle = arg; + SEvtMgt *pEvtMgt = handle->data; + + SCliThrd2 *pThrd = pEvtMgt->arg; + + queue wq; + QUEUE_INIT(&wq); + + taosThreadMutexLock(&pThrd->msgMtx); + QUEUE_MOVE(&pThrd->msg, &wq); + taosThreadMutexUnlock(&pThrd->msgMtx); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + if (pReq == NULL) { + continue; + } + STraceId *trace = &pReq->msg.info.traceId; + tGDebug("handle request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pThrd->pid, pReq->ctx->epSet->eps[0].fqdn, + pReq->ctx->epSet->eps[0].port, pReq->msg.info.ahandle); + + if (pReq->msg.info.handle == 0) { + destroyReq(pReq); + continue; + } + } +} + static void *cliWorkThread2(void *arg) { int32_t line = 0; int32_t code = 0; @@ -951,7 +994,10 @@ static void *cliWorkThread2(void *arg) { TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); setThreadName(threadName); - code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleReq, EVT_ASYNC_T); + pThrd->pEvtMgt->arg = pThrd; + + code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReq, EVT_ASYNC_T, + (void *)pThrd); TAOS_CHECK_GOTO(code, &line, _end); while (!pThrd->quit) { @@ -959,6 +1005,8 @@ static void *cliWorkThread2(void *arg) { if (code != 0) { tError("failed to dispatch since %s", tstrerror(code)); continue; + } else { + tDebug("success to dispatch"); } } @@ -990,6 +1038,12 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads if (code != 0) { goto _err; } + code = pipe(pThrd->pipe_queue_fd); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + TAOS_CHECK_GOTO(code, NULL, _err); + } + pThrd->pInst = pInst; int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); if (err != 0) { @@ -1002,6 +1056,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads pThrd->thrdInited = 1; cli->pThreadObj[i] = pThrd; } + return cli; _err: if (cli) { diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 106e5ed1006d..49f308b405fd 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -306,6 +306,7 @@ TEST_F(TransEnv, 01sendAndReq) { req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendReq(&req); + taosMsleep(1000); assert(resp.code == 0); } } From 963194b10c92fea06d083009c89d7ddea195074c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 16:47:14 +0800 Subject: [PATCH 28/76] refactor code --- source/libs/transport/src/transImpl2.c | 21 +++++++++++++++---- .../libs/transport/test/transImpl2_test.cpp | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 6da609e3d807..c1b56ca1ecc3 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -127,6 +127,9 @@ typedef struct { fd_set *evtWriteSetOut; SHashObj *pFdTable; void *arg; + int32_t fd[2048]; + int32_t fdIdx; + } SEvtMgt; static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); @@ -152,6 +155,8 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { pRes->evtWriteSetIn = NULL; pRes->evtReadSetOut = NULL; pRes->evtWriteSetOut = NULL; + pRes->pFdTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pRes->fdIdx = 0; code = evtMgtResize(pRes, (32 + 1)); if (code != 0) { @@ -223,14 +228,15 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { } for (j = 0; j < nfds; j++) { + int32_t fd = pOpt->fd[j]; res = 0; - if (FD_ISSET(j, pOpt->evtReadSetOut)) { + if (FD_ISSET(fd, pOpt->evtReadSetOut)) { res |= EVT_READ; } - if (FD_ISSET(j, pOpt->evtWriteSetOut)) { + if (FD_ISSET(fd, pOpt->evtWriteSetOut)) { res |= EVT_WRITE; } - code = evtMgtHandle(pOpt, res, i); + code = evtMgtHandle(pOpt, res, fd); if (code != 0) { tError("failed to handle fd %d since %s", i, tstrerror(code)); } @@ -292,7 +298,9 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar if (events & EVT_WRITE) { FD_SET(fd, pOpt->evtWriteSetIn); } + pOpt->fd[pOpt->fdIdx++] = fd; code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); + return 0; } @@ -317,6 +325,11 @@ static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg } else { code = taosHashRemove(pOpt->pFdTable, &fd, sizeof(fd)); } + for (int32_t i = 0; i < sizeof(pOpt->fd) / sizeof(pOpt->fd[0]); i++) { + if (pOpt->fd[i] == fd) { + pOpt->fd[i] = -1; + } + } return code; } static void evtMgtDestroy(SEvtMgt *pOpt) { @@ -990,7 +1003,7 @@ static void *cliWorkThread2(void *arg) { pThrd->pid = taosGetSelfPthreadId(); - tsEnableRandErr = true; + tsEnableRandErr = false; TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); setThreadName(threadName); diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 49f308b405fd..89db27628503 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -49,7 +49,7 @@ class Client { memset(&rpcInit_, 0, sizeof(rpcInit_)); rpcInit_.localPort = 0; rpcInit_.label = (char *)label; - rpcInit_.numOfThreads = nThread; + rpcInit_.numOfThreads = 1; rpcInit_.cfp = processResp; rpcInit_.user = (char *)user; rpcInit_.parent = this; From f0ee1bcf51d1c18db40cbe193a7317cd5c3b4f7a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 20:44:53 +0800 Subject: [PATCH 29/76] refactor code --- source/libs/transport/src/transImpl2.c | 97 +++++++++++++------ .../libs/transport/test/transImpl2_test.cpp | 4 +- 2 files changed, 71 insertions(+), 30 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index c1b56ca1ecc3..40bec74c1aae 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -118,7 +118,7 @@ typedef struct { void *arg; } SFdCbArg; typedef struct { - int32_t evtFds; + int32_t evtFds; // highest fd in the set int32_t evtFdsSize; int32_t resizeOutSets; fd_set *evtReadSetIn; @@ -177,7 +177,12 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg) { handle->cb(handle, 0); } else if (pArg->evtType == EVT_ASYNC_T) { SAsyncHandle *handle = pArg->arg; - handle->cb(handle, 0); + char buf[2] = {0}; + int32_t nBytes = read(pArg->fd, buf, 1); + if (nBytes == 1) { + handle->cb(handle, 0); + } + } else { } return code; @@ -198,10 +203,11 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { return code; } static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { - int32_t code = 0, res = 0, i, j, nfds = 0; + int32_t code = 0, res = 0, j, nfds = 0; if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; int32_t sz = pOpt->evtFdsSize; + readSetOut = taosMemoryRealloc(pOpt->evtReadSetOut, sz); if (readSetOut == NULL) { return terrno; @@ -212,8 +218,8 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (writeSetOut == NULL) { return terrno; } - pOpt->evtWriteSetOut = writeSetOut; + pOpt->resizeOutSets = 0; } @@ -236,9 +242,16 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (FD_ISSET(fd, pOpt->evtWriteSetOut)) { res |= EVT_WRITE; } - code = evtMgtHandle(pOpt, res, fd); - if (code != 0) { - tError("failed to handle fd %d since %s", i, tstrerror(code)); + + if (res == 0) { + tDebug("do nothing"); + } else { + code = evtMgtHandle(pOpt, res, fd); + if (code != 0) { + tError("failed to handle fd %d since %s", fd, tstrerror(code)); + } else { + tDebug("success to handle fd %d", fd); + } } } @@ -288,18 +301,20 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar if (evtMgtResize(pOpt, fdSize)) { return -1; } - pOpt->evtFds = fd; } + pOpt->evtFds = fd; } - if (events & EVT_READ) { - FD_SET(fd, pOpt->evtReadSetIn); - } + if (fd != 0) { + if (events & EVT_READ) { + FD_SET(fd, pOpt->evtReadSetIn); + } - if (events & EVT_WRITE) { - FD_SET(fd, pOpt->evtWriteSetIn); + if (events & EVT_WRITE) { + FD_SET(fd, pOpt->evtWriteSetIn); + } + pOpt->fd[pOpt->fdIdx++] = fd; + code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); } - pOpt->fd[pOpt->fdIdx++] = fd; - code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); return 0; } @@ -588,10 +603,27 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads goto End; } - QUEUE_INIT(&thrd->fdQueue.q); - if (pipe(thrd->pipe_fd) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto End; + // QUEUE_INIT(&thrd->fdQueue.q); + { + int fd[2] = {0}; + code = pipe(fd); + if (code < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto End; + } + thrd->pipe_fd[0] = fd[0]; + thrd->pipe_fd[1] = fd[1]; + } + + { + int fd2[2] = {0}; + code = pipe(fd2); + if (code < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto End; + } + thrd->pipe_queue_fd[0] = fd2[0]; + thrd->pipe_queue_fd[1] = fd2[1]; } int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); @@ -796,10 +828,11 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { if (pThrd == NULL) { TAOS_CHECK_GOTO(terrno, &line, _end); } - code = evtMgtCreate(&pThrd->pEvtMgt); - if (code != 0) { - TAOS_CHECK_GOTO(code, &line, _end); - } + + taosThreadMutexInit(&pThrd->msgMtx, NULL); + + QUEUE_INIT(&pThrd->msg); + *ppThrd = pThrd; return code; _end: @@ -967,14 +1000,13 @@ static void evtHandleCliReq(void *arg, int32_t status) { queue wq; QUEUE_INIT(&wq); - taosThreadMutexLock(&pThrd->msgMtx); - QUEUE_MOVE(&pThrd->msg, &wq); - taosThreadMutexUnlock(&pThrd->msgMtx); + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); if (QUEUE_IS_EMPTY(&wq)) { return; } - while (!QUEUE_IS_EMPTY(&wq)) { queue *el = QUEUE_HEAD(&wq); QUEUE_REMOVE(el); @@ -1007,6 +1039,11 @@ static void *cliWorkThread2(void *arg) { TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); setThreadName(threadName); + code = evtMgtCreate(&pThrd->pEvtMgt); + if (code != 0) { + TAOS_CHECK_GOTO(code, &line, _end); + } + pThrd->pEvtMgt->arg = pThrd; code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReq, EVT_ASYNC_T, @@ -1051,11 +1088,15 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads if (code != 0) { goto _err; } - code = pipe(pThrd->pipe_queue_fd); + int fd[2] = {0}; + + code = pipe(fd); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(code, NULL, _err); } + pThrd->pipe_queue_fd[0] = fd[0]; + pThrd->pipe_queue_fd[1] = fd[1]; pThrd->pInst = pInst; int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 89db27628503..8ed9023baf9c 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -124,14 +124,14 @@ class Server { memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); rpcInit_.localPort = port; rpcInit_.label = (char *)label; - rpcInit_.numOfThreads = 5; + rpcInit_.numOfThreads = 1; rpcInit_.cfp = processReq; rpcInit_.user = (char *)user; rpcInit_.connType = TAOS_CONN_SERVER; taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); } void Start() { - this->transSrv = rpcOpen(&this->rpcInit_); + //this->transSrv = rpcOpen(&this->rpcInit_); taosMsleep(1000); } void SetSrvContinueSend(CB cb) { From ddb31c512b2bc37d8b08a8559fbb408c5641f743 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 20:54:26 +0800 Subject: [PATCH 30/76] refactor code --- source/libs/transport/src/transImpl2.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 40bec74c1aae..81bbce847c2a 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -231,6 +231,9 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { code = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); if (code < 0) { return TAOS_SYSTEM_ERROR(errno); + } else if (code == 0) { + tDebug("select timeout occurred"); + return code; } for (j = 0; j < nfds; j++) { @@ -244,7 +247,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { } if (res == 0) { - tDebug("do nothing"); + continue; } else { code = evtMgtHandle(pOpt, res, fd); if (code != 0) { From 5776a8b2979678a1b57b8f14901f4be0fc0f2026 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Dec 2024 21:24:14 +0800 Subject: [PATCH 31/76] refactor code --- source/libs/transport/src/transImpl2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 81bbce847c2a..0b5db0dfe90f 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -235,8 +235,9 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { tDebug("select timeout occurred"); return code; } + int32_t active_Fds = code; - for (j = 0; j < nfds; j++) { + for (j = 0; j < nfds && active_Fds > 0; j++) { int32_t fd = pOpt->fd[j]; res = 0; if (FD_ISSET(fd, pOpt->evtReadSetOut)) { @@ -249,6 +250,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (res == 0) { continue; } else { + active_Fds--; code = evtMgtHandle(pOpt, res, fd); if (code != 0) { tError("failed to handle fd %d since %s", fd, tstrerror(code)); From 26d303932a09c3fb4db8055eb3c30a2a235a3e14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 11:45:15 +0800 Subject: [PATCH 32/76] refactor code --- include/libs/transport/trpc.h | 1 + source/libs/transport/inc/transComm.h | 14 + source/libs/transport/src/transImpl2.c | 367 ++++++++++++++++++++----- 3 files changed, 317 insertions(+), 65 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0a0e6bae5fdd..d388afa16888 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -250,6 +250,7 @@ typedef struct SRpcHandleInfo { int64_t qId; int32_t msgType; void *reqWithSem; + int32_t refIdMgt; } SRpcHandleInfo; typedef struct SRpcMsg { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 218b1092173e..8f16b787643e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -532,6 +532,20 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); #ifdef TD_ACORE +#define ASYNC_CHECK_HANDLE(idMgt, id, exh1) \ + do { \ + if (id > 0) { \ + SExHandle* exh2 = transAcquireExHandle(idMgt, id); \ + if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) { \ + tDebug("handle not match, exh1:%p, exh2:%p, refId:%" PRId64 "", exh1, exh2, id); \ + code = TSDB_CODE_INVALID_MSG; \ + goto _return1; \ + } \ + } else { \ + tDebug("invalid handle to release"); \ + goto _return2; \ + } \ + } while (0) int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport); int32_t transGetCb(RPC_TYPE type, STrans** ppTransport); diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 0b5db0dfe90f..bd4b9cf164f3 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -54,6 +54,76 @@ typedef struct { typedef struct { queue q; } SFdQueue; + +typedef struct SSvrConn { + int32_t ref; + // uv_tcp_t* pTcp; + // uv_timer_t pTimer; + + queue queue; + SConnBuffer readBuf; // read buf, + int inType; + void *pInst; // rpc init + void *ahandle; // + void *hostThrd; + STransQueue resps; + + // SSvrRegArg regArg; + bool broken; // conn broken; + + ConnStatus status; + + uint32_t serverIp; + uint32_t clientIp; + uint16_t port; + + char src[32]; + char dst[32]; + + int64_t refId; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + int8_t userInited; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + int64_t whiteListVer; + + // state req dict + SHashObj *pQTable; + // uv_buf_t *buf; + int32_t bufSize; + queue wq; // uv_write_t queue + int32_t fd; +} SSvrConn; +typedef struct SSvrRespMsg { + SSvrConn *pConn; + STransMsg msg; + queue q; + STransMsgType type; + int64_t seqNum; + void *arg; + FilteFunc func; + int8_t sent; + +} SSvrRespMsg; + +#define ASYNC_ERR_JRET(thrd) \ + do { \ + if (thrd->quit) { \ + tTrace("worker thread already quit, ignore msg"); \ + goto _return1; \ + } \ + } while (0) + +static FORCE_INLINE void destroySmsg(SSvrRespMsg *smsg) { + if (smsg == NULL) { + return; + } + transFreeMsg(smsg->msg.pCont); + taosMemoryFree(smsg); +} typedef struct SWorkThrd { TdThread thread; // //uv_connect_t connect_req; @@ -73,15 +143,16 @@ typedef struct SWorkThrd { int32_t connRefMgt; - int32_t pipe_fd[2]; // - int32_t pipe_queue_fd[2]; int32_t client_count; int8_t inited; TdThreadMutex mutex; SFdQueue fdQueue; + int32_t pipe_fd[2]; // SAsyncHandle *notifyNewConnHandle; - SAsyncHandle *handle; + + int32_t pipe_queue_fd[2]; + SAsyncHandle *asyncHandle; } SWorkThrd2; typedef struct SServerObj { TdThread thread; @@ -103,20 +174,21 @@ typedef void (*__sendCb)(SFdArg *arg, int32_t status); typedef void (*__readCb)(SFdArg *arg, int32_t status); typedef void (*__asyncCb)(SFdArg *arg, int32_t status); -enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T }; +enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T, EVT_NEW_CONN_T }; typedef struct { int32_t fd; void *data; int32_t event; - __sendCb sentFn; + __sendCb sendFn; __readCb readFn; __asyncCb asyncFn; int8_t evtType; void *arg; } SFdCbArg; + typedef struct { int32_t evtFds; // highest fd in the set int32_t evtFdsSize; @@ -170,40 +242,45 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { // int32_t selectUtilRange() -int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg) { +int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { + int32_t nBytes = 0; + char buf[2] = {0}; int32_t code = 0; - if (pArg->evtType == EVT_CONN_T) { - SAsyncHandle *handle = pArg->arg; - handle->cb(handle, 0); - } else if (pArg->evtType == EVT_ASYNC_T) { - SAsyncHandle *handle = pArg->arg; - char buf[2] = {0}; - int32_t nBytes = read(pArg->fd, buf, 1); - if (nBytes == 1) { - handle->cb(handle, 0); + if (pArg->evtType == EVT_NEW_CONN_T || pArg->evtType == EVT_ASYNC_T) { + // handle new coming conn; + if (res & EVT_READ) { + SAsyncHandle *handle = pArg->arg; + nBytes = read(pArg->fd, buf, 2); + if (nBytes >= 1) { + handle->cb(handle, 0); + } + } + if (res & EVT_WRITE) { + // handle err + } + } else if (pArg->event == EVT_CONN_T) { + if (res & EVT_READ) { + pArg->readFn(NULL, 0); + // handle read + } + if (res & EVT_WRITE) { + pArg->sendFn(NULL, 0); + // handle write } - - } else { } return code; } int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { - int32_t code = 0; - if (res & EVT_READ) { - // handle read - } - if (res & EVT_WRITE) { - // handle write - } + int32_t code = 0; SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); if (pArg == NULL) { return TAOS_SYSTEM_ERROR(EBADF); } - code = evtMgtHandleImpl(pOpt, pArg); - return code; + return evtMgtHandleImpl(pOpt, pArg, res); } static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { - int32_t code = 0, res = 0, j, nfds = 0; + int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; + if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; int32_t sz = pOpt->evtFdsSize; @@ -228,14 +305,13 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { nfds = pOpt->evtFds + 1; // TODO lock or not - code = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); - if (code < 0) { + active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); + if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); - } else if (code == 0) { + } else if (active_Fds == 0) { tDebug("select timeout occurred"); return code; } - int32_t active_Fds = code; for (j = 0; j < nfds && active_Fds > 0; j++) { int32_t fd = pOpt->fd[j]; @@ -447,8 +523,116 @@ void *transAcceptThread(void *arg) { } return NULL; } +int32_t uvGetConnRefOfThrd(SWorkThrd2 *thrd) { return thrd ? thrd->connRefMgt : -1; } -void evtNewConnNotify(void *async, int32_t status) { +void uvDestroyResp(void *e) { + SSvrRespMsg *pMsg = QUEUE_DATA(e, SSvrRespMsg, q); + destroySmsg(pMsg); +} +static int32_t connGetBasicInfo(SSvrConn *pConn) { + int32_t code = 0; + + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + char ip_str[INET_ADDRSTRLEN]; + int port; + + if (getpeername(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->dst, sizeof(pConn->dst), "%s:%d", ip_str, port); + + if (getsockname(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + + snprintf(pConn->src, sizeof(pConn->src), "%s:%d", ip_str, port); + return code; +} +static SSvrConn *createConn(void *tThrd, int32_t fd) { + int32_t code = 0; + int32_t lino = 0; + SWorkThrd2 *pThrd = tThrd; + SSvrConn *pConn = taosMemoryCalloc(1, sizeof(SSvrConn)); + if (pConn == NULL) { + return NULL; + } + pConn->fd = fd; + QUEUE_INIT(&pConn->queue); + + code = connGetBasicInfo(pConn); + TAOS_CHECK_GOTO(code, &lino, _end); + + // if ((code = transInitBuffer(&pConn->readBuf)) != 0) { + // TAOS_CHECK_GOTO(code, &lino, _end); + // } + + // if ((code = transQueueInit(&pConn->resps, uvDestroyResp)) != 0) { + // TAOS_CHECK_GOTO(code, &lino, _end); + //} + + pConn->broken = false; + pConn->status = ConnNormal; + + SExHandle *exh = taosMemoryMalloc(sizeof(SExHandle)); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); + } + + exh->handle = pConn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(uvGetConnRefOfThrd(pThrd), exh); + if (exh->refId < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, &lino, _end); + } + + SExHandle *pSelf = transAcquireExHandle(uvGetConnRefOfThrd(pThrd), exh->refId); + if (pSelf != exh) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _end); + } + + STrans *pInst = pThrd->pInst; + pConn->refId = exh->refId; + + QUEUE_INIT(&exh->q); + tTrace("%s handle %p, conn %p created, refId:%" PRId64, transLabel(pInst), exh, pConn, pConn->refId); + + pConn->pQTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pConn->pQTable == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); + } + QUEUE_PUSH(&pThrd->conn, &pConn->queue); + + // code = initWQ(&pConn->wq); + // TAOS_CHECK_GOTO(code, &lino, _end); + // wqInited = 1; + + return pConn; +_end: + if (code != 0) { + tError("failed to create conn since %s", tstrerror(code)); + // destroryConn(pConn); + } + return NULL; +} +static void destroryConn(SSvrConn *pConn) { + if (pConn == NULL) { + return; + } + // transDestroyBuffer(&pConn->readBuf); + // transQueueCleanup(&pConn->resps); + taosHashCleanup(pConn->pQTable); + QUEUE_REMOVE(&pConn->queue); + taosMemoryFree(pConn); +} +void evtNewConnNotifyCb(void *async, int32_t status) { int32_t code = 0; SAsyncHandle *handle = async; @@ -470,18 +654,32 @@ void evtNewConnNotify(void *async, int32_t status) { SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); - SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pArg, .fd = pArg->acceptFd}; + SSvrConn *pConn = createConn(pEvtMgt->arg, pArg->acceptFd); + if (pConn == NULL) { + tError("failed to create conn since %s", tstrerror(code)); + taosMemoryFree(pArg); + continue; + } else { + tDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); + } + + SFdCbArg arg = { + .evtType = EVT_CONN_T, .arg = pArg, .fd = pArg->acceptFd, .readFn = NULL, .sendFn = NULL, .data = NULL}; code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); + + if (code != 0) { + tError("failed to add fd to evt since %s", tstrerror(code)); + } taosMemoryFree(pArg); } return; } -void evtHandleReq(void *async, int32_t status) { +void evtHandleRespCb(void *async, int32_t status) { int32_t code = 0; SAsyncHandle *handle = async; - SEvtMgt *pEvtMgt = handle->data; + // SEvtMgt *pEvtMgt = handle->data; queue wq; QUEUE_INIT(&wq); @@ -496,9 +694,7 @@ void evtHandleReq(void *async, int32_t status) { queue *el = QUEUE_HEAD(&wq); QUEUE_REMOVE(el); - SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); - // code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, NULL); - taosMemoryFree(pArg); + SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); } return; @@ -522,13 +718,13 @@ void *transWorkerThread(void *arg) { pOpt->arg = pThrd; - code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotify, EVT_CONN_T, (void *)pThrd); + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_CONN_T, (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } - code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->handle, evtHandleReq, EVT_ASYNC_T, (void *)pThrd); + code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleRespCb, EVT_ASYNC_T, (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); @@ -650,6 +846,70 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads return NULL; } +// int32_t transReleaseSrvHandle(void *handle) { return 0; } +// void transRefSrvHandle(void *handle) { return; } + +// void transUnrefSrvHandle(void *handle) { return; } + +int32_t transSendResponse2(STransMsg *msg) { + int32_t code = 0; + + if (msg->info.noResp) { + rpcFreeCont(msg->pCont); + tTrace("no need send resp"); + return 0; + } + SExHandle *exh = msg->info.handle; + + if (exh == NULL) { + rpcFreeCont(msg->pCont); + return 0; + } + int64_t refId = msg->info.refId; + ASYNC_CHECK_HANDLE(msg->info.refIdMgt, refId, exh); + + STransMsg tmsg = *msg; + tmsg.info.refId = refId; + if (tmsg.info.qId == 0) { + tmsg.msgType = msg->info.msgType + 1; + } + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + m->msg = tmsg; + + m->type = Normal; + + STraceId *trace = (STraceId *)&msg->info.traceId; + tGDebug("conn %p start to send resp (1/2)", exh->handle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; + } + + transReleaseExHandle(msg->info.refIdMgt, refId); + return 0; + +_return1: + tDebug("handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; +_return2: + tDebug("handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + return code; +} +// int32_t transRegisterMsg(const STransMsg *msg) { return 0; } +// int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } + void transCloseServer2(void *arg) { // impl later return; @@ -995,7 +1255,7 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, return code; } -static void evtHandleCliReq(void *arg, int32_t status) { +static void evtHandleCliReqCb(void *arg, int32_t status) { int32_t code = 0; SAsyncHandle *handle = arg; SEvtMgt *pEvtMgt = handle->data; @@ -1051,7 +1311,7 @@ static void *cliWorkThread2(void *arg) { pThrd->pEvtMgt->arg = pThrd; - code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReq, EVT_ASYNC_T, + code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReqCb, EVT_ASYNC_T, (void *)pThrd); TAOS_CHECK_GOTO(code, &line, _end); @@ -1133,26 +1393,3 @@ void transCloseClient2(void *arg) { int32_t code = 0; return; } - -// int32_t transReleaseSrvHandle(void *handle) { return 0; } -// void transRefSrvHandle(void *handle) { return; } - -// void transUnrefSrvHandle(void *handle) { return; } -// int32_t transSendResponse(STransMsg *msg) { -// // -// int32_t code = 0; -// if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { -// msg->msgType = msg->info.msgType + 1; -// } -// if (msg->info.noResp) { -// rpcFreeCont(msg->pCont); -// return 0; -// } -// int32_t svrVer = 0; -// code = taosVersionStrToInt(td_version, &svrVer); -// msg->info.cliVer = svrVer; -// msg->type = msg->info.connType; -// return transSendResp(msg); -// } -// int32_t transRegisterMsg(const STransMsg *msg) { return 0; } -// int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } From 1b3a68e2bc91c2b0a062be78404a3b6ae32e7cd4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 16:17:01 +0800 Subject: [PATCH 33/76] refactor code --- include/libs/transport/trpc.h | 1 - source/libs/transport/inc/transComm.h | 13 +- source/libs/transport/src/transComm.c | 86 ++++++++++ source/libs/transport/src/transImpl2.c | 213 +++++++++++++++++++++++-- 4 files changed, 293 insertions(+), 20 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index d388afa16888..956f42f675fc 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -246,7 +246,6 @@ typedef struct SRpcHandleInfo { int8_t compressed; int16_t connType; int64_t seq; - int64_t sidSeq; int64_t qId; int32_t msgType; void *reqWithSem; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 8f16b787643e..4f43eb4d29fc 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -248,6 +248,11 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #ifdef TD_ACORE typedef struct SConnBuffer { char* buf; + int len; + int cap; + int left; + int total; + int invalid; } SConnBuffer; #else @@ -316,14 +321,18 @@ bool transAsyncPoolIsEmpty(SAsyncPool* pool); int32_t transInitBuffer(SConnBuffer* buf); int32_t transClearBuffer(SConnBuffer* buf); void transDestroyBuffer(SConnBuffer* buf); -int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); -bool transReadComplete(SConnBuffer* connBuf); int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); +int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +bool transReadComplete(SConnBuffer* connBuf); int32_t transSetConnOption(uv_tcp_t* stream, int keepalive); #endif +int32_t transInitBuffer(SConnBuffer* buf); +int32_t transClearBuffer(SConnBuffer* buf); +void transDestroyBuffer(SConnBuffer* buf); +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 48decac516e2..84757d6f57df 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1897,6 +1897,92 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return 0; #endif +int32_t transInitBuffer(SConnBuffer* buf) { + buf->buf = taosMemoryCalloc(1, BUFFER_CAP); + if (buf->buf == NULL) { + return terrno; + } + + buf->cap = BUFFER_CAP; + buf->left = -1; + buf->len = 0; + buf->total = 0; + buf->invalid = 0; + return 0; +} +void transDestroyBuffer(SConnBuffer* p) { + taosMemoryFree(p->buf); + p->buf = NULL; +} + +int32_t transClearBuffer(SConnBuffer* buf) { + SConnBuffer* p = buf; + if (p->cap > BUFFER_CAP) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); + if (p->buf == NULL) { + return terrno; + } + } + p->left = -1; + p->len = 0; + p->total = 0; + p->invalid = 0; + return 0; +} + +int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { + static const int HEADSIZE = sizeof(STransMsgHead); + int32_t code = 0; + SConnBuffer* p = connBuf; + if (p->left != 0 || p->total <= 0) { + return TSDB_CODE_INVALID_MSG; + } + int total = p->total; + if (total >= HEADSIZE && !p->invalid) { + *buf = taosMemoryCalloc(1, total); + if (*buf == NULL) { + return terrno; + } + memcpy(*buf, p->buf, total); + if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { + return code; + } + } else { + total = -1; + return TSDB_CODE_INVALID_MSG; + } + return total; +} + +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { + SConnBuffer* p = connBuf; + if (p->total < p->len) { + int left = p->len - p->total; + memmove(p->buf, p->buf + p->total, left); + p->left = -1; + p->total = 0; + p->len = left; + } else if (p->total == p->len) { + p->left = -1; + p->total = 0; + p->len = 0; + if (p->cap > BUFFER_CAP) { + if (resetBuf) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, p->cap); + if (p->buf == NULL) { + return terrno; + } + } + } + } else { + tError("failed to reset buffer, total:%d, len:%d since %s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); + return TSDB_CODE_INVALID_MSG; + } + return 0; +} + int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) { if (pEpset == NULL) { return TSDB_CODE_INVALID_PARA; diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index bd4b9cf164f3..fae4eaba185d 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -96,6 +96,7 @@ typedef struct SSvrConn { int32_t bufSize; queue wq; // uv_write_t queue int32_t fd; + } SSvrConn; typedef struct SSvrRespMsg { SSvrConn *pConn; @@ -124,6 +125,12 @@ static FORCE_INLINE void destroySmsg(SSvrRespMsg *smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } +typedef struct { + char *buf; + int32_t len; + int8_t inited; + void *data; +} SEvtBuf; typedef struct SWorkThrd { TdThread thread; // //uv_connect_t connect_req; @@ -170,9 +177,9 @@ typedef struct SServerObj { bool inited; } SServerObj2; -typedef void (*__sendCb)(SFdArg *arg, int32_t status); -typedef void (*__readCb)(SFdArg *arg, int32_t status); -typedef void (*__asyncCb)(SFdArg *arg, int32_t status); +typedef void (*__sendCb)(void *arg, int32_t status); +typedef int32_t (*__readCb)(void *arg, SEvtBuf *buf, int32_t status); +typedef void (*__asyncCb)(void *arg, int32_t status); enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T, EVT_NEW_CONN_T }; @@ -181,12 +188,13 @@ typedef struct { void *data; int32_t event; - __sendCb sendFn; - __readCb readFn; - __asyncCb asyncFn; + __sendCb sendCb; + __readCb readCb; + __asyncCb asyncCb; - int8_t evtType; - void *arg; + int8_t evtType; + void *arg; + SEvtBuf buf; } SFdCbArg; typedef struct { @@ -242,6 +250,19 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { // int32_t selectUtilRange() +static int32_t evtMayShoudInitBuf(SEvtBuf *evtBuf) { + int32_t code = 0; + if (evtBuf->inited == 0) { + evtBuf->buf = taosMemoryCalloc(1, 4096); + if (evtBuf->buf == NULL) { + code = terrno; + } else { + evtBuf->len = 4096; + evtBuf->inited = 1; + } + } + return code; +} int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { int32_t nBytes = 0; char buf[2] = {0}; @@ -260,14 +281,25 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { } } else if (pArg->event == EVT_CONN_T) { if (res & EVT_READ) { - pArg->readFn(NULL, 0); + SEvtBuf *pBuf = &pArg->buf; + code = evtMayShoudInitBuf(pBuf); + if (code != 0) { + tError("failed to init buf since %s", tstrerror(code)); + return code; + } + + nBytes = read(pArg->fd, pBuf->buf, pBuf->len); + if (nBytes > 0) { + code = pArg->readCb(pArg, pBuf, nBytes); + } // handle read } if (res & EVT_WRITE) { - pArg->sendFn(NULL, 0); + pArg->sendCb(NULL, 0); // handle write } } + return code; } int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { @@ -529,7 +561,7 @@ void uvDestroyResp(void *e) { SSvrRespMsg *pMsg = QUEUE_DATA(e, SSvrRespMsg, q); destroySmsg(pMsg); } -static int32_t connGetBasicInfo(SSvrConn *pConn) { +static int32_t connGetSockInfo(SSvrConn *pConn) { int32_t code = 0; struct sockaddr_in addr; @@ -554,6 +586,7 @@ static int32_t connGetBasicInfo(SSvrConn *pConn) { port = ntohs(addr.sin_port); snprintf(pConn->src, sizeof(pConn->src), "%s:%d", ip_str, port); + return code; } static SSvrConn *createConn(void *tThrd, int32_t fd) { @@ -567,12 +600,12 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { pConn->fd = fd; QUEUE_INIT(&pConn->queue); - code = connGetBasicInfo(pConn); + code = connGetSockInfo(pConn); TAOS_CHECK_GOTO(code, &lino, _end); - // if ((code = transInitBuffer(&pConn->readBuf)) != 0) { - // TAOS_CHECK_GOTO(code, &lino, _end); - // } + if ((code = transInitBuffer(&pConn->readBuf)) != 0) { + TAOS_CHECK_GOTO(code, &lino, _end); + } // if ((code = transQueueInit(&pConn->resps, uvDestroyResp)) != 0) { // TAOS_CHECK_GOTO(code, &lino, _end); @@ -632,6 +665,148 @@ static void destroryConn(SSvrConn *pConn) { QUEUE_REMOVE(&pConn->queue); taosMemoryFree(pConn); } + +bool uvConnMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { + if (pConn->userInited) { + return false; + } + + STrans *pInst = pConn->pInst; + STransMsgHead *pHead = *ppHead; + int32_t len = *msgLen; + if (pHead->withUserInfo) { + STransMsgHead *tHead = taosMemoryCalloc(1, len - sizeof(pInst->user)); + if (tHead == NULL) { + tError("conn %p failed to get user info since %s", pConn, tstrerror(terrno)); + return false; + } + memcpy((char *)tHead, (char *)pHead, TRANS_MSG_OVERHEAD); + memcpy((char *)tHead + TRANS_MSG_OVERHEAD, (char *)pHead + TRANS_MSG_OVERHEAD + sizeof(pInst->user), + len - sizeof(STransMsgHead) - sizeof(pInst->user)); + tHead->msgLen = htonl(htonl(pHead->msgLen) - sizeof(pInst->user)); + + memcpy(pConn->user, (char *)pHead + TRANS_MSG_OVERHEAD, sizeof(pConn->user)); + pConn->userInited = 1; + + taosMemoryFree(pHead); + *ppHead = tHead; + *msgLen = len - sizeof(pInst->user); + return true; + } + return false; +} + +static int32_t evtConnHandleReleaseReq(SSvrConn *pConn, STransMsgHead *phead) { + int32_t code = 0; + return code; +} +static int32_t evtSvrHandleRep(SSvrConn *pConn, char *req, int32_t len) { + SWorkThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + int32_t code = 0; + int32_t msgLen = 0; + STransMsgHead *pHead = (STransMsgHead *)req; + if (uvConnMayGetUserInfo(pConn, &pHead, &msgLen) == true) { + tDebug("%s conn %p get user info", transLabel(pInst), pConn); + } else { + if (pConn->userInited == 0) { + taosMemoryFree(pHead); + tDebug("%s conn %p failed get user info since %s", transLabel(pInst), pConn, tstrerror(terrno)); + return TSDB_CODE_INVALID_MSG; + } + tDebug("%s conn %p no need get user info", transLabel(pInst), pConn); + } + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); + + pConn->inType = pHead->msgType; + + STransMsg transMsg = {0}; + transMsg.contLen = transContLenFromMsg(pHead->msgLen); + transMsg.pCont = pHead->content; + transMsg.msgType = pHead->msgType; + transMsg.code = pHead->code; + if (pHead->seqNum == 0) { + STraceId *trace = &pHead->traceId; + tGError("%s conn %p received invalid seqNum, msgType:%s", transLabel(pInst), pConn, TMSG_INFO(pHead->msgType)); + return TSDB_CODE_INVALID_MSG; + } + + transMsg.info.handle = (void *)transAcquireExHandle(uvGetConnRefOfThrd(pThrd), pConn->refId); + transMsg.info.refIdMgt = pThrd->connRefMgt; + + transMsg.info.refId = pHead->noResp == 1 ? -1 : pConn->refId; + transMsg.info.traceId = pHead->traceId; + transMsg.info.cliVer = htonl(pHead->compatibilityVer); + transMsg.info.forbiddenIp = 0; + transMsg.info.noResp = pHead->noResp == 1 ? 1 : 0; + transMsg.info.seq = taosHton64(pHead->seqNum); + transMsg.info.qId = taosHton64(pHead->qid); + transMsg.info.msgType = pHead->msgType; + + SRpcConnInfo *pConnInfo = &(transMsg.info.conn); + pConnInfo->clientIp = pConn->clientIp; + pConnInfo->clientPort = pConn->port; + tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); + + transReleaseExHandle(uvGetConnRefOfThrd(pThrd), pConn->refId); + + (*pInst->cfp)(pInst->parent, &transMsg, NULL); + + return code; +} +static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { + int32_t code = 0; + int32_t lino = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + + if (bytes == 0) { + tDebug("client %s closed", pConn->src); + return TSDB_CODE_RPC_NETWORK_ERROR; + } + SConnBuffer *p = &pConn->readBuf; + if (p->cap - p->len < bytes) { + int32_t newCap = p->cap + bytes; + char *newBuf = taosMemoryRealloc(p->buf, newCap); + if (newBuf == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _end); + } + p->buf = newBuf; + p->cap = newCap; + } + + memcpy(p->buf + p->len, buf->buf, bytes); + p->len += bytes; + + while (p->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy(&head, p->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + if (p->len >= msgLen) { + char *pMsg = taosMemoryCalloc(1, msgLen); + if (pMsg == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _end); + } + memcpy(pMsg, p->buf, msgLen); + memcpy(p->buf + msgLen, p->buf, p->len - msgLen); + p->len -= msgLen; + + code = evtSvrHandleRep(pConn, pMsg, msgLen); + TAOS_CHECK_GOTO(terrno, &lino, _end); + } else { + break; + } + } + + return code; +_end: + if (code != 0) { + tError("failed to handle read since %s", tstrerror(code)); + } + return code; +} void evtNewConnNotifyCb(void *async, int32_t status) { int32_t code = 0; @@ -663,8 +838,12 @@ void evtNewConnNotifyCb(void *async, int32_t status) { tDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); } - SFdCbArg arg = { - .evtType = EVT_CONN_T, .arg = pArg, .fd = pArg->acceptFd, .readFn = NULL, .sendFn = NULL, .data = NULL}; + SFdCbArg arg = {.evtType = EVT_CONN_T, + .arg = pArg, + .fd = pArg->acceptFd, + .readCb = evtSvrReadCb, + .sendCb = NULL, + .data = pConn}; code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); if (code != 0) { From 6efc2583d1dbcaa93587ee8c819d4716e986690c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 18:27:03 +0800 Subject: [PATCH 34/76] refactor code --- source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/transImpl2.c | 188 ++++++++++++++++++++++- 2 files changed, 185 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index fec9d96b04c9..c4c7fe4ee4cf 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -159,6 +159,7 @@ typedef struct { TdThreadMutex seqMutex; int64_t seq; SHashObj* seqTable; + int8_t shareConnLimit; } SRpcInfo; #endif diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index fae4eaba185d..330ac3403d87 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -803,7 +803,7 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { return code; _end: if (code != 0) { - tError("failed to handle read since %s", tstrerror(code)); + tError("%s failed to handle read at line %d since %s", __func__, lino, tstrerror(code)); } return code; } @@ -1150,9 +1150,10 @@ typedef struct SCliConn { queue wq; // uv_write_t queue - queue batchSendq; - int8_t inThreadSendq; - + queue batchSendq; + int8_t inThreadSendq; + int32_t fd; + queue reqsToSend2; } SCliConn; typedef struct { @@ -1239,6 +1240,110 @@ typedef struct SCliObj2 { SCliThrd2 **pThreadObj; } SCliObj2; +static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { + int32_t code = 0; + int32_t line = 0; + int32_t sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + if (inet_pton(AF_INET, ip, &server_addr.sin_addr) <= 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + + if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + + return code; +_end: + if (code != 0) { + tError("%s failed to connect to %s:%d at line %d since %s", __func__, ip, port, line, tstrerror(code)); + } + return code; +} +static int32_t cliConnGetSockInfo(SCliConn *pConn) { + int32_t code = 0; + int32_t line = 0; + struct sockaddr_in addr; + + socklen_t addr_len = sizeof(addr); + char ip_str[INET_ADDRSTRLEN]; + int port; + + // 获取对端地址 + if (getpeername(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->dst, sizeof(pConn->dst), "%s:%d", ip_str, port); + + // 获取本地地址 + if (getsockname(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->src, sizeof(pConn->src), "%s:%d", ip_str, port); + return code; +_end: + if (code != 0) { + tError("%s failed to get sock info at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} +static int32_t createCliConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { + int32_t code = 0; + int32_t line = 0; + STrans *pInst = pThrd->pInst; + SCliConn *pConn = taosMemoryCalloc(1, sizeof(SCliConn)); + if (pConn == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", ip, port); + pConn->hostThrd = pThrd; + pConn->dstAddr = taosStrdup(addr); + pConn->ipStr = taosStrdup(ip); + pConn->port = port; + if (pConn->dstAddr == NULL || pConn->ipStr == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &line, _end); + } + pConn->status = ConnNormal; + pConn->broken = false; + QUEUE_INIT(&pConn->q); + + TAOS_CHECK_GOTO(transInitBuffer(&pConn->readBuf), NULL, _end); + pConn->seq = 0; + + pConn->pQTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pConn->pQTable == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + QUEUE_INIT(&pConn->batchSendq); + pConn->bufSize = pInst->shareConnLimit; + return code; + + QUEUE_INIT(&pConn->reqsToSend2); + + TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); + TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); + return code; + +_end: + if (code != 0) { + // TODO, delete conn mem + tError("%s failed to create conn at line %d since %s", __func__, line, tstrerror(code)); + taosMemoryFree(pConn); + } + return code; +} + static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { if (ctx) { taosMemoryFree(ctx->epSet); @@ -1434,6 +1539,81 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, return code; } +static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { + int32_t code = 0; + return code; +} +static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { + int32_t code; + int32_t line = 0; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + if (bytes == 0) { + tDebug("client %s closed", pConn->src); + return TSDB_CODE_RPC_NETWORK_ERROR; + } + SConnBuffer *p = &pConn->readBuf; + if (p->cap - p->len < bytes) { + int32_t newCap = p->cap + bytes; + char *newBuf = taosMemoryRealloc(p->buf, newCap); + if (newBuf == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + p->buf = newBuf; + p->cap = newCap; + } + + memcpy(p->buf + p->len, buf->buf, bytes); + p->len += bytes; + + while (p->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy(&head, p->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + if (p->len >= msgLen) { + char *pMsg = taosMemoryCalloc(1, msgLen); + if (pMsg == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + memcpy(pMsg, p->buf, msgLen); + memcpy(p->buf + msgLen, p->buf, p->len - msgLen); + p->len -= msgLen; + + code = evtCliHandleResp(pConn, pMsg, msgLen); + TAOS_CHECK_GOTO(terrno, &line, _end); + } else { + break; + } + } + + return code; +_end: + if (code != 0) { + tError("%s failed to handle resp at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} +static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { + int32_t code = 0; + char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); + int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); + + SCliConn *pConn = NULL; + code = createCliConn(pThrd, fqdn, port, &pConn); + if (code != 0) { + tError("failed to create conn since %s", tstrerror(code)); + return code; + } else { + STraceId *trace = &req->msg.info.traceId; + tGDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); + QUEUE_PUSH(&pConn->reqsToSend2, &req->q); + + SFdCbArg arg = { + .evtType = EVT_CONN_T, .arg = NULL, .fd = pConn->fd, .readCb = evtCliReadResp, .sendCb = NULL, .data = pConn}; + code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ, &arg); + } + return code; +} static void evtHandleCliReqCb(void *arg, int32_t status) { int32_t code = 0; SAsyncHandle *handle = arg; From 9b3632218ffe315600f28d12b8be9d75495e05a8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 20:25:31 +0800 Subject: [PATCH 35/76] refactor code --- source/libs/transport/src/transImpl2.c | 138 +++++++++++++++++++++---- 1 file changed, 118 insertions(+), 20 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 330ac3403d87..c06752e55a3b 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -46,6 +46,7 @@ typedef struct { AsyncCb cb; queue q; TdThreadMutex mutex; + void *hostThrd; } SAsyncHandle; typedef struct { queue q; @@ -177,8 +178,9 @@ typedef struct SServerObj { bool inited; } SServerObj2; -typedef void (*__sendCb)(void *arg, int32_t status); +typedef int32_t (*__sendCb)(void *arg, SEvtBuf *buf, int32_t status); typedef int32_t (*__readCb)(void *arg, SEvtBuf *buf, int32_t status); +typedef int32_t (*__sendFinishCb)(void *arg, int32_t status); typedef void (*__asyncCb)(void *arg, int32_t status); enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T, EVT_NEW_CONN_T }; @@ -188,13 +190,16 @@ typedef struct { void *data; int32_t event; - __sendCb sendCb; - __readCb readCb; - __asyncCb asyncCb; + __sendCb sendCb; + __sendFinishCb sendFinishCb; + __readCb readCb; + __asyncCb asyncCb; int8_t evtType; void *arg; SEvtBuf buf; + SEvtBuf sendBuf; + int8_t rwRef; } SFdCbArg; typedef struct { @@ -206,7 +211,7 @@ typedef struct { fd_set *evtReadSetOut; fd_set *evtWriteSetOut; SHashObj *pFdTable; - void *arg; + void *hostThrd; int32_t fd[2048]; int32_t fdIdx; @@ -291,12 +296,48 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { nBytes = read(pArg->fd, pBuf->buf, pBuf->len); if (nBytes > 0) { code = pArg->readCb(pArg, pBuf, nBytes); + } else { + code = pArg->readCb(pArg, pBuf, nBytes); } // handle read } + if (res & EVT_WRITE) { - pArg->sendCb(NULL, 0); - // handle write + SEvtBuf *pBuf = &pArg->sendBuf; + + code = evtMayShoudInitBuf(pBuf); + if (code != 0) { + tError("failed to init wbuf since %s", tstrerror(code)); + return code; + } + + code = pArg->sendCb(pArg->arg, pBuf, 0); + if (code != 0) { + tError("failed to build send buf since %s", tstrerror(code)); + } + int32_t total = pBuf->len; + int32_t offset = 0; + do { + int32_t n = send(pArg->fd, pBuf->buf + offset, total, 0); + if (n < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + code = TAOS_SYSTEM_ERROR(errno); + break; + } + code = TAOS_SYSTEM_ERROR(errno); + break; + } + offset += n; + total -= n; + } while (total > 0); + + if (code != 0) { + tError("failed to send buf since %s", tstrerror(code)); + pArg->sendFinishCb(pArg->arg, code); + return code; + } + code = evtMgtRemove(pOpt, pArg->fd, EVT_WRITE, pArg); + return code; } } @@ -417,13 +458,16 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar } pOpt->evtFds = fd; } + int8_t rwRef = 0; if (fd != 0) { if (events & EVT_READ) { FD_SET(fd, pOpt->evtReadSetIn); + rwRef++; } if (events & EVT_WRITE) { FD_SET(fd, pOpt->evtWriteSetIn); + rwRef++; } pOpt->fd[pOpt->fdIdx++] = fd; code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); @@ -433,29 +477,35 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar } static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { + int32_t line = 0; int32_t code = 0; ASSERT((events & EVT_SIGNAL) == 0); - + int8_t rwRef = 0; if (pOpt->evtFds < fd) { return TAOS_SYSTEM_ERROR(EBADF); } if (events & EVT_READ) { FD_CLR(fd, pOpt->evtReadSetIn); + rwRef++; } if (events & EVT_WRITE) { FD_CLR(fd, pOpt->evtWriteSetIn); + rwRef++; } SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); if (pArg == NULL) { - // TODO, destroy pArg - } else { - code = taosHashRemove(pOpt->pFdTable, &fd, sizeof(fd)); + tError("%s failed to get fd %d since %s", __func__, fd, tstrerror(TAOS_SYSTEM_ERROR(EBADF))); + return 0; } - for (int32_t i = 0; i < sizeof(pOpt->fd) / sizeof(pOpt->fd[0]); i++) { - if (pOpt->fd[i] == fd) { - pOpt->fd[i] = -1; + pArg->rwRef -= rwRef; + if (pArg->rwRef == 0) { + code = taosHashRemove(pOpt->pFdTable, &fd, sizeof(fd)); + for (int32_t i = 0; i < sizeof(pOpt->fd) / sizeof(pOpt->fd[0]); i++) { + if (pOpt->fd[i] == fd) { + pOpt->fd[i] = -1; + } } } return code; @@ -490,6 +540,7 @@ static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, if (pAsync == NULL) { return terrno; } + pAsync->hostThrd = pThrd; pAsync->data = pOpt; taosThreadMutexInit(&pAsync->mutex, NULL); @@ -829,7 +880,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); - SSvrConn *pConn = createConn(pEvtMgt->arg, pArg->acceptFd); + SSvrConn *pConn = createConn(pEvtMgt->hostThrd, pArg->acceptFd); if (pConn == NULL) { tError("failed to create conn since %s", tstrerror(code)); taosMemoryFree(pArg); @@ -854,6 +905,12 @@ void evtNewConnNotifyCb(void *async, int32_t status) { return; } + +int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { + // TODO + int32_t code = 0; + return code; +} void evtHandleRespCb(void *async, int32_t status) { int32_t code = 0; @@ -874,6 +931,7 @@ void evtHandleRespCb(void *async, int32_t status) { QUEUE_REMOVE(el); SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); + code = evtSvrHandleSendResp(handle->hostThrd, pResp); } return; @@ -895,7 +953,7 @@ void *transWorkerThread(void *arg) { TAOS_CHECK_GOTO(code, &line, _end); } - pOpt->arg = pThrd; + pOpt->hostThrd = pThrd; code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_CONN_T, (void *)pThrd); if (code != 0) { @@ -1541,6 +1599,21 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { int32_t code = 0; + + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SCliReq *pReq = NULL; + STransMsgHead *pHead = (STransMsgHead *)msg; + + int64_t qId = taosHton64(pHead->qid); + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); + int64_t seq = taosHton64(pHead->seqNum); + + STransMsg resp = {0}; + (pInst->cfp)(pInst->parent, &resp, NULL); + return code; } static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { @@ -1593,6 +1666,30 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } + +static int32_t evtCliSendReq(SCliConn *pConn, SCliReq *pReq) { + int32_t code = 0; + // int32_t code = 0; + // STransMsg *msg = &pReq->msg; + // STransMsgHead head = {0}; + // head.qid = taosHton64(msg->info.qId); + // head.seqNum = taosHton64(msg->info.traceId.msgId); + // head.msgLen = htonl(msg->pCont->len); + + // char *pMsg = taosMemoryCalloc(1, sizeof(head) + msg->pCont->len); + // if (pMsg == NULL) { + // return TSDB_CODE_OUT_OF_MEMORY; + // } + // memcpy(pMsg, &head, sizeof(head)); + // memcpy(pMsg + sizeof(head), msg->pCont->data, msg->pCont->len); + + // code = evtAsyncSend(pConn->hostThrd->asyncHandle, pMsg); + // if (code != 0) { + // taosMemoryFree(pMsg); + // return code; + // } + return code; +} static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { int32_t code = 0; char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); @@ -1604,13 +1701,14 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { tError("failed to create conn since %s", tstrerror(code)); return code; } else { + QUEUE_PUSH(&pConn->reqsToSend2, &req->q); STraceId *trace = &req->msg.info.traceId; tGDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); - QUEUE_PUSH(&pConn->reqsToSend2, &req->q); SFdCbArg arg = { .evtType = EVT_CONN_T, .arg = NULL, .fd = pConn->fd, .readCb = evtCliReadResp, .sendCb = NULL, .data = pConn}; - code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ, &arg); + + code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); } return code; } @@ -1619,7 +1717,7 @@ static void evtHandleCliReqCb(void *arg, int32_t status) { SAsyncHandle *handle = arg; SEvtMgt *pEvtMgt = handle->data; - SCliThrd2 *pThrd = pEvtMgt->arg; + SCliThrd2 *pThrd = pEvtMgt->hostThrd; queue wq; QUEUE_INIT(&wq); @@ -1668,7 +1766,7 @@ static void *cliWorkThread2(void *arg) { TAOS_CHECK_GOTO(code, &line, _end); } - pThrd->pEvtMgt->arg = pThrd; + pThrd->pEvtMgt->hostThrd = pThrd; code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReqCb, EVT_ASYNC_T, (void *)pThrd); From 696f40ff8703863461e647f41e9b485d9b9a2960 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 20:47:24 +0800 Subject: [PATCH 36/76] refactor code --- source/libs/transport/src/transImpl2.c | 50 +++++++++++++------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index c06752e55a3b..7209d5049088 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1616,6 +1616,23 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { return code; } +static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0; + SCliConn *pConn = arg; + + return code; +} + +static int32_t evtCliSendCb(void *arg, int32_t status) { + int32_t code = status; + SCliConn *pConn = arg; + if (code != 0) { + tError("failed to send request since %s", tstrerror(code)); + return code; + } + return code; +} + static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { int32_t code; int32_t line = 0; @@ -1666,30 +1683,6 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } - -static int32_t evtCliSendReq(SCliConn *pConn, SCliReq *pReq) { - int32_t code = 0; - // int32_t code = 0; - // STransMsg *msg = &pReq->msg; - // STransMsgHead head = {0}; - // head.qid = taosHton64(msg->info.qId); - // head.seqNum = taosHton64(msg->info.traceId.msgId); - // head.msgLen = htonl(msg->pCont->len); - - // char *pMsg = taosMemoryCalloc(1, sizeof(head) + msg->pCont->len); - // if (pMsg == NULL) { - // return TSDB_CODE_OUT_OF_MEMORY; - // } - // memcpy(pMsg, &head, sizeof(head)); - // memcpy(pMsg + sizeof(head), msg->pCont->data, msg->pCont->len); - - // code = evtAsyncSend(pConn->hostThrd->asyncHandle, pMsg); - // if (code != 0) { - // taosMemoryFree(pMsg); - // return code; - // } - return code; -} static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { int32_t code = 0; char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); @@ -1705,8 +1698,13 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { STraceId *trace = &req->msg.info.traceId; tGDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); - SFdCbArg arg = { - .evtType = EVT_CONN_T, .arg = NULL, .fd = pConn->fd, .readCb = evtCliReadResp, .sendCb = NULL, .data = pConn}; + SFdCbArg arg = {.evtType = EVT_CONN_T, + .arg = pConn, + .fd = pConn->fd, + .readCb = evtCliReadResp, + .sendCb = evtCliPreSendReq, + .sendFinishCb = evtCliSendCb, + .data = pConn}; code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); } From d5666fa3e8840da01379facf275b256e138982e3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Dec 2024 20:55:37 +0800 Subject: [PATCH 37/76] refactor code --- source/libs/transport/src/transImpl2.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 7209d5049088..1ccd6db712e1 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -311,7 +311,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { return code; } - code = pArg->sendCb(pArg->arg, pBuf, 0); + code = pArg->sendCb(pArg, pBuf, 0); if (code != 0) { tError("failed to build send buf since %s", tstrerror(code)); } @@ -333,7 +333,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (code != 0) { tError("failed to send buf since %s", tstrerror(code)); - pArg->sendFinishCb(pArg->arg, code); + pArg->sendFinishCb(pArg, code); return code; } code = evtMgtRemove(pOpt, pArg->fd, EVT_WRITE, pArg); @@ -1617,15 +1617,18 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { return code; } static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0; - SCliConn *pConn = arg; - + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + return code; } static int32_t evtCliSendCb(void *arg, int32_t status) { int32_t code = status; - SCliConn *pConn = arg; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; if (code != 0) { tError("failed to send request since %s", tstrerror(code)); return code; From 85e9b7ddad2229451bf43f19566c46eb7b7cb11a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 11:15:49 +0800 Subject: [PATCH 38/76] refactor code --- source/libs/transport/src/transImpl2.c | 173 +++++++++++++++++++++++-- 1 file changed, 161 insertions(+), 12 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 1ccd6db712e1..84292258bd6d 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -130,7 +130,9 @@ typedef struct { char *buf; int32_t len; int8_t inited; + int32_t offset; void *data; + } SEvtBuf; typedef struct SWorkThrd { TdThread thread; @@ -255,7 +257,7 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { // int32_t selectUtilRange() -static int32_t evtMayShoudInitBuf(SEvtBuf *evtBuf) { +static int32_t evtBufInit(SEvtBuf *evtBuf) { int32_t code = 0; if (evtBuf->inited == 0) { evtBuf->buf = taosMemoryCalloc(1, 4096); @@ -264,10 +266,49 @@ static int32_t evtMayShoudInitBuf(SEvtBuf *evtBuf) { } else { evtBuf->len = 4096; evtBuf->inited = 1; + evtBuf->offset = 0; + } + } + return code; +} +static int32_t evtBufPush(SEvtBuf *evtBuf, char *buf, int32_t len) { + int32_t code = 0; + if (evtBuf->inited == 0) { + code = evtBufInit(evtBuf); + if (code != 0) { + return code; } } + int32_t need = evtBuf->offset + len; + if (need >= evtBuf->len) { + // TOOD opt need + char *tbuf = taosMemoryRealloc(evtBuf->buf, need); + if (tbuf == NULL) { + return terrno; + } + evtBuf->buf = tbuf; + evtBuf->len = need; + } + memcpy(evtBuf->buf + evtBuf->offset, buf, len); + evtBuf->offset += len; return code; } + +static int32_t evtBufClear(SEvtBuf *evtBuf) { + int32_t code = 0; + if (evtBuf->inited) { + evtBuf->offset = 0; + memset(evtBuf->buf, 0, evtBuf->len); + } + return code; +} + +static int32_t evtBufDestroy(SEvtBuf *evtBuf) { + if (evtBuf->inited) { + taosMemoryFree(evtBuf->buf); + } + return 0; +} int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { int32_t nBytes = 0; char buf[2] = {0}; @@ -287,7 +328,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { } else if (pArg->event == EVT_CONN_T) { if (res & EVT_READ) { SEvtBuf *pBuf = &pArg->buf; - code = evtMayShoudInitBuf(pBuf); + code = evtBufInit(pBuf); if (code != 0) { tError("failed to init buf since %s", tstrerror(code)); return code; @@ -305,7 +346,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (res & EVT_WRITE) { SEvtBuf *pBuf = &pArg->sendBuf; - code = evtMayShoudInitBuf(pBuf); + code = evtBufInit(pBuf); if (code != 0) { tError("failed to init wbuf since %s", tstrerror(code)); return code; @@ -1616,12 +1657,109 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { return code; } +bool connMayAddUserInfo(SCliConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + if (pConn->userInited == 1) { + return false; + } + STransMsgHead *pHead = *ppHead; + STransMsgHead *tHead = taosMemoryCalloc(1, *msgLen + sizeof(pInst->user)); + if (tHead == NULL) { + return false; + } + memcpy((char *)tHead, (char *)pHead, TRANS_MSG_OVERHEAD); + memcpy((char *)tHead + TRANS_MSG_OVERHEAD, pInst->user, sizeof(pInst->user)); + + memcpy((char *)tHead + TRANS_MSG_OVERHEAD + sizeof(pInst->user), (char *)pHead + TRANS_MSG_OVERHEAD, + *msgLen - TRANS_MSG_OVERHEAD); + + tHead->withUserInfo = 1; + *ppHead = tHead; + *msgLen += sizeof(pInst->user); + + pConn->pInitUserReq = tHead; + pConn->userInited = 1; + return true; +} static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { - - int32_t code = 0; - SFdCbArg *pArg = arg; - SCliConn *pConn = pArg->data; - + int32_t code = 0, line = 0; + code = evtBufInit(buf); + TAOS_CHECK_GOTO(code, &line, _end); + + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + int32_t j = 0; + int32_t batchLimit = 16; + queue reqToSend; + QUEUE_INIT(&reqToSend); + while (!QUEUE_IS_EMPTY(&pConn->reqsToSend2)) { + queue *el = QUEUE_HEAD(&pConn->reqsToSend2); + QUEUE_REMOVE(el); + + SCliReq *pCliMsg = QUEUE_DATA(el, SCliReq, q); + SReqCtx *pCtx = pCliMsg->ctx; + pConn->seq++; + + STransMsg *pReq = (STransMsg *)(&pCliMsg->msg); + if (pReq->pCont == 0) { + pReq->pCont = (void *)rpcMallocCont(0); + if (pReq->pCont == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->contLen = 0; + } + int32_t msgLen = transMsgLenFromCont(pReq->contLen); + STransMsgHead *pHead = transHeadFromCont(pReq->pCont); + + char *content = pReq->pCont; + int32_t contLen = pReq->contLen; + if (connMayAddUserInfo(pConn, &pHead, &msgLen)) { + content = transContFromHead(pHead); + contLen = transContLenFromMsg(msgLen); + pReq->pCont = (void *)pHead; + } else { + if (pConn->userInited == 0) { + return terrno; + } + } + if (pHead->comp == 0) { + pHead->noResp = (pReq->info.noResp ? 1 : 0); + pHead->msgType = pReq->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->traceId = pReq->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->version = TRANS_VER; + pHead->compatibilityVer = htonl(pInst->compatibilityVer); + } + pHead->timestamp = taosHton64(pCliMsg->st); + pHead->seqNum = taosHton64(pConn->seq); + pHead->qid = taosHton64(pReq->info.qId); + + pCliMsg->seq = pConn->seq; + pCliMsg->sent = 1; + + QUEUE_PUSH(&pConn->reqsSentOut.node, &pCliMsg->q); + + QUEUE_INIT(&pCliMsg->sendQ); + QUEUE_PUSH(&reqToSend, &pCliMsg->sendQ); + + code = evtBufPush(buf, (char *)pHead, msgLen); + TAOS_CHECK_GOTO(code, &line, _end); + j++; + + if (j >= batchLimit) { + break; + } + } + return code; +_end: + if (code != 0) { + tError("%s failed to send request at line %d since %s", __func__, line, tstrerror(code)); + } return code; } @@ -1629,10 +1767,24 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { int32_t code = status; SFdCbArg *pArg = arg; SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; if (code != 0) { tError("failed to send request since %s", tstrerror(code)); return code; } + while (!QUEUE_IS_EMPTY(&pConn->reqsSentOut.node)) { + queue *el = QUEUE_HEAD(&pConn->reqsSentOut.node); + QUEUE_REMOVE(el); + + SCliReq *pCliMsg = QUEUE_DATA(el, SCliReq, q); + if (pCliMsg == NULL) { + continue; + } + STraceId *trace = &pCliMsg->msg.info.traceId; + tGDebug("success to send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pThrd->pid, + pCliMsg->ctx->epSet->eps[0].fqdn, pCliMsg->ctx->epSet->eps[0].port, pCliMsg->msg.info.ahandle); + } + return code; } @@ -1742,10 +1894,7 @@ static void evtHandleCliReqCb(void *arg, int32_t status) { tGDebug("handle request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pThrd->pid, pReq->ctx->epSet->eps[0].fqdn, pReq->ctx->epSet->eps[0].port, pReq->msg.info.ahandle); - if (pReq->msg.info.handle == 0) { - destroyReq(pReq); - continue; - } + code = evtHandleCliReq(pThrd, pReq); } } From 77ada938987bcd72f72ae72ebf5538e0205f1e66 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 15:28:06 +0800 Subject: [PATCH 39/76] refactor code --- source/libs/transport/src/transImpl2.c | 159 ++++++++++++++++++++----- 1 file changed, 127 insertions(+), 32 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 84292258bd6d..3b54dd631066 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -99,6 +99,15 @@ typedef struct SSvrConn { int32_t fd; } SSvrConn; + +typedef struct { + char *buf; + int32_t len; + int8_t inited; + int32_t offset; + void *data; + +} SEvtBuf; typedef struct SSvrRespMsg { SSvrConn *pConn; STransMsg msg; @@ -109,6 +118,7 @@ typedef struct SSvrRespMsg { FilteFunc func; int8_t sent; + SEvtBuf buf; } SSvrRespMsg; #define ASYNC_ERR_JRET(thrd) \ @@ -126,14 +136,6 @@ static FORCE_INLINE void destroySmsg(SSvrRespMsg *smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -typedef struct { - char *buf; - int32_t len; - int8_t inited; - int32_t offset; - void *data; - -} SEvtBuf; typedef struct SWorkThrd { TdThread thread; // //uv_connect_t connect_req; @@ -163,6 +165,8 @@ typedef struct SWorkThrd { int32_t pipe_queue_fd[2]; SAsyncHandle *asyncHandle; + + void *pEvtMgt; } SWorkThrd2; typedef struct SServerObj { TdThread thread; @@ -374,10 +378,10 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (code != 0) { tError("failed to send buf since %s", tstrerror(code)); + evtBufClear(&pArg->sendBuf); pArg->sendFinishCb(pArg, code); return code; } - code = evtMgtRemove(pOpt, pArg->fd, EVT_WRITE, pArg); return code; } } @@ -482,6 +486,7 @@ static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap) { static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { // add new fd to the set + int8_t rwRef = 0; int32_t code = 0; if (pOpt->evtFds < fd) { int32_t fdSize = pOpt->evtFdsSize; @@ -499,19 +504,38 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar } pOpt->evtFds = fd; } - int8_t rwRef = 0; - if (fd != 0) { + + SFdCbArg *p = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (p != NULL) { if (events & EVT_READ) { FD_SET(fd, pOpt->evtReadSetIn); rwRef++; } - if (events & EVT_WRITE) { FD_SET(fd, pOpt->evtWriteSetIn); rwRef++; } - pOpt->fd[pOpt->fdIdx++] = fd; - code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); + p->rwRef += rwRef; + if (p->rwRef >= 2) { + p->rwRef = 2; + } + } else { + if (arg == NULL) { + return TSDB_CODE_INVALID_MSG; + } + if (fd != 0) { + if (events & EVT_READ) { + FD_SET(fd, pOpt->evtReadSetIn); + rwRef++; + } + + if (events & EVT_WRITE) { + FD_SET(fd, pOpt->evtWriteSetIn); + rwRef++; + } + pOpt->fd[pOpt->fdIdx++] = fd; + code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); + } } return 0; @@ -899,6 +923,79 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } +static int32_t evtSvrHandleSendRespImpl(SSvrConn *pConn, SEvtBuf *pBuf) { + int32_t code = 0; + int32_t batchLimit = 32; + int32_t j = 0; + while (!QUEUE_IS_EMPTY(&pConn->resps.node)) { + queue *el = QUEUE_HEAD(&pConn->resps.node); + QUEUE_REMOVE(el); + + SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); + STransMsg *pMsg = &pResp->msg; + if (pMsg->pCont == 0) { + pMsg->pCont = (void *)rpcMallocCont(0); + if (pMsg->pCont == NULL) { + return terrno; + } + pMsg->contLen = 0; + } + STransMsgHead *pHead = transHeadFromCont(pMsg->pCont); + pHead->traceId = pMsg->info.traceId; + pHead->hasEpSet = pMsg->info.hasEpSet; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->compatibilityVer = htonl(((STrans *)pConn->pInst)->compatibilityVer); + pHead->version = TRANS_VER; + pHead->seqNum = taosHton64(pMsg->info.seq); + pHead->qid = taosHton64(pMsg->info.qId); + pHead->withUserInfo = pConn->userInited == 0 ? 1 : 0; + + pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); + pHead->code = htonl(pMsg->code); + pHead->msgLen = htonl(pMsg->contLen + sizeof(STransMsgHead)); + + char *msg = (char *)pHead; + int32_t len = transMsgLenFromCont(pMsg->contLen); + + STrans *pInst = pConn->pInst; + STraceId *trace = &pMsg->info.traceId; + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, len, pMsg->info.seq, + pMsg->info.qId); + evtBufPush(pBuf, (char *)pHead, len); + } + return code; +} +static int32_t evtSvtPreSend(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + + code = evtSvrHandleSendRespImpl(pConn, buf); + return code; +} + +static int32_t evtSvrSendCb(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + SWorkThrd2 *pThrd = pConn->hostThrd; + + return code; +} +static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { + int32_t code = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + + SWorkThrd2 *pThrd = pConn->hostThrd; + + if (QUEUE_IS_EMPTY(&pConn->resps.node)) { + // stop write evt + code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); + } + return code; +} void evtNewConnNotifyCb(void *async, int32_t status) { int32_t code = 0; @@ -934,7 +1031,8 @@ void evtNewConnNotifyCb(void *async, int32_t status) { .arg = pArg, .fd = pArg->acceptFd, .readCb = evtSvrReadCb, - .sendCb = NULL, + .sendCb = evtSvrSendCb, + .sendFinishCb = evtSvrSendFinishCb, .data = pConn}; code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); @@ -947,9 +1045,14 @@ void evtNewConnNotifyCb(void *async, int32_t status) { return; } -int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { - // TODO - int32_t code = 0; +static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { + int32_t code = 0; + SSvrConn *pConn = pResp->pConn; + QUEUE_PUSH(&pConn->resps.node, &pResp->q); + if (QUEUE_IS_EMPTY(&pConn->resps.node)) { + return code; + } + code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); return code; } void evtHandleRespCb(void *async, int32_t status) { @@ -994,6 +1097,7 @@ void *transWorkerThread(void *arg) { TAOS_CHECK_GOTO(code, &line, _end); } + pThrd->pEvtMgt = pOpt; pOpt->hostThrd = pThrd; code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_CONN_T, (void *)pThrd); @@ -1764,25 +1868,16 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { } static int32_t evtCliSendCb(void *arg, int32_t status) { - int32_t code = status; - SFdCbArg *pArg = arg; - SCliConn *pConn = pArg->data; + int32_t code = status; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; SCliThrd2 *pThrd = pConn->hostThrd; if (code != 0) { tError("failed to send request since %s", tstrerror(code)); return code; } - while (!QUEUE_IS_EMPTY(&pConn->reqsSentOut.node)) { - queue *el = QUEUE_HEAD(&pConn->reqsSentOut.node); - QUEUE_REMOVE(el); - - SCliReq *pCliMsg = QUEUE_DATA(el, SCliReq, q); - if (pCliMsg == NULL) { - continue; - } - STraceId *trace = &pCliMsg->msg.info.traceId; - tGDebug("success to send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pThrd->pid, - pCliMsg->ctx->epSet->eps[0].fqdn, pCliMsg->ctx->epSet->eps[0].port, pCliMsg->msg.info.ahandle); + if (QUEUE_IS_EMPTY(&pConn->reqsToSend2)) { + code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } return code; From 3981b167f43a9a098cbba42bc4b2dcdcd71a57d2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 15:33:58 +0800 Subject: [PATCH 40/76] support select --- source/libs/transport/src/transImpl2.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 3b54dd631066..98d1493726fe 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -782,7 +782,7 @@ static void destroryConn(SSvrConn *pConn) { taosMemoryFree(pConn); } -bool uvConnMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { +bool connMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { if (pConn->userInited) { return false; } @@ -823,7 +823,7 @@ static int32_t evtSvrHandleRep(SSvrConn *pConn, char *req, int32_t len) { int32_t code = 0; int32_t msgLen = 0; STransMsgHead *pHead = (STransMsgHead *)req; - if (uvConnMayGetUserInfo(pConn, &pHead, &msgLen) == true) { + if (connMayGetUserInfo(pConn, &pHead, &msgLen) == true) { tDebug("%s conn %p get user info", transLabel(pInst), pConn); } else { if (pConn->userInited == 0) { @@ -878,10 +878,11 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { SFdCbArg *pArg = arg; SSvrConn *pConn = pArg->data; - if (bytes == 0) { + if (bytes <= 0) { tDebug("client %s closed", pConn->src); return TSDB_CODE_RPC_NETWORK_ERROR; } + SConnBuffer *p = &pConn->readBuf; if (p->cap - p->len < bytes) { int32_t newCap = p->cap + bytes; @@ -966,7 +967,7 @@ static int32_t evtSvrHandleSendRespImpl(SSvrConn *pConn, SEvtBuf *pBuf) { } return code; } -static int32_t evtSvtPreSend(void *arg, SEvtBuf *buf, int32_t status) { +static int32_t evtSvrPreSend(void *arg, SEvtBuf *buf, int32_t status) { int32_t code = 0; SFdCbArg *pArg = arg; SSvrConn *pConn = pArg->data; @@ -975,14 +976,6 @@ static int32_t evtSvtPreSend(void *arg, SEvtBuf *buf, int32_t status) { return code; } -static int32_t evtSvrSendCb(void *arg, SEvtBuf *buf, int32_t status) { - int32_t code = 0; - SFdCbArg *pArg = arg; - SSvrConn *pConn = pArg->data; - SWorkThrd2 *pThrd = pConn->hostThrd; - - return code; -} static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { int32_t code = 0; SFdCbArg *pArg = arg; @@ -1031,7 +1024,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { .arg = pArg, .fd = pArg->acceptFd, .readCb = evtSvrReadCb, - .sendCb = evtSvrSendCb, + .sendCb = evtSvrPreSend, .sendFinishCb = evtSvrSendFinishCb, .data = pConn}; code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); From f92cf74243f0adf3c0e9378c277a7149e2fcaf18 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 16:20:25 +0800 Subject: [PATCH 41/76] support select --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/trans.c | 8 +++++++- source/libs/transport/src/transImpl2.c | 14 ++++++++------ source/libs/transport/test/transImpl2_test.cpp | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 4f43eb4d29fc..2f8ec38ca88e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -352,6 +352,7 @@ int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pRe int32_t transFreeConnById(void* pInit, int64_t transpointId); int32_t transSendResponse(STransMsg* msg); +int32_t transSendResponse2(STransMsg* msg); int32_t transRegisterMsg(const STransMsg* msg); int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1d9bd9a1d9ec..41e9d10de567 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -463,7 +463,13 @@ int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRp } int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); } -int32_t rpcSendResponse(SRpcMsg* pMsg) { return transSendResponse(pMsg); } +int32_t rpcSendResponse(SRpcMsg* pMsg) { +#ifndef TD_ACORE + return transSendResponse(pMsg); +#else + return transSendResponse2(pMsg); +#endif +} void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 98d1493726fe..4a0f94f03701 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -189,7 +189,7 @@ typedef int32_t (*__readCb)(void *arg, SEvtBuf *buf, int32_t status); typedef int32_t (*__sendFinishCb)(void *arg, int32_t status); typedef void (*__asyncCb)(void *arg, int32_t status); -enum EVT_TYPE { EVT_ASYNC_T = 0, EVT_CONN_T = 1, EVT_SIGANL_T, EVT_NEW_CONN_T }; +enum EVT_TYPE { EVT_ASYNC_T = 0x1, EVT_CONN_T = 0x2, EVT_SIGANL_T = 0x4, EVT_NEW_CONN_T = 0x8 }; typedef struct { int32_t fd; @@ -1093,7 +1093,8 @@ void *transWorkerThread(void *arg) { pThrd->pEvtMgt = pOpt; pOpt->hostThrd = pThrd; - code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_CONN_T, (void *)pThrd); + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_NEW_CONN_T, + (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); @@ -1179,7 +1180,6 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads goto End; } - // QUEUE_INIT(&thrd->fdQueue.q); { int fd[2] = {0}; code = pipe(fd); @@ -1201,6 +1201,7 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads thrd->pipe_queue_fd[0] = fd2[0]; thrd->pipe_queue_fd[1] = fd2[1]; } + QUEUE_INIT(&thrd->conn); int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); if (err == 0) { @@ -1211,6 +1212,7 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads goto End; } thrd->inited = 1; + srv->pThreadObj[i] = thrd; } code = addHandleToAcceptloop(srv); if (code != 0) { @@ -1454,7 +1456,7 @@ static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); } - + *fd = sockfd; return code; _end: if (code != 0) { @@ -1523,12 +1525,12 @@ static int32_t createCliConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn } QUEUE_INIT(&pConn->batchSendq); pConn->bufSize = pInst->shareConnLimit; - return code; - QUEUE_INIT(&pConn->reqsToSend2); TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); + + *ppConn = pConn; return code; _end: diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 8ed9023baf9c..16f47180195b 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -131,7 +131,7 @@ class Server { taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); } void Start() { - //this->transSrv = rpcOpen(&this->rpcInit_); + this->transSrv = rpcOpen(&this->rpcInit_); taosMsleep(1000); } void SetSrvContinueSend(CB cb) { From 71a4ae00ee66b2158bc7b6774cdbe609b8ad7c26 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 18:26:39 +0800 Subject: [PATCH 42/76] support select --- source/libs/transport/src/transImpl2.c | 42 ++++++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 4a0f94f03701..9cf8a6e4bf80 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -329,7 +329,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (res & EVT_WRITE) { // handle err } - } else if (pArg->event == EVT_CONN_T) { + } else if (pArg->evtType == EVT_CONN_T) { if (res & EVT_READ) { SEvtBuf *pBuf = &pArg->buf; code = evtBufInit(pBuf); @@ -758,11 +758,11 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); } QUEUE_PUSH(&pThrd->conn, &pConn->queue); - + QUEUE_INIT(&pConn->resps.node); // code = initWQ(&pConn->wq); // TAOS_CHECK_GOTO(code, &lino, _end); // wqInited = 1; - + pConn->hostThrd = pThrd; return pConn; _end: if (code != 0) { @@ -816,12 +816,12 @@ static int32_t evtConnHandleReleaseReq(SSvrConn *pConn, STransMsgHead *phead) { int32_t code = 0; return code; } -static int32_t evtSvrHandleRep(SSvrConn *pConn, char *req, int32_t len) { +static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { SWorkThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; int32_t code = 0; - int32_t msgLen = 0; + int32_t msgLen = len; STransMsgHead *pHead = (STransMsgHead *)req; if (connMayGetUserInfo(pConn, &pHead, &msgLen) == true) { tDebug("%s conn %p get user info", transLabel(pInst), pConn); @@ -910,7 +910,7 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { memcpy(p->buf + msgLen, p->buf, p->len - msgLen); p->len -= msgLen; - code = evtSvrHandleRep(pConn, pMsg, msgLen); + code = evtSvrHandleReq(pConn, pMsg, msgLen); TAOS_CHECK_GOTO(terrno, &lino, _end); } else { break; @@ -1048,10 +1048,11 @@ static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); return code; } -void evtHandleRespCb(void *async, int32_t status) { +void evtSvrHandleAyncCb(void *async, int32_t status) { int32_t code = 0; SAsyncHandle *handle = async; + SWorkThrd2 *pThrd = handle->hostThrd; // SEvtMgt *pEvtMgt = handle->data; queue wq; @@ -1068,6 +1069,20 @@ void evtHandleRespCb(void *async, int32_t status) { QUEUE_REMOVE(el); SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); + + STransMsg transMsg = pResp->msg; + SExHandle *exh1 = transMsg.info.handle; + int64_t refId = transMsg.info.refId; + SExHandle *exh2 = transAcquireExHandle(uvGetConnRefOfThrd(pThrd), refId); + if (exh1 != exh2) { + tError("failed to acquire handle since %s", tstrerror(TSDB_CODE_REF_INVALID_ID)); + continue; + } + + pResp->seqNum = transMsg.info.seq; + pResp->pConn = exh2->handle; + + transReleaseExHandle(uvGetConnRefOfThrd(pThrd), refId); code = evtSvrHandleSendResp(handle->hostThrd, pResp); } @@ -1100,7 +1115,7 @@ void *transWorkerThread(void *arg) { TAOS_CHECK_GOTO(code, &line, _end); } - code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleRespCb, EVT_ASYNC_T, (void *)pThrd); + code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtSvrHandleAyncCb, EVT_ASYNC_T, (void *)pThrd); if (code != 0) { tError("failed to create select op since %s", tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); @@ -1527,6 +1542,8 @@ static int32_t createCliConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn pConn->bufSize = pInst->shareConnLimit; QUEUE_INIT(&pConn->reqsToSend2); + QUEUE_INIT(&pConn->reqsToSend.node); + QUEUE_INIT(&pConn->reqsSentOut.node); TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); @@ -1853,6 +1870,7 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { if (j >= batchLimit) { break; } + tDebug("%s send req %p, seq:%" PRId64, pInst->label, pCliMsg, pConn->seq); } return code; _end: @@ -1867,9 +1885,12 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { SFdCbArg *pArg = arg; SCliConn *pConn = pArg->data; SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; if (code != 0) { tError("failed to send request since %s", tstrerror(code)); return code; + } else { + tDebug("%s success to send out request", pInst->label); } if (QUEUE_IS_EMPTY(&pConn->reqsToSend2)) { code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); @@ -1935,13 +1956,14 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { SCliConn *pConn = NULL; code = createCliConn(pThrd, fqdn, port, &pConn); + STrans *pInst = pThrd->pInst; if (code != 0) { - tError("failed to create conn since %s", tstrerror(code)); + tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); return code; } else { QUEUE_PUSH(&pConn->reqsToSend2, &req->q); STraceId *trace = &req->msg.info.traceId; - tGDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); + tGDebug("%s success to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pConn, From 8527bfc313a88fd72d8919fcc0da361d4a954f8c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 20:58:50 +0800 Subject: [PATCH 43/76] support select --- source/libs/transport/src/transImpl2.c | 21 ++++++++++++------- .../libs/transport/test/transImpl2_test.cpp | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 9cf8a6e4bf80..96ead9c6d99b 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -338,7 +338,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { return code; } - nBytes = read(pArg->fd, pBuf->buf, pBuf->len); + nBytes = recv(pArg->fd, pBuf->buf, pBuf->len, 0); if (nBytes > 0) { code = pArg->readCb(pArg, pBuf, nBytes); } else { @@ -360,7 +360,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (code != 0) { tError("failed to build send buf since %s", tstrerror(code)); } - int32_t total = pBuf->len; + int32_t total = pBuf->offset; int32_t offset = 0; do { int32_t n = send(pArg->fd, pBuf->buf + offset, total, 0); @@ -376,11 +376,12 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { total -= n; } while (total > 0); + evtBufClear(&pArg->sendBuf); + pArg->sendFinishCb(pArg, code); if (code != 0) { tError("failed to send buf since %s", tstrerror(code)); - evtBufClear(&pArg->sendBuf); - pArg->sendFinishCb(pArg, code); - return code; + } else { + tError("succ to send buf since %s", tstrerror(code)); } return code; } @@ -399,6 +400,9 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; + struct timeval ttv; + ttv.tv_sec = 5; // 设置为5秒 + ttv.tv_usec = 0; if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; int32_t sz = pOpt->evtFdsSize; @@ -423,7 +427,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { nfds = pOpt->evtFds + 1; // TODO lock or not - active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); + active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, &ttv); if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); } else if (active_Fds == 0) { @@ -714,6 +718,7 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { return NULL; } pConn->fd = fd; + pConn->pInst = pThrd->pInst; QUEUE_INIT(&pConn->queue); code = connGetSockInfo(pConn); @@ -1977,7 +1982,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { } return code; } -static void evtHandleCliReqCb(void *arg, int32_t status) { +static void evtCliHandleAsyncCb(void *arg, int32_t status) { int32_t code = 0; SAsyncHandle *handle = arg; SEvtMgt *pEvtMgt = handle->data; @@ -2030,7 +2035,7 @@ static void *cliWorkThread2(void *arg) { pThrd->pEvtMgt->hostThrd = pThrd; - code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtHandleCliReqCb, EVT_ASYNC_T, + code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtCliHandleAsyncCb, EVT_ASYNC_T, (void *)pThrd); TAOS_CHECK_GOTO(code, &line, _end); diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 16f47180195b..a31ea941eabb 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -300,13 +300,13 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndReq) { - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 1; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 0; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendReq(&req); - taosMsleep(1000); + taosMsleep(100000); assert(resp.code == 0); } } From a64d13f503680bb4f073a0c458c9acd0cd84076e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Dec 2024 22:01:32 +0800 Subject: [PATCH 44/76] support select --- source/libs/transport/src/transImpl2.c | 44 ++++++++++--------- .../libs/transport/test/transImpl2_test.cpp | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 96ead9c6d99b..48203c76ab10 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -259,8 +259,6 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { return code; } -// int32_t selectUtilRange() - static int32_t evtBufInit(SEvtBuf *evtBuf) { int32_t code = 0; if (evtBuf->inited == 0) { @@ -275,6 +273,7 @@ static int32_t evtBufInit(SEvtBuf *evtBuf) { } return code; } + static int32_t evtBufPush(SEvtBuf *evtBuf, char *buf, int32_t len) { int32_t code = 0; if (evtBuf->inited == 0) { @@ -349,7 +348,6 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (res & EVT_WRITE) { SEvtBuf *pBuf = &pArg->sendBuf; - code = evtBufInit(pBuf); if (code != 0) { tError("failed to init wbuf since %s", tstrerror(code)); @@ -381,7 +379,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (code != 0) { tError("failed to send buf since %s", tstrerror(code)); } else { - tError("succ to send buf since %s", tstrerror(code)); + tDebug("succ to send buf"); } return code; } @@ -401,7 +399,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; struct timeval ttv; - ttv.tv_sec = 5; // 设置为5秒 + ttv.tv_sec = 30; // 设置为5秒 ttv.tv_usec = 0; if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; @@ -882,9 +880,10 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { int32_t lino = 0; SFdCbArg *pArg = arg; SSvrConn *pConn = pArg->data; + STrans *pInst = pConn->pInst; if (bytes <= 0) { - tDebug("client %s closed", pConn->src); + tDebug("%s conn %p closed", transLabel(pInst), pConn); return TSDB_CODE_RPC_NETWORK_ERROR; } @@ -929,7 +928,7 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } -static int32_t evtSvrHandleSendRespImpl(SSvrConn *pConn, SEvtBuf *pBuf) { +static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { int32_t code = 0; int32_t batchLimit = 32; int32_t j = 0; @@ -977,7 +976,7 @@ static int32_t evtSvrPreSend(void *arg, SEvtBuf *buf, int32_t status) { SFdCbArg *pArg = arg; SSvrConn *pConn = pArg->data; - code = evtSvrHandleSendRespImpl(pConn, buf); + code = evtSvrPreSendImpl(pConn, buf); return code; } @@ -999,6 +998,8 @@ void evtNewConnNotifyCb(void *async, int32_t status) { SAsyncHandle *handle = async; SEvtMgt *pEvtMgt = handle->data; + SWorkThrd2 *pThrd = handle->hostThrd; + STrans *pInst = pThrd->pInst; queue wq; QUEUE_INIT(&wq); @@ -1018,11 +1019,11 @@ void evtNewConnNotifyCb(void *async, int32_t status) { SSvrConn *pConn = createConn(pEvtMgt->hostThrd, pArg->acceptFd); if (pConn == NULL) { - tError("failed to create conn since %s", tstrerror(code)); + tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); taosMemoryFree(pArg); continue; } else { - tDebug("success to create conn %p, src:%s, dst:%s", pConn, pConn->src, pConn->dst); + tDebug("%s success to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); } SFdCbArg arg = {.evtType = EVT_CONN_T, @@ -1035,7 +1036,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); if (code != 0) { - tError("failed to add fd to evt since %s", tstrerror(code)); + tError("%s failed to add fd to evt since %s", pInst->label, tstrerror(code)); } taosMemoryFree(pArg); } @@ -1905,12 +1906,14 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { } static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { - int32_t code; - int32_t line = 0; - SFdCbArg *pArg = arg; - SCliConn *pConn = pArg->data; + int32_t code; + int32_t line = 0; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; if (bytes == 0) { - tDebug("client %s closed", pConn->src); + tDebug("%s client %p closed", pInst->label, pConn); return TSDB_CODE_RPC_NETWORK_ERROR; } SConnBuffer *p = &pConn->readBuf; @@ -1946,11 +1949,11 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { break; } } - + tDebug("%s success to read resp", pInst->label); return code; _end: if (code != 0) { - tError("%s failed to handle resp at line %d since %s", __func__, line, tstrerror(code)); + tError("%s %s failed to handle resp at line %d since %s", pInst->label, __func__, line, tstrerror(code)); } return code; } @@ -1988,6 +1991,7 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { SEvtMgt *pEvtMgt = handle->data; SCliThrd2 *pThrd = pEvtMgt->hostThrd; + STrans *pInst = pThrd->pInst; queue wq; QUEUE_INIT(&wq); @@ -2008,8 +2012,8 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { continue; } STraceId *trace = &pReq->msg.info.traceId; - tGDebug("handle request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pThrd->pid, pReq->ctx->epSet->eps[0].fqdn, - pReq->ctx->epSet->eps[0].port, pReq->msg.info.ahandle); + tGDebug("%s handle request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pInst->label, pThrd->pid, + pReq->ctx->epSet->eps[0].fqdn, pReq->ctx->epSet->eps[0].port, pReq->msg.info.ahandle); code = evtHandleCliReq(pThrd, pReq); } diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index a31ea941eabb..29dd2e0bfe0d 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -302,7 +302,7 @@ class TransEnv : public ::testing::Test { TEST_F(TransEnv, 01sendAndReq) { for (int i = 0; i < 1; i++) { SRpcMsg req = {0}, resp = {0}; - req.msgType = 0; + req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendReq(&req); From aa814ea053c64bd96788630b09e2d5b915e2171b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 21 Dec 2024 10:16:33 +0800 Subject: [PATCH 45/76] support select --- source/libs/transport/src/transImpl2.c | 147 +++++++++++-------------- 1 file changed, 63 insertions(+), 84 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 48203c76ab10..2032780e4f76 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -232,6 +232,18 @@ static void evtMgtDestroy(SEvtMgt *pOpt); int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd); +static int32_t evtInitPipe(int32_t dst[2]) { + int32_t code = 0; + int fd[2] = {0}; + code = pipe(fd); + if (code < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + dst[0] = fd[0]; + dst[1] = fd[1]; + return code; +} static int32_t evtMgtCreate(SEvtMgt **pOpt) { int32_t code = 0; SEvtMgt *pRes = taosMemoryCalloc(1, sizeof(SEvtMgt)); @@ -398,9 +410,6 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; - struct timeval ttv; - ttv.tv_sec = 30; // 设置为5秒 - ttv.tv_usec = 0; if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; int32_t sz = pOpt->evtFdsSize; @@ -425,7 +434,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { nfds = pOpt->evtFds + 1; // TODO lock or not - active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, &ttv); + active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); } else if (active_Fds == 0) { @@ -1102,12 +1111,13 @@ void *transWorkerThread(void *arg) { struct timeval tv = {5, 0}; setThreadName("trans-svr-work"); SWorkThrd2 *pThrd = (SWorkThrd2 *)arg; + STrans *pInst = pThrd->pInst; SEvtMgt *pOpt = NULL; code = evtMgtCreate(&pOpt); if (code != 0) { - tError("failed to create select op since %s", tstrerror(code)); + tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } @@ -1117,26 +1127,27 @@ void *transWorkerThread(void *arg) { code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_NEW_CONN_T, (void *)pThrd); if (code != 0) { - tError("failed to create select op since %s", tstrerror(code)); + tError("%s failed to create evt since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtSvrHandleAyncCb, EVT_ASYNC_T, (void *)pThrd); if (code != 0) { - tError("failed to create select op since %s", tstrerror(code)); + tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } while (!pThrd->quit) { + struct timeval tv = {30, 0}; code = evtMgtDispath(pOpt, &tv); if (code != 0) { - tError("failed to dispatch since %s", tstrerror(code)); + tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); continue; } } _end: if (code != 0) { - tError("failed to do work %s", tstrerror(code)); + tError("%s failed to do work %s", pInst->label, tstrerror(code)); } evtMgtDestroy(pOpt); return NULL; @@ -1170,14 +1181,15 @@ static int32_t addHandleToAcceptloop(void *arg) { return code; } -void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { +void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *arg) { int32_t code = 0; + int32_t lino = 0; + STrans *pInst = arg; SServerObj2 *srv = taosMemoryCalloc(1, sizeof(SServerObj2)); if (srv == NULL) { code = terrno; - tError("failed to init server since: %s", tstrerror(code)); - return NULL; + TAOS_CHECK_EXIT(code); } srv->ip = ip; @@ -1188,60 +1200,35 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads srv->pThreadObj = (SWorkThrd2 **)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd2 *)); if (srv->pThreadObj == NULL) { code = terrno; - return NULL; + TAOS_CHECK_EXIT(code); } for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrd2 *thrd = (SWorkThrd2 *)taosMemoryCalloc(1, sizeof(SWorkThrd2)); - thrd->pInst = pInit; - thrd->quit = false; - thrd->pInst = pInit; + thrd->pInst = arg; thrd->connRefMgt = transOpenRefMgt(50000, transDestroyExHandle); if (thrd->connRefMgt < 0) { code = thrd->connRefMgt; - goto End; + TAOS_CHECK_EXIT(code); } - { - int fd[2] = {0}; - code = pipe(fd); - if (code < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto End; - } - thrd->pipe_fd[0] = fd[0]; - thrd->pipe_fd[1] = fd[1]; - } - - { - int fd2[2] = {0}; - code = pipe(fd2); - if (code < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto End; - } - thrd->pipe_queue_fd[0] = fd2[0]; - thrd->pipe_queue_fd[1] = fd2[1]; - } + TAOS_CHECK_EXIT(evtInitPipe(thrd->pipe_fd)); + TAOS_CHECK_EXIT(evtInitPipe(thrd->pipe_queue_fd)); QUEUE_INIT(&thrd->conn); - int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); - if (err == 0) { - tDebug("success to create worker-thread:%d", i); - } else { - // TODO: clear all other resource later - tError("failed to create worker-thread:%d", i); - goto End; - } + code = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); + TAOS_CHECK_EXIT(code); thrd->inited = 1; + thrd->quit = false; srv->pThreadObj[i] = thrd; } code = addHandleToAcceptloop(srv); + TAOS_CHECK_EXIT(code); + return NULL; +_exit: if (code != 0) { - goto End; + tError("%s failed to init server at line %d since %s", pInst->label, lino, tstrerror(code)); } return NULL; -End: - return NULL; } // int32_t transReleaseSrvHandle(void *handle) { return 0; } @@ -2020,11 +2007,11 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { } static void *cliWorkThread2(void *arg) { - int32_t line = 0; - int32_t code = 0; - char threadName[TSDB_LABEL_LEN] = {0}; - struct timeval tv = {30, 0}; - SCliThrd2 *pThrd = (SCliThrd2 *)arg; + int32_t line = 0; + int32_t code = 0; + char threadName[TSDB_LABEL_LEN] = {0}; + SCliThrd2 *pThrd = (SCliThrd2 *)arg; + STrans *pInst = pThrd->pInst; pThrd->pid = taosGetSelfPthreadId(); @@ -2044,68 +2031,60 @@ static void *cliWorkThread2(void *arg) { TAOS_CHECK_GOTO(code, &line, _end); while (!pThrd->quit) { + struct timeval tv = {30, 0}; code = evtMgtDispath(pThrd->pEvtMgt, &tv); if (code != 0) { - tError("failed to dispatch since %s", tstrerror(code)); + tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); continue; } else { - tDebug("success to dispatch"); + tDebug("%s success to dispatch", pInst->label); } } - tDebug("thread quit-thread:%08" PRId64 "", pThrd->pid); + tDebug("%s thread quit-thread:%08" PRId64 "", pInst->label, pThrd->pid); return NULL; _end: if (code != 0) { - tError("failed to do work %s", tstrerror(code)); + tError("%s failed to do work %s", pInst->label, tstrerror(code)); } return NULL; } -void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInstRef) { +void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *arg) { int32_t code = 0; + int32_t lino = 0; SCliObj2 *cli = taosMemoryCalloc(1, sizeof(SCliObj2)); if (cli == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _err); + TAOS_CHECK_EXIT(terrno); } - STrans *pInst = pInstRef; + STrans *pInst = arg; memcpy(cli->label, label, TSDB_LABEL_LEN); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrd2 **)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd2 *)); if (cli->pThreadObj == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _err); + TAOS_CHECK_EXIT(terrno); } + for (int i = 0; i < cli->numOfThreads; i++) { SCliThrd2 *pThrd = NULL; - code = createThrdObj(pInstRef, &pThrd); - if (code != 0) { - goto _err; - } - int fd[2] = {0}; - - code = pipe(fd); - if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_CHECK_GOTO(code, NULL, _err); - } - pThrd->pipe_queue_fd[0] = fd[0]; - pThrd->pipe_queue_fd[1] = fd[1]; + code = createThrdObj(arg, &pThrd); + TAOS_CHECK_EXIT(code); pThrd->pInst = pInst; - int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); - if (err != 0) { - destroyThrdObj(pThrd); - code = TAOS_SYSTEM_ERROR(errno); - TAOS_CHECK_GOTO(code, NULL, _err); - } else { - tDebug("success to create tranport-cli thread:%d", i); - } + code = evtInitPipe(pThrd->pipe_queue_fd); + TAOS_CHECK_EXIT(code); + + code = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); + TAOS_CHECK_EXIT(code); pThrd->thrdInited = 1; cli->pThreadObj[i] = pThrd; } return cli; -_err: +_exit: + if (code != 0) { + tError("%s failed to init client since %s", pInst->label, tstrerror(code)); + } if (cli) { for (int i = 0; i < cli->numOfThreads; i++) { // send quit msg From 0f94ca26c3b0a1236b44473ba6cb3b7c366d0123 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 21 Dec 2024 10:20:03 +0800 Subject: [PATCH 46/76] support select --- source/libs/transport/test/transImpl2_test.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 29dd2e0bfe0d..9b66e18433e8 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -25,6 +25,8 @@ using namespace std; const char *label = "APP"; +const char *svrLabel = "SERVER"; +const char *cliLabel = "CLIENT"; const char *secret = "secret"; const char *user = "user"; const char *ckey = "ckey"; @@ -48,7 +50,7 @@ class Client { memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); memset(&rpcInit_, 0, sizeof(rpcInit_)); rpcInit_.localPort = 0; - rpcInit_.label = (char *)label; + rpcInit_.label = (char *)cliLabel; rpcInit_.numOfThreads = 1; rpcInit_.cfp = processResp; rpcInit_.user = (char *)user; @@ -123,7 +125,8 @@ class Server { memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); rpcInit_.localPort = port; - rpcInit_.label = (char *)label; + + rpcInit_.label = (char *)svrLabel; rpcInit_.numOfThreads = 1; rpcInit_.cfp = processReq; rpcInit_.user = (char *)user; From 9b367a1454466522392ee3d96f21224db543a3c4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 22 Dec 2024 13:26:01 +0800 Subject: [PATCH 47/76] support select --- source/libs/transport/src/transComm.c | 180 ++++++++++++------------- source/libs/transport/src/transImpl2.c | 153 ++++++++++++++++++++- 2 files changed, 240 insertions(+), 93 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 84757d6f57df..0c0aeeaff6e5 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -453,95 +453,6 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { return ret; } -int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { - QUEUE_INIT(&wq->node); - wq->freeFunc = (void (*)(void*))freeFunc; - wq->size = 0; - wq->inited = 1; - return 0; -} -void transQueuePush(STransQueue* q, void* arg) { - queue* node = arg; - QUEUE_PUSH(&q->node, node); - q->size++; -} -void* transQueuePop(STransQueue* q) { - if (q->size == 0) return NULL; - - queue* head = QUEUE_HEAD(&q->node); - QUEUE_REMOVE(head); - q->size--; - return head; -} -int32_t transQueueSize(STransQueue* q) { return q->size; } - -void* transQueueGet(STransQueue* q, int idx) { - if (q->size == 0) return NULL; - - while (idx-- > 0) { - queue* node = QUEUE_NEXT(&q->node); - if (node == &q->node) return NULL; - } - return NULL; -} - -void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { - queue* d = dst; - queue* node = QUEUE_NEXT(&q->node); - while (node != &q->node) { - queue* next = QUEUE_NEXT(node); - if (filter && filter(node, arg)) { - QUEUE_REMOVE(node); - q->size--; - QUEUE_PUSH(d, node); - if (--size == 0) { - break; - } - } - node = next; - } -} - -void* tranQueueHead(STransQueue* q) { - if (q->size == 0) return NULL; - - queue* head = QUEUE_HEAD(&q->node); - return head; -} - -void* transQueueRm(STransQueue* q, int i) { - // if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { - // return NULL; - // } - // if (i >= taosArrayGetSize(queue->q)) { - // return NULL; - // } - // void* ptr = taosArrayGetP(queue->q, i); - // taosArrayRemove(queue->q, i); - // return ptr; - return NULL; -} - -void transQueueRemove(STransQueue* q, void* e) { - if (q->size == 0) return; - queue* node = e; - QUEUE_REMOVE(node); - q->size--; -} - -bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; } - -void transQueueClear(STransQueue* q) { - if (q->inited == 0) return; - while (!QUEUE_IS_EMPTY(&q->node)) { - queue* h = QUEUE_HEAD(&q->node); - QUEUE_REMOVE(h); - if (q->freeFunc != NULL) (q->freeFunc)(h); - q->size--; - } -} -void transQueueDestroy(STransQueue* q) { transQueueClear(q); } - static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { SDelayTask* arg1 = container_of(a, SDelayTask, node); SDelayTask* arg2 = container_of(b, SDelayTask, node); @@ -2026,4 +1937,93 @@ int32_t transValidReqEpset(SReqEpSet* pReqEpSet) { return TSDB_CODE_INVALID_PARA; } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} + +int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { + QUEUE_INIT(&wq->node); + wq->freeFunc = (void (*)(void*))freeFunc; + wq->size = 0; + wq->inited = 1; + return 0; +} +void transQueuePush(STransQueue* q, void* arg) { + queue* node = arg; + QUEUE_PUSH(&q->node, node); + q->size++; +} +void* transQueuePop(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(head); + q->size--; + return head; +} +int32_t transQueueSize(STransQueue* q) { return q->size; } + +void* transQueueGet(STransQueue* q, int idx) { + if (q->size == 0) return NULL; + + while (idx-- > 0) { + queue* node = QUEUE_NEXT(&q->node); + if (node == &q->node) return NULL; + } + return NULL; +} + +void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { + queue* d = dst; + queue* node = QUEUE_NEXT(&q->node); + while (node != &q->node) { + queue* next = QUEUE_NEXT(node); + if (filter && filter(node, arg)) { + QUEUE_REMOVE(node); + q->size--; + QUEUE_PUSH(d, node); + if (--size == 0) { + break; + } + } + node = next; + } +} + +void* tranQueueHead(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + return head; +} + +void* transQueueRm(STransQueue* q, int i) { + // if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + // return NULL; + // } + // if (i >= taosArrayGetSize(queue->q)) { + // return NULL; + // } + // void* ptr = taosArrayGetP(queue->q, i); + // taosArrayRemove(queue->q, i); + // return ptr; + return NULL; +} + +void transQueueRemove(STransQueue* q, void* e) { + if (q->size == 0) return; + queue* node = e; + QUEUE_REMOVE(node); + q->size--; +} + +bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; } + +void transQueueClear(STransQueue* q) { + if (q->inited == 0) return; + while (!QUEUE_IS_EMPTY(&q->node)) { + queue* h = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(h); + if (q->freeFunc != NULL) (q->freeFunc)(h); + q->size--; + } +} +void transQueueDestroy(STransQueue* q) { transQueueClear(q); } \ No newline at end of file diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 2032780e4f76..bd1228020de1 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -12,9 +12,9 @@ * along with this program. If not, see . */ +#include "theap.h" #include "transComm.h" #include "tversion.h" - #ifdef TD_ACORE #include #include @@ -1108,7 +1108,6 @@ void *transWorkerThread(void *arg) { int32_t code = 0; int32_t line = 0; - struct timeval tv = {5, 0}; setThreadName("trans-svr-work"); SWorkThrd2 *pThrd = (SWorkThrd2 *)arg; STrans *pInst = pThrd->pInst; @@ -1324,7 +1323,7 @@ typedef struct SCliConn { // SDelayTask *task; - // HeapNode node; // for heap + HeapNode node; // for heap int8_t inHeap; int32_t reqRefCnt; uint32_t clientIp; @@ -1944,6 +1943,30 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } +#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) +typedef struct { + void *p; + HeapNode node; +} SHeapNode; +typedef struct { + // void* p; + Heap *heap; + int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b); + int64_t lastUpdateTs; + int64_t lastConnFailTs; +} SHeap; + +static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); +static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); +static void transHeapDestroy(SHeap *heap); + +static int32_t transHeapGet(SHeap *heap, SCliConn **p); +static int32_t transHeapInsert(SHeap *heap, SCliConn *p); +static int32_t transHeapDelete(SHeap *heap, SCliConn *p); +static int32_t transHeapBalance(SHeap *heap, SCliConn *p); +static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); +static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); + static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { int32_t code = 0; char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); @@ -2100,3 +2123,127 @@ void transCloseClient2(void *arg) { int32_t code = 0; return; } + +static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b) { + SCliConn *args1 = container_of(a, SCliConn, node); + SCliConn *args2 = container_of(b, SCliConn, node); + + int32_t totalReq1 = REQS_ON_CONN(args1); + int32_t totalReq2 = REQS_ON_CONN(args2); + if (totalReq1 > totalReq2) { + return 0; + } + return 1; +} +static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)) { + heap->heap = heapCreate(cmpFunc); + if (heap->heap == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + heap->cmpFunc = cmpFunc; + return 0; +} +static void transHeapDestroy(SHeap *heap) { + if (heap != NULL) { + heapDestroy(heap->heap); + } +} + +static int32_t transHeapGet(SHeap *heap, SCliConn **p) { + if (heapSize(heap->heap) == 0) { + *p = NULL; + return -1; + } + HeapNode *minNode = heapMin(heap->heap); + if (minNode == NULL) { + *p = NULL; + return -1; + } + *p = container_of(minNode, SCliConn, node); + return 0; +} +static int32_t transHeapInsert(SHeap *heap, SCliConn *p) { + int32_t code = 0; + p->reqRefCnt++; + if (p->inHeap == 1) { + tTrace("failed to insert conn %p since already in heap", p); + return TSDB_CODE_DUP_KEY; + } + + heapInsert(heap->heap, &p->node); + p->inHeap = 1; + p->lastAddHeapTime = taosGetTimestampMs(); + p->heap = heap; + return 0; +} +static int32_t transHeapDelete(SHeap *heap, SCliConn *p) { + if (p->connnected == 0) { + TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); + } + + if (p->inHeap == 0) { + tTrace("failed to del conn %p since not in heap", p); + return 0; + } else { + int64_t now = taosGetTimestampMs(); + if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { + tTrace("conn %p not added/delete to heap frequently", p); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + } + + p->inHeap = 0; + p->reqRefCnt--; + if (p->reqRefCnt == 0) { + heapRemove(heap->heap, &p->node); + tTrace("conn %p delete from heap", p); + } else if (p->reqRefCnt < 0) { + tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); + } else { + tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + } + return 0; +} +static int32_t transHeapBalance(SHeap *heap, SCliConn *p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + heapRemove(heap->heap, &p->node); + heapInsert(heap->heap, &p->node); + return 0; +} +static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p) { + heap->lastConnFailTs = taosGetTimestampMs(); + return 0; +} +static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + SCliThrd2 *pThrd = p->hostThrd; + STrans *pInst = pThrd->pInst; + int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; + + SCliConn *topConn = NULL; + int32_t code = transHeapGet(heap, &topConn); + if (code != 0) { + return code; + } + + if (topConn == p) return code; + + int32_t reqsOnTop = REQS_ON_CONN(topConn); + int32_t reqsOnCur = REQS_ON_CONN(p); + + if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { + TAOS_UNUSED(transHeapBalance(heap, p)); + } + return code; +} +static int32_t cliGetConnOrCreateConn(SCliThrd2 *pThrd, SCliConn **ppConn) { + int32_t code = 0; + SCliConn *pConn = NULL; + + return code; +} \ No newline at end of file From a273f57d636bc74d53e39e4a1ad3d4d100fd991e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 11:42:37 +0800 Subject: [PATCH 48/76] support select --- source/libs/transport/src/transImpl2.c | 401 ++++++++++++++++++++++--- 1 file changed, 363 insertions(+), 38 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index bd1228020de1..e1fd85b2a72f 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -770,7 +770,7 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); } QUEUE_PUSH(&pThrd->conn, &pConn->queue); - QUEUE_INIT(&pConn->resps.node); + transQueueInit(&pConn->resps, NULL); // code = initWQ(&pConn->wq); // TAOS_CHECK_GOTO(code, &lino, _end); // wqInited = 1; @@ -941,8 +941,8 @@ static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { int32_t code = 0; int32_t batchLimit = 32; int32_t j = 0; - while (!QUEUE_IS_EMPTY(&pConn->resps.node)) { - queue *el = QUEUE_HEAD(&pConn->resps.node); + while (!transQueueEmpty(&pConn->resps)) { + queue *el = transQueuePop(&pConn->resps); QUEUE_REMOVE(el); SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); @@ -996,7 +996,7 @@ static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { SWorkThrd2 *pThrd = pConn->hostThrd; - if (QUEUE_IS_EMPTY(&pConn->resps.node)) { + if (transQueueEmpty(&pConn->resps)) { // stop write evt code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } @@ -1056,8 +1056,8 @@ void evtNewConnNotifyCb(void *async, int32_t status) { static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { int32_t code = 0; SSvrConn *pConn = pResp->pConn; - QUEUE_PUSH(&pConn->resps.node, &pResp->q); - if (QUEUE_IS_EMPTY(&pConn->resps.node)) { + transQueuePush(&pConn->resps, &pResp->q); + if (transQueueEmpty(&pConn->resps)) { return code; } code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); @@ -1299,6 +1299,13 @@ void transCloseServer2(void *arg) { return; } // impl client with poll + +typedef struct SConnList { + queue conns; + int32_t size; + int32_t totalSize; +} SConnList; + typedef struct SCliConn { int32_t ref; // uv_connect_t connReq; @@ -1312,8 +1319,8 @@ typedef struct SCliConn { STransQueue reqsToSend; STransQueue reqsSentOut; - queue q; - // SConnList *list; + queue q; + SConnList *list; STransCtx ctx; bool broken; // link broken or not @@ -1444,6 +1451,35 @@ typedef struct SCliObj2 { int numOfThreads; SCliThrd2 **pThreadObj; } SCliObj2; +#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) +typedef struct { + void *p; + HeapNode node; +} SHeapNode; +typedef struct { + // void* p; + Heap *heap; + int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b); + int64_t lastUpdateTs; + int64_t lastConnFailTs; +} SHeap; + +static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); +static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); +static void transHeapDestroy(SHeap *heap); + +static int32_t transHeapGet(SHeap *heap, SCliConn **p); +static int32_t transHeapInsert(SHeap *heap, SCliConn *p); +static int32_t transHeapDelete(SHeap *heap, SCliConn *p); +static int32_t transHeapBalance(SHeap *heap, SCliConn *p); +static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); +static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); + +static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn); +static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn); +static int32_t getOrCreateHeap(SHashObj *pConnHeapCache, char *key, SHeap **pHeap); + +static SCliConn *getConnFromHeap(SHashObj *pConnHeap, char *key); static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { int32_t code = 0; @@ -1502,16 +1538,123 @@ static int32_t cliConnGetSockInfo(SCliConn *pConn) { } return code; } -static int32_t createCliConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { + +static int32_t getOrCreateConnList(SCliThrd2 *pThrd, const char *key, SConnList **ppList) { + int32_t code = 0; + void *pool = pThrd->pool; + size_t klen = strlen(key); + SConnList *plist = taosHashGet((SHashObj *)pool, key, klen); + if (plist == NULL) { + SConnList list = {0}; + QUEUE_INIT(&list.conns); + code = taosHashPut((SHashObj *)pool, key, klen, (void *)&list, sizeof(list)); + if (code != 0) { + return code; + } + + plist = taosHashGet(pool, key, klen); + if (plist == NULL) { + return TSDB_CODE_INVALID_PTR; + } + QUEUE_INIT(&plist->conns); + *ppList = plist; + tDebug("create conn list %p for key %s", plist, key); + } else { + *ppList = plist; + } + return 0; +} +static void addConnToPool(void *pool, SCliConn *conn) { + if (conn->status == ConnInPool) { + return; + } + + SCliThrd2 *thrd = conn->hostThrd; + if (thrd->quit == true) { + return; + } + + // cliResetConnTimer(conn); + if (conn->list == NULL && conn->dstAddr != NULL) { + conn->list = taosHashGet((SHashObj *)pool, conn->dstAddr, strlen(conn->dstAddr)); + } + + conn->status = ConnInPool; + QUEUE_INIT(&conn->q); + QUEUE_PUSH(&conn->list->conns, &conn->q); + conn->list->size += 1; + tDebug("conn %p added to pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr); + + conn->heapMissHit = 0; + + if (conn->list->size >= 5) { + STaskArg *arg = taosMemoryCalloc(1, sizeof(STaskArg)); + if (arg == NULL) return; + arg->param1 = conn; + arg->param2 = thrd; + + STrans *pInst = thrd->pInst; + // conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, (10 * CONN_PERSIST_TIME(pInst->idleTime) / + // 3)); + } +} + +static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppConn) { + int32_t code = 0; + void *pool = pThrd->pool; + STrans *pInst = pThrd->pInst; + + SConnList *plist = NULL; + code = getOrCreateConnList(pThrd, key, &plist); + if (code != 0) { + return code; + } + + if (QUEUE_IS_EMPTY(&plist->conns)) { + if (plist->totalSize >= pInst->connLimitNum) { + return TSDB_CODE_RPC_MAX_SESSIONS; + } + return TSDB_CODE_RPC_NETWORK_BUSY; + } + + queue *h = QUEUE_HEAD(&plist->conns); + plist->size -= 1; + QUEUE_REMOVE(h); + + SCliConn *conn = QUEUE_DATA(h, SCliConn, q); + conn->status = ConnNormal; + QUEUE_INIT(&conn->q); + conn->list = plist; + + tDebug("conn %p get from pool, pool size:%d, dst:%s", conn, conn->list->size, conn->dstAddr); + + *ppConn = conn; + return 0; +} +static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { int32_t code = 0; int32_t line = 0; STrans *pInst = pThrd->pInst; - SCliConn *pConn = taosMemoryCalloc(1, sizeof(SCliConn)); + SCliConn *pConn = NULL; + + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", ip, port); + pConn = getConnFromHeap(pThrd->connHeapCache, addr); + if (pConn != NULL) { + *ppConn = pConn; + return code; + } + + code = getConnFromPool(pThrd, addr, &pConn); + if (pConn != NULL) { + *ppConn = pConn; + return 0; + } + + pConn = taosMemoryCalloc(1, sizeof(SCliConn)); if (pConn == NULL) { TAOS_CHECK_GOTO(terrno, &line, _end); } - char addr[TSDB_FQDN_LEN + 64] = {0}; - snprintf(addr, sizeof(addr), "%s:%d", ip, port); pConn->hostThrd = pThrd; pConn->dstAddr = taosStrdup(addr); pConn->ipStr = taosStrdup(ip); @@ -1534,8 +1677,8 @@ static int32_t createCliConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn pConn->bufSize = pInst->shareConnLimit; QUEUE_INIT(&pConn->reqsToSend2); - QUEUE_INIT(&pConn->reqsToSend.node); - QUEUE_INIT(&pConn->reqsSentOut.node); + transQueueInit(&pConn->reqsToSend, NULL); + transQueueInit(&pConn->reqsSentOut, NULL); TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); @@ -1746,6 +1889,43 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, return code; } +static int32_t evtCliRecycleConn(SCliConn *pConn) { + int32_t code = 0; + + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + tTrace("%s conn %p in-process req summary:reqsToSend:%d, reqsSentOut:%d, statusTableSize:%d", pInst->label, pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable)); + + if (transQueueSize(&pConn->reqsToSend) == 0 && transQueueSize(&pConn->reqsSentOut) == 0 && + taosHashGetSize(pConn->pQTable) == 0) { + // cliResetConnTimer(conn); + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + tDebug("%s conn %p failed to remove conn from heap cache since %s", pInst->label, pConn, tstrerror(code)); + + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + return 1; + } else { + if (code != 0) { + tDebug("%s conn %p failed to remove conn from heap cache since %s", pInst->label, pConn, tstrerror(code)); + return 0; + } + } + addConnToPool(pThrd->pool, pConn); + return 1; + } else if ((transQueueSize(&pConn->reqsToSend) == 0) && (transQueueSize(&pConn->reqsSentOut) == 0) && + (taosHashGetSize(pConn->pQTable) != 0)) { + tDebug("%s conn %p do balance directly", pInst->label, pConn); + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + } else { + // tTrace("%s conn %p may do balance", CONN_GET_INST_LABEL(conn), conn); + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + } + + return code; +} static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { int32_t code = 0; @@ -1850,7 +2030,7 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { pCliMsg->seq = pConn->seq; pCliMsg->sent = 1; - QUEUE_PUSH(&pConn->reqsSentOut.node, &pCliMsg->q); + transQueuePush(&pConn->reqsSentOut, &pCliMsg->q); QUEUE_INIT(&pCliMsg->sendQ); QUEUE_PUSH(&reqToSend, &pCliMsg->sendQ); @@ -1943,38 +2123,183 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { } return code; } -#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) -typedef struct { - void *p; - HeapNode node; -} SHeapNode; -typedef struct { - // void* p; - Heap *heap; - int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b); - int64_t lastUpdateTs; - int64_t lastConnFailTs; -} SHeap; +// #define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) -static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); -static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); -static void transHeapDestroy(SHeap *heap); +// static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); +// static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); +// static void transHeapDestroy(SHeap *heap); -static int32_t transHeapGet(SHeap *heap, SCliConn **p); -static int32_t transHeapInsert(SHeap *heap, SCliConn *p); -static int32_t transHeapDelete(SHeap *heap, SCliConn *p); -static int32_t transHeapBalance(SHeap *heap, SCliConn *p); -static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); -static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); +// static int32_t transHeapGet(SHeap *heap, SCliConn **p); +// static int32_t transHeapInsert(SHeap *heap, SCliConn *p); +// static int32_t transHeapDelete(SHeap *heap, SCliConn *p); +// static int32_t transHeapBalance(SHeap *heap, SCliConn *p); +// static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); +// static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); +// static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn); +static int32_t getOrCreateHeap(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { + int32_t code = 0; + size_t klen = strlen(key); + + SHeap *p = taosHashGet(pConnHeapCache, key, klen); + if (p == NULL) { + SHeap heap = {0}; + code = transHeapInit(&heap, compareHeapNode); + if (code != 0) { + tError("failed to init heap cache for key:%s since %s", key, tstrerror(code)); + return code; + } + + code = taosHashPut(pConnHeapCache, key, klen, &heap, sizeof(heap)); + if (code != 0) { + transHeapDestroy(&heap); + tError("failed to put heap to cache for key:%s since %s", key, tstrerror(code)); + return code; + } + p = taosHashGet(pConnHeapCache, key, klen); + if (p == NULL) { + code = TSDB_CODE_INVALID_PARA; + } + } + *pHeap = p; + return code; +} +static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + tDebug("get conn %p from heap cache for key:%s, status:%d, refCnt:%d", pConn, key, pConn->inHeap, pConn->reqRefCnt); + int32_t reqsNum = transQueueSize(&pConn->reqsToSend); + int32_t reqsSentOut = transQueueSize(&pConn->reqsSentOut); + int32_t stateNum = taosHashGetSize(pConn->pQTable); + int32_t totalReqs = reqsNum + reqsSentOut; + + if (totalReqs >= pInst->shareConnLimit) { + // logConnMissHit(pConn); + + if (pConn->list == NULL && pConn->dstAddr != NULL) { + pConn->list = taosHashGet((SHashObj *)pThrd->pool, pConn->dstAddr, strlen(pConn->dstAddr)); + if (pConn->list != NULL) { + tTrace("conn %p get list %p from pool for key:%s", pConn, pConn->list, key); + } + } + if (pConn->list && pConn->list->totalSize >= pInst->connLimitNum / 4) { + tWarn("%s conn %p try to remove timeout msg since too many conn created", transLabel(pInst), pConn); + + // TODO + // if (cliConnRemoveTimeoutMsg(pConn)) { + // tWarn("%s conn %p succ to remove timeout msg", transLabel(pInst), pConn); + // } + return 1; + } + // check req timeout or not + return 1; + } + + return 0; +} +static SCliConn *getConnFromHeap(SHashObj *pConnHeap, char *key) { + int32_t code = 0; + SCliConn *pConn = NULL; + SHeap *pHeap = NULL; + + code = getOrCreateHeap(pConnHeap, key, &pHeap); + if (code != 0) { + tTrace("failed to get conn heap from cache for key:%s", key); + return NULL; + } + + code = transHeapGet(pHeap, &pConn); + if (code != 0) { + tTrace("failed to get conn from heap cache for key:%s", key); + return NULL; + } else { + tTrace("conn %p get conn from heap cache for key:%s", pConn, key); + if (shouldSWitchToOtherConn(pConn, key)) { + SCliConn *pNewConn = NULL; + code = balanceConnHeapCache(pConnHeap, pConn, &pNewConn); + if (code == 1) { + tTrace("conn %p start to handle reqs", pNewConn); + return pNewConn; + } + return NULL; + } + } + + return pConn; +} +static int32_t addConnToHeapCache(SHashObj *pConnHeapCacahe, SCliConn *pConn) { + SHeap *p = NULL; + int32_t code = 0; + + if (pConn->heap != NULL) { + p = pConn->heap; + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, + pConn->inHeap, pConn->reqRefCnt); + } else { + code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + if (code != 0) { + return code; + } + if (pConn->connnected == 0) { + int64_t now = taosGetTimestampMs(); + if (now - p->lastConnFailTs < 3000) { + return TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + } + } + + code = transHeapInsert(p, pConn); + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, + pConn->reqRefCnt); + return code; +} + +static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn) { + if (pConn->heap != NULL) { + tTrace("conn %p try to delete from heap cache direct", pConn); + return transHeapDelete(pConn->heap, pConn); + } + + SHeap *p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); + if (p == NULL) { + tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); + return 0; + } + int32_t code = transHeapDelete(p, pConn); + if (code != 0) { + tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); + } + return code; +} + +static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + SCliConn *pTopConn = NULL; + if (pConn->heap != NULL && pConn->inHeap != 0) { + TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); + if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { + int32_t curReqs = REQS_ON_CONN(pConn); + int32_t topReqs = REQS_ON_CONN(pTopConn); + if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { + *pNewConn = pTopConn; + return 1; + } + } + } + return 0; +} static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { int32_t code = 0; + STrans *pInst = pThrd->pInst; char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); SCliConn *pConn = NULL; - code = createCliConn(pThrd, fqdn, port, &pConn); - STrans *pInst = pThrd->pInst; + code = getOrCreateConn(pThrd, fqdn, port, &pConn); if (code != 0) { tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); return code; From 3f93cd6e41ae5d29ca86595280b50fb8aede48c0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 16:20:40 +0800 Subject: [PATCH 49/76] support select --- include/libs/transport/trpc.h | 1 + source/libs/transport/src/transComm.c | 94 +++--- source/libs/transport/src/transImpl2.c | 447 +++++++++++++++++++++++-- 3 files changed, 476 insertions(+), 66 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 956f42f675fc..863c10e4638b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -339,6 +339,7 @@ typedef struct { SHashObj *args; SRpcBrokenlinkVal brokenVal; void (*freeFunc)(const void *arg); + int64_t st; } SRpcCtx; int32_t rpcInit(); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 0c0aeeaff6e5..c74c69428b73 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -597,52 +597,6 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, return task; } -void transPrintEpSet(SEpSet* pEpSet) { - if (pEpSet == NULL) { - tTrace("NULL epset"); - return; - } - char buf[512] = {0}; - int len = tsnprintf(buf, sizeof(buf), "epset:{"); - for (int i = 0; i < pEpSet->numOfEps; i++) { - if (i == pEpSet->numOfEps - 1) { - len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } else { - len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - } - len += tsnprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse:%d", buf, pEpSet->inUse); -} -bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) { - if (a == NULL && b == NULL) { - return true; - } else if (a == NULL || b == NULL) { - return false; - } - - if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} -bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { - if (a->numOfEps != b->numOfEps) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} - static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); @@ -2026,4 +1980,50 @@ void transQueueClear(STransQueue* q) { q->size--; } } -void transQueueDestroy(STransQueue* q) { transQueueClear(q); } \ No newline at end of file +void transQueueDestroy(STransQueue* q) { transQueueClear(q); } + +void transPrintEpSet(SEpSet* pEpSet) { + if (pEpSet == NULL) { + tTrace("NULL epset"); + return; + } + char buf[512] = {0}; + int len = tsnprintf(buf, sizeof(buf), "epset:{"); + for (int i = 0; i < pEpSet->numOfEps; i++) { + if (i == pEpSet->numOfEps - 1) { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } else { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + } + len += tsnprintf(buf + len, sizeof(buf) - len, "}"); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); +} +bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) { + if (a == NULL && b == NULL) { + return true; + } else if (a == NULL || b == NULL) { + return false; + } + + if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} +bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} \ No newline at end of file diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index e1fd85b2a72f..c49c358aae04 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -24,6 +24,7 @@ #include #include #include +#include "tmisce.h" #endif @@ -56,6 +57,11 @@ typedef struct { queue q; } SFdQueue; +typedef struct { + int notifyCount; // + int init; // init or not + STransMsg msg; +} SSvrRegArg; typedef struct SSvrConn { int32_t ref; // uv_tcp_t* pTcp; @@ -682,7 +688,7 @@ void *transAcceptThread(void *arg) { } return NULL; } -int32_t uvGetConnRefOfThrd(SWorkThrd2 *thrd) { return thrd ? thrd->connRefMgt : -1; } +int32_t evtSvrGetConnRefOfThrd(SWorkThrd2 *thrd) { return thrd ? thrd->connRefMgt : -1; } void uvDestroyResp(void *e) { SSvrRespMsg *pMsg = QUEUE_DATA(e, SSvrRespMsg, q); @@ -749,12 +755,12 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { exh->handle = pConn; exh->pThrd = pThrd; - exh->refId = transAddExHandle(uvGetConnRefOfThrd(pThrd), exh); + exh->refId = transAddExHandle(evtSvrGetConnRefOfThrd(pThrd), exh); if (exh->refId < 0) { TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, &lino, _end); } - SExHandle *pSelf = transAcquireExHandle(uvGetConnRefOfThrd(pThrd), exh->refId); + SExHandle *pSelf = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), exh->refId); if (pSelf != exh) { TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _end); } @@ -793,6 +799,7 @@ static void destroryConn(SSvrConn *pConn) { QUEUE_REMOVE(&pConn->queue); taosMemoryFree(pConn); } +static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp); bool connMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { if (pConn->userInited) { @@ -828,6 +835,56 @@ static int32_t evtConnHandleReleaseReq(SSvrConn *pConn, STransMsgHead *phead) { int32_t code = 0; return code; } + +static int32_t evtSvrHandleReleaseReq(SSvrConn *pConn, STransMsgHead *pHead) { + int32_t code = 0; + STrans *pInst = pConn->pInst; + if (pHead->msgType == TDMT_SCH_TASK_RELEASE) { + int64_t qId = taosHton64(pHead->qid); + if (qId <= 0) { + tError("conn %p recv release, but invalid sid:%" PRId64 "", pConn, qId); + code = TSDB_CODE_RPC_NO_STATE; + } else { + void *p = taosHashGet(pConn->pQTable, &qId, sizeof(qId)); + if (p == NULL) { + code = TSDB_CODE_RPC_NO_STATE; + tTrace("conn %p recv release, and releady release by server sid:%" PRId64 "", pConn, qId); + } else { + SSvrRegArg *arg = p; + (pInst->cfp)(pInst->parent, &(arg->msg), NULL); + tTrace("conn %p recv release, notify server app, sid:%" PRId64 "", pConn, qId); + + code = taosHashRemove(pConn->pQTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("conn %p failed to remove sid:%" PRId64 "", pConn, qId); + } + tTrace("conn %p clear state,sid:%" PRId64 "", pConn, qId); + } + } + + STransMsg tmsg = {.code = code, + .msgType = pHead->msgType + 1, + .info.qId = qId, + .info.traceId = pHead->traceId, + .info.seq = taosHton64(pHead->seqNum)}; + + SSvrRespMsg *srvMsg = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (srvMsg == NULL) { + tError("conn %p recv release, failed to send release-resp since %s", pConn, tstrerror(terrno)); + taosMemoryFree(pHead); + return terrno; + } + srvMsg->msg = tmsg; + srvMsg->type = Normal; + srvMsg->pConn = pConn; + + evtSvrHandleSendResp(pConn->hostThrd, srvMsg); + taosMemoryFree(pHead); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + + return 0; +} static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { SWorkThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; @@ -850,6 +907,9 @@ static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { pConn->inType = pHead->msgType; + if (evtSvrHandleReleaseReq(pConn, pHead) != 0) { + return 0; + } STransMsg transMsg = {0}; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; @@ -861,7 +921,7 @@ static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { return TSDB_CODE_INVALID_MSG; } - transMsg.info.handle = (void *)transAcquireExHandle(uvGetConnRefOfThrd(pThrd), pConn->refId); + transMsg.info.handle = (void *)transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); transMsg.info.refIdMgt = pThrd->connRefMgt; transMsg.info.refId = pHead->noResp == 1 ? -1 : pConn->refId; @@ -878,7 +938,7 @@ static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { pConnInfo->clientPort = pConn->port; tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); - transReleaseExHandle(uvGetConnRefOfThrd(pThrd), pConn->refId); + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); (*pInst->cfp)(pInst->parent, &transMsg, NULL); @@ -977,6 +1037,9 @@ static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { transLabel(pInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, len, pMsg->info.seq, pMsg->info.qId); evtBufPush(pBuf, (char *)pHead, len); + if (j++ > batchLimit) { + break; + } } return code; } @@ -1053,15 +1116,35 @@ void evtNewConnNotifyCb(void *async, int32_t status) { return; } +static int32_t evtSvrMayHandleReleaseResp(SSvrRespMsg *pMsg) { + int32_t code = 0; + SSvrConn *pConn = pMsg->pConn; + int64_t qid = pMsg->msg.info.qId; + if (pMsg->msg.msgType == TDMT_SCH_TASK_RELEASE && qid > 0) { + SSvrRegArg *p = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (p == NULL) { + tError("%s conn %p already release sid:%" PRId64 "", transLabel(pConn->pInst), pConn, qid); + return TSDB_CODE_RPC_NO_STATE; + } else { + transFreeMsg(p->msg.pCont); + code = taosHashRemove(pConn->pQTable, &qid, sizeof(qid)); + if (code != 0) { + tError("%s conn %p failed to release sid:%" PRId64 " since %s", transLabel(pConn->pInst), pConn, qid, + tstrerror(code)); + } + } + } + return 0; +} static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { int32_t code = 0; SSvrConn *pConn = pResp->pConn; - transQueuePush(&pConn->resps, &pResp->q); - if (transQueueEmpty(&pConn->resps)) { - return code; + if (evtSvrMayHandleReleaseResp(pResp) == TSDB_CODE_RPC_NO_STATE) { + destroySmsg(pResp); + return 0; } - code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); - return code; + transQueuePush(&pConn->resps, &pResp->q); + return evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } void evtSvrHandleAyncCb(void *async, int32_t status) { int32_t code = 0; @@ -1088,7 +1171,7 @@ void evtSvrHandleAyncCb(void *async, int32_t status) { STransMsg transMsg = pResp->msg; SExHandle *exh1 = transMsg.info.handle; int64_t refId = transMsg.info.refId; - SExHandle *exh2 = transAcquireExHandle(uvGetConnRefOfThrd(pThrd), refId); + SExHandle *exh2 = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); if (exh1 != exh2) { tError("failed to acquire handle since %s", tstrerror(TSDB_CODE_REF_INVALID_ID)); continue; @@ -1097,7 +1180,7 @@ void evtSvrHandleAyncCb(void *async, int32_t status) { pResp->seqNum = transMsg.info.seq; pResp->pConn = exh2->handle; - transReleaseExHandle(uvGetConnRefOfThrd(pThrd), refId); + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); code = evtSvrHandleSendResp(handle->hostThrd, pResp); } @@ -1365,7 +1448,6 @@ typedef struct SCliConn { queue batchSendq; int8_t inThreadSendq; int32_t fd; - queue reqsToSend2; } SCliConn; typedef struct { @@ -1675,7 +1757,6 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon } QUEUE_INIT(&pConn->batchSendq); pConn->bufSize = pInst->shareConnLimit; - QUEUE_INIT(&pConn->reqsToSend2); transQueueInit(&pConn->reqsToSend, NULL); transQueueInit(&pConn->reqsSentOut, NULL); @@ -1926,6 +2007,282 @@ static int32_t evtCliRecycleConn(SCliConn *pConn) { return code; } +static bool filteBySeq(void *key, void *arg) { + SFiterArg *targ = arg; + SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); + if (pReq->seq == targ->seq && pReq->msg.msgType + 1 == targ->msgType) { + removeReqFromSendQ(pReq); + return true; + } else { + return false; + } +} +static int32_t cliGetReqBySeq(SCliConn *pConn, int64_t seq, int32_t msgType, SCliReq **ppReq) { + int32_t code = 0; + queue set; + QUEUE_INIT(&set); + + SFiterArg arg = {.seq = seq, .msgType = msgType}; + transQueueRemoveByFilter(&pConn->reqsSentOut, filteBySeq, &arg, &set, 1); + while (QUEUE_IS_EMPTY(&set)) { + return TSDB_CODE_OUT_OF_RANGE; + } + + queue *e = QUEUE_HEAD(&set); + SCliReq *pReq = QUEUE_DATA(e, SCliReq, q); + + *ppReq = pReq; + return 0; +} + +#define CONN_GET_INST_LABEL(conn) (((STrans *)(((SCliThrd2 *)(conn)->hostThrd)->pInst))->label) +static bool cliIsEpsetUpdated(int32_t code, SReqCtx *pCtx) { + if (code != 0) return false; + return transReqEpsetIsEqual(pCtx->epSet, pCtx->origEpSet) ? false : true; +} +static int32_t cliNotifyImplCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + STraceId *trace = &pResp->info.traceId; + + if (pCtx == NULL) { + pInst->cfp(pInst->parent, pResp, NULL); + return 0; + } + if (pCtx->pSem || pCtx->syncMsgRef != 0) { + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); + if (pCtx->pSem) { + if (pCtx->pRsp == NULL) { + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); + } else { + memcpy((char *)pCtx->pRsp, (char *)pResp, sizeof(*pResp)); + } + TAOS_UNUSED(tsem_post(pCtx->pSem)); + pCtx->pRsp = NULL; + } else { + STransSyncMsg *pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + if (pSyncMsg != NULL) { + memcpy(pSyncMsg->pRsp, (char *)pResp, sizeof(*pResp)); + if (cliIsEpsetUpdated(pResp->code, pCtx)) { + pSyncMsg->hasEpSet = 1; + + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + epsetAssign(&pSyncMsg->epSet, &epset); + } + TAOS_UNUSED(tsem2_post(pSyncMsg->pSem)); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef)); + } else { + rpcFreeCont(pResp->pCont); + } + } + } else { + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + if (pResp->info.hasEpSet == 1) { + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + pInst->cfp(pInst->parent, pResp, &epset); + } else { + if (!cliIsEpsetUpdated(pResp->code, pCtx)) { + pInst->cfp(pInst->parent, pResp, NULL); + } else { + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + pInst->cfp(pInst->parent, pResp, &epset); + } + } + } + + return code; +} +static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + if (pReq != NULL) { + removeReqFromSendQ(pReq); + // if (pResp->code != TSDB_CODE_SUCCESS) { + // if (cliMayRetry(pConn, pReq, pResp)) { + // return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + // } + // cliMayResetRespCode(pReq, pResp); + // } + // if (cliTryUpdateEpset(pReq, pResp)) { + // cliPerfLog_epset(pConn, pReq); + // } + } + return cliNotifyImplCb(pConn, pReq, pResp); +} +static int32_t evtCliBuildRespFromCont(SCliReq *pReq, STransMsg *pResp, STransMsgHead *pHead) { + pResp->contLen = transContLenFromMsg(pHead->msgLen); + pResp->pCont = transContFromHead((char *)pHead); + pResp->code = pHead->code; + pResp->msgType = pHead->msgType; + if (pResp->info.ahandle == 0) { + pResp->info.ahandle = (pReq && pReq->ctx) ? pReq->ctx->ahandle : NULL; + } + pResp->info.traceId = pHead->traceId; + pResp->info.hasEpSet = pHead->hasEpSet; + pResp->info.cliVer = htonl(pHead->compatibilityVer); + pResp->info.seq = taosHton64(pHead->seqNum); + + int64_t qid = taosHton64(pHead->qid); + pResp->info.handle = (void *)qid; + return 0; +} + +static bool filterByQid(void *key, void *arg) { + int64_t *qid = arg; + SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); + + if (pReq->msg.info.qId == *qid) { + removeReqFromSendQ(pReq); + return true; + } else { + return false; + } +} + +static FORCE_INLINE void destroyReqWrapper(void *arg, void *param) { + if (arg == NULL) return; + + SCliReq *pReq = arg; + SCliThrd2 *pThrd = param; + + if (pReq->ctx != NULL && pReq->ctx->ahandle != NULL) { + if (pReq->msg.info.notFreeAhandle == 0 && pThrd != NULL && pThrd->destroyAhandleFp != NULL) { + (*pThrd->destroyAhandleFp)(pReq->ctx->ahandle); + } + } + destroyReq(pReq); +} +static FORCE_INLINE void destroyReqAndAhanlde(void *param) { + if (param == NULL) return; + + STaskArg *arg = param; + SCliReq *pReq = arg->param1; + SCliThrd2 *pThrd = arg->param2; + destroyReqWrapper(pReq, pThrd); +} + +int32_t cliHandleState_mayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + if (pHead->msgType == TDMT_SCH_TASK_RELEASE || pHead->msgType == TDMT_SCH_TASK_RELEASE + 1) { + int64_t qId = taosHton64(pHead->qid); + STraceId *trace = &pHead->traceId; + int64_t seqNum = taosHton64(pHead->seqNum); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, pHead->msgLen, seqNum, + qId); + + STransCtx *p = taosHashGet(conn->pQTable, &qId, sizeof(qId)); + transCtxCleanup(p); + + code = taosHashRemove(conn->pQTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("%s conn %p failed to release req:%" PRId64 " from conn", CONN_GET_INST_LABEL(conn), conn, qId); + } + + code = taosHashRemove(pThrd->pIdConnTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("%s conn %p failed to release req:%" PRId64 " from thrd ", CONN_GET_INST_LABEL(conn), conn, qId); + } + + tDebug("%s %p reqToSend:%d, sentOut:%d", CONN_GET_INST_LABEL(conn), conn, transQueueSize(&conn->reqsToSend), + transQueueSize(&conn->reqsSentOut)); + + queue set; + QUEUE_INIT(&set); + transQueueRemoveByFilter(&conn->reqsSentOut, filterByQid, &qId, &set, -1); + transQueueRemoveByFilter(&conn->reqsToSend, filterByQid, &qId, &set, -1); + + transReleaseExHandle(transGetRefMgt(), qId); + transRemoveExHandle(transGetRefMgt(), qId); + + while (!QUEUE_IS_EMPTY(&set)) { + queue *el = QUEUE_HEAD(&set); + QUEUE_REMOVE(el); + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + removeReqFromSendQ(pReq); + STraceId *trace = &pReq->msg.info.traceId; + tGDebug("start to free msg %p", pReq); + destroyReqWrapper(pReq, pThrd); + } + taosMemoryFree(pHead); + return 1; + } + return 0; +} +int32_t cliHandleState_mayCreateAhandle(SCliConn *conn, STransMsgHead *pHead, STransMsg *pResp) { + int32_t code = 0; + int64_t qId = taosHton64(pHead->qid); + if (qId == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + + STransCtx *pCtx = taosHashGet(conn->pQTable, &qId, sizeof(qId)); + if (pCtx == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + pCtx->st = taosGetTimestampUs(); + STraceId *trace = &pHead->traceId; + pResp->info.ahandle = transCtxDumpVal(pCtx, pHead->msgType); + tGDebug("%s conn %p %s received from %s, local info:%s, sid:%" PRId64 ", create ahandle %p by %s", + CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, qId, pResp->info.ahandle, + TMSG_INFO(pHead->msgType)); + return 0; +} + +static FORCE_INLINE void cliConnClearInitUserMsg(SCliConn *conn) { + if (conn->pInitUserReq) { + taosMemoryFree(conn->pInitUserReq); + conn->pInitUserReq = NULL; + } +} +int32_t cliHandleState_mayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { + int64_t qid = pReq->msg.info.qId; + if (qid > 0) { + STransCtx *pUserCtx = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (pUserCtx != NULL) { + pUserCtx->st = taosGetTimestampUs(); + } + } + return 0; +} + +int32_t cliHandleState_mayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + SReqCtx *pCtx = pReq->ctx; + SCliThrd2 *pThrd = pConn->hostThrd; + if (pCtx == NULL) { + tDebug("%s conn %p not need to update statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + return 0; + } + + STransCtx *pUserCtx = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (pUserCtx == NULL) { + pCtx->userCtx.st = taosGetTimestampUs(); + code = taosHashPut(pConn->pQTable, &qid, sizeof(qid), &pCtx->userCtx, sizeof(pCtx->userCtx)); + tDebug("%s conn %p succ to add statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } else { + transCtxMerge(pUserCtx, &pCtx->userCtx); + pUserCtx->st = taosGetTimestampUs(); + tDebug("%s conn %p succ to update statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } + return 0; +} + static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { int32_t code = 0; @@ -1941,10 +2298,62 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { int64_t seq = taosHton64(pHead->seqNum); STransMsg resp = {0}; - (pInst->cfp)(pInst->parent, &resp, NULL); + cliConnClearInitUserMsg(pConn); + + if (cliHandleState_mayHandleReleaseResp(pConn, pHead)) { + if (evtCliRecycleConn(pConn)) { + return code; + } + } + + code = cliGetReqBySeq(pConn, seq, pHead->msgType, &pReq); + if (code == TSDB_CODE_OUT_OF_RANGE) { + code = cliHandleState_mayCreateAhandle(pConn, pHead, &resp); + if (code == 0) { + code = evtCliBuildRespFromCont(NULL, &resp, pHead); + code = cliNotifyCb(pConn, NULL, &resp); + return 0; + } + + if (code != 0) { + tWarn("%s conn %p recv unexpected packet, msgType:%s, seqNum:%" PRId64 ", sid:%" PRId64 + ", the sever may sends repeated response since %s", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), seq, qId, tstrerror(code)); + // TODO: notify cb + taosMemoryFree(pHead); + if (evtCliRecycleConn(pConn)) { + return 0; + } + return 0; + } + } else { + code = cliHandleState_mayUpdateStateTime(pConn, pReq); + if (code != 0) { + tDebug("%s conn %p failed to update state time sid:%" PRId64 " since %s", CONN_GET_INST_LABEL(pConn), pConn, qId, + tstrerror(code)); + } + } + removeReqFromSendQ(pReq); + + code = evtCliBuildRespFromCont(pReq, &resp, pHead); + STraceId *trace = &resp.info.traceId; + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seq:%" PRId64 ", sid:%" PRId64 ", code:%s", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(resp.msgType), pConn->dst, pConn->src, pHead->msgLen, seq, qId, + tstrerror(pHead->code)); + code = cliNotifyCb(pConn, pReq, &resp); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + tGWarn("%s msg need retry", CONN_GET_INST_LABEL(pConn)); + } else { + destroyReq(pReq); + } + + if (evtCliRecycleConn(pConn)) { + return 0; + } return code; } + bool connMayAddUserInfo(SCliConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { SCliThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; @@ -1984,8 +2393,8 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { int32_t batchLimit = 16; queue reqToSend; QUEUE_INIT(&reqToSend); - while (!QUEUE_IS_EMPTY(&pConn->reqsToSend2)) { - queue *el = QUEUE_HEAD(&pConn->reqsToSend2); + while (!transQueueEmpty(&pConn->reqsToSend)) { + queue *el = transQueuePop(&pConn->reqsToSend); QUEUE_REMOVE(el); SCliReq *pCliMsg = QUEUE_DATA(el, SCliReq, q); @@ -2064,7 +2473,7 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { } else { tDebug("%s success to send out request", pInst->label); } - if (QUEUE_IS_EMPTY(&pConn->reqsToSend2)) { + if (transQueueEmpty(&pConn->reqsToSend)) { code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } @@ -2304,7 +2713,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); return code; } else { - QUEUE_PUSH(&pConn->reqsToSend2, &req->q); + transQueuePush(&pConn->reqsToSend, &req->q); STraceId *trace = &req->msg.info.traceId; tGDebug("%s success to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); From ce229f7614f735a6f77330b0e719f963d20ea7a7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 16:52:34 +0800 Subject: [PATCH 50/76] support select --- source/libs/transport/src/transImpl2.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index c49c358aae04..358235514337 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1808,10 +1808,10 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { if (pThrd == NULL) { TAOS_CHECK_GOTO(terrno, &line, _end); } - - taosThreadMutexInit(&pThrd->msgMtx, NULL); + pThrd->pool = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); QUEUE_INIT(&pThrd->msg); + taosThreadMutexInit(&pThrd->msgMtx, NULL); *ppThrd = pThrd; return code; @@ -2417,7 +2417,7 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { if (connMayAddUserInfo(pConn, &pHead, &msgLen)) { content = transContFromHead(pHead); contLen = transContLenFromMsg(msgLen); - pReq->pCont = (void *)pHead; + // pReq->pCont = (void *)pHead; } else { if (pConn->userInited == 0) { return terrno; From b88940b0d33ff13112b125af97136877cd654fe1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 18:19:39 +0800 Subject: [PATCH 51/76] support select --- source/libs/transport/src/transImpl2.c | 84 ++++++++++++------- .../libs/transport/test/transImpl2_test.cpp | 5 +- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 358235514337..23185d235126 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -451,10 +451,10 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { for (j = 0; j < nfds && active_Fds > 0; j++) { int32_t fd = pOpt->fd[j]; res = 0; - if (FD_ISSET(fd, pOpt->evtReadSetOut)) { + if ((fd > 0) && FD_ISSET(fd, pOpt->evtReadSetOut)) { res |= EVT_READ; } - if (FD_ISSET(fd, pOpt->evtWriteSetOut)) { + if ((fd > 0) && FD_ISSET(fd, pOpt->evtWriteSetOut)) { res |= EVT_WRITE; } @@ -1557,11 +1557,12 @@ static int32_t transHeapBalance(SHeap *heap, SCliConn *p); static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); +static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap); static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn); static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn); -static int32_t getOrCreateHeap(SHashObj *pConnHeapCache, char *key, SHeap **pHeap); -static SCliConn *getConnFromHeap(SHashObj *pConnHeap, char *key); +static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key); +static int32_t addConnToHeapCache(SHashObj *pConnHeap, SCliConn *pConn); static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { int32_t code = 0; @@ -1721,7 +1722,7 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon char addr[TSDB_FQDN_LEN + 64] = {0}; snprintf(addr, sizeof(addr), "%s:%d", ip, port); - pConn = getConnFromHeap(pThrd->connHeapCache, addr); + pConn = getConnFromHeapCache(pThrd->connHeapCache, addr); if (pConn != NULL) { *ppConn = pConn; return code; @@ -1763,6 +1764,12 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); + pConn->connnected = 1; + addConnToHeapCache(pThrd->connHeapCache, pConn); + + pConn->list = taosHashGet(pThrd->pool, addr, strlen(addr)); + pConn->list->totalSize += 1; + *ppConn = pConn; return code; @@ -1813,6 +1820,33 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); + pThrd->destroyAhandleFp = pInst->destroyFp; + + pThrd->fqdn2ipCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->fqdn2ipCache == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + + pThrd->batchCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->batchCache == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + + pThrd->connHeapCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->connHeapCache == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + pThrd->pIdConnTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pThrd->connHeapCache == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + pThrd->pQIdBuf = taosArrayInit(8, sizeof(int64_t)); + if (pThrd->pQIdBuf == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + *ppThrd = pThrd; return code; _end: @@ -2524,29 +2558,17 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { break; } } - tDebug("%s success to read resp", pInst->label); + tDebug("%s conn %p success to read resp", pInst->label, pConn); return code; _end: if (code != 0) { - tError("%s %s failed to handle resp at line %d since %s", pInst->label, __func__, line, tstrerror(code)); + tError("%s %s conn %p failed to handle resp at line %d since %s", pInst->label, __func__, pConn, line, + tstrerror(code)); } return code; } -// #define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) - -// static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); -// static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); -// static void transHeapDestroy(SHeap *heap); - -// static int32_t transHeapGet(SHeap *heap, SCliConn **p); -// static int32_t transHeapInsert(SHeap *heap, SCliConn *p); -// static int32_t transHeapDelete(SHeap *heap, SCliConn *p); -// static int32_t transHeapBalance(SHeap *heap, SCliConn *p); -// static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); -// static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); -// static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn); -static int32_t getOrCreateHeap(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { +static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { int32_t code = 0; size_t klen = strlen(key); @@ -2607,12 +2629,12 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { return 0; } -static SCliConn *getConnFromHeap(SHashObj *pConnHeap, char *key) { +static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key) { int32_t code = 0; SCliConn *pConn = NULL; SHeap *pHeap = NULL; - code = getOrCreateHeap(pConnHeap, key, &pHeap); + code = getOrCreateHeapCache(pConnHeap, key, &pHeap); if (code != 0) { tTrace("failed to get conn heap from cache for key:%s", key); return NULL; @@ -2646,7 +2668,7 @@ static int32_t addConnToHeapCache(SHashObj *pConnHeapCacahe, SCliConn *pConn) { tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, pConn->inHeap, pConn->reqRefCnt); } else { - code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + code = getOrCreateHeapCache(pConnHeapCacahe, pConn->dstAddr, &p); if (code != 0) { return code; } @@ -2815,6 +2837,10 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads } STrans *pInst = arg; + if (pInst->connLimitNum >= 512) { + pInst->connLimitNum = 512; + } + memcpy(cli->label, label, TSDB_LABEL_LEN); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrd2 **)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd2 *)); @@ -2975,9 +3001,9 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { } return code; } -static int32_t cliGetConnOrCreateConn(SCliThrd2 *pThrd, SCliConn **ppConn) { - int32_t code = 0; - SCliConn *pConn = NULL; +// static int32_t cliGetConnOrCreateConn(SCliThrd2 *pThrd, SCliConn **ppConn) { +// int32_t code = 0; +// SCliConn *pConn = NULL; - return code; -} \ No newline at end of file +// return code; +// } \ No newline at end of file diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index 9b66e18433e8..df4861c1c37a 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -303,13 +303,14 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndReq) { - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 100; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendReq(&req); - taosMsleep(100000); + //taosMsleep(100000); assert(resp.code == 0); } + taosMsleep(20000); } From 67796de315cecca72778d5af3eecd3cbd66993b5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 18:31:40 +0800 Subject: [PATCH 52/76] support select --- source/libs/transport/src/transImpl2.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 23185d235126..6899a5a23017 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1037,6 +1037,7 @@ static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { transLabel(pInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, len, pMsg->info.seq, pMsg->info.qId); evtBufPush(pBuf, (char *)pHead, len); + pResp->sent = 1; if (j++ > batchLimit) { break; } From fc549f96034fde9b83570644ea1cc4214d29113b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 24 Dec 2024 09:25:40 +0800 Subject: [PATCH 53/76] support select --- source/libs/transport/src/transImpl2.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 6899a5a23017..7ee554628a9e 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -980,7 +980,7 @@ static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { TAOS_CHECK_GOTO(terrno, &lino, _end); } memcpy(pMsg, p->buf, msgLen); - memcpy(p->buf + msgLen, p->buf, p->len - msgLen); + memmove(p->buf, p->buf + msgLen, p->len - msgLen); p->len -= msgLen; code = evtSvrHandleReq(pConn, pMsg, msgLen); @@ -2550,7 +2550,7 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { TAOS_CHECK_GOTO(terrno, &line, _end); } memcpy(pMsg, p->buf, msgLen); - memcpy(p->buf + msgLen, p->buf, p->len - msgLen); + memmove(p->buf, p->buf + msgLen, p->len - msgLen); p->len -= msgLen; code = evtCliHandleResp(pConn, pMsg, msgLen); @@ -2841,6 +2841,9 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads if (pInst->connLimitNum >= 512) { pInst->connLimitNum = 512; } + if (pInst->shareConnLimit <= 0) { + pInst->shareConnLimit = 32; + } memcpy(cli->label, label, TSDB_LABEL_LEN); cli->numOfThreads = numOfThreads; From e086812c9a7fdb25d5393202fe1ee84785382956 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 25 Dec 2024 10:03:16 +0800 Subject: [PATCH 54/76] support select --- source/libs/transport/src/transImpl2.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 7ee554628a9e..742b82e9f9ab 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -2209,7 +2209,7 @@ static FORCE_INLINE void destroyReqAndAhanlde(void *param) { destroyReqWrapper(pReq, pThrd); } -int32_t cliHandleState_mayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { +int32_t cliHandleStateMayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { int32_t code = 0; SCliThrd2 *pThrd = conn->hostThrd; if (pHead->msgType == TDMT_SCH_TASK_RELEASE || pHead->msgType == TDMT_SCH_TASK_RELEASE + 1) { @@ -2258,7 +2258,7 @@ int32_t cliHandleState_mayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead } return 0; } -int32_t cliHandleState_mayCreateAhandle(SCliConn *conn, STransMsgHead *pHead, STransMsg *pResp) { +int32_t cliHandleStateMayCreateAhandle(SCliConn *conn, STransMsgHead *pHead, STransMsg *pResp) { int32_t code = 0; int64_t qId = taosHton64(pHead->qid); if (qId == 0) { @@ -2284,7 +2284,7 @@ static FORCE_INLINE void cliConnClearInitUserMsg(SCliConn *conn) { conn->pInitUserReq = NULL; } } -int32_t cliHandleState_mayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { +int32_t cliHandleStateMayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { int64_t qid = pReq->msg.info.qId; if (qid > 0) { STransCtx *pUserCtx = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); @@ -2295,7 +2295,7 @@ int32_t cliHandleState_mayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { return 0; } -int32_t cliHandleState_mayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { +int32_t cliHandleStateMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { int32_t code = 0; int64_t qid = pReq->msg.info.qId; SReqCtx *pCtx = pReq->ctx; @@ -2336,7 +2336,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { cliConnClearInitUserMsg(pConn); - if (cliHandleState_mayHandleReleaseResp(pConn, pHead)) { + if (cliHandleStateMayHandleReleaseResp(pConn, pHead)) { if (evtCliRecycleConn(pConn)) { return code; } @@ -2344,7 +2344,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { code = cliGetReqBySeq(pConn, seq, pHead->msgType, &pReq); if (code == TSDB_CODE_OUT_OF_RANGE) { - code = cliHandleState_mayCreateAhandle(pConn, pHead, &resp); + code = cliHandleStateMayCreateAhandle(pConn, pHead, &resp); if (code == 0) { code = evtCliBuildRespFromCont(NULL, &resp, pHead); code = cliNotifyCb(pConn, NULL, &resp); @@ -2363,7 +2363,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { return 0; } } else { - code = cliHandleState_mayUpdateStateTime(pConn, pReq); + code = cliHandleStateMayUpdateStateTime(pConn, pReq); if (code != 0) { tDebug("%s conn %p failed to update state time sid:%" PRId64 " since %s", CONN_GET_INST_LABEL(pConn), pConn, qId, tstrerror(code)); @@ -2486,7 +2486,10 @@ static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { if (j >= batchLimit) { break; } - tDebug("%s send req %p, seq:%" PRId64, pInst->label, pCliMsg, pConn->seq); + STraceId *trace = &pReq->info.traceId; + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, contLen, pConn->seq, + pReq->info.qId); } return code; _end: From 1041232b521038c9f86c19e65160bb8ef5b94991 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Dec 2024 19:29:59 +0800 Subject: [PATCH 55/76] support select --- source/libs/transport/inc/transComm.h | 39 +- source/libs/transport/src/trans.c | 74 ++- source/libs/transport/src/transCli.c | 386 +++++++-------- source/libs/transport/src/transImpl2.c | 653 ++++++++++++++++++++++++- 4 files changed, 910 insertions(+), 242 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2f8ec38ca88e..a0ed95f67341 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -333,30 +333,53 @@ int32_t transInitBuffer(SConnBuffer* buf); int32_t transClearBuffer(SConnBuffer* buf); void transDestroyBuffer(SConnBuffer* buf); int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); -void transRefSrvHandle(void* handle); -void transUnrefSrvHandle(void* handle); -void transRefCliHandle(void* handle); -void transUnrefCliHandle(void* handle); + +#ifdef TD_ACORE +int32_t transSendRequest2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); +int32_t transSendRecv2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); +int32_t transSendRecvWithTimeout2(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); +int32_t transSendRequestWithId2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); +int32_t transFreeConnById2(void* pInit, int64_t transpointId); +int32_t transSendResponse2(STransMsg* msg); +int32_t transRegisterMsg2(const STransMsg* msg); +int32_t transSetDefaultAddr2(void* pInit, const char* ip, const char* fqdn); +int32_t transSetIpWhiteList2(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle2(void* handle); +void transUnrefSrvHandle2(void* handle); + +void transRefCliHandle2(void* handle); +void transUnrefCliHandle2(void* handle); int32_t transGetRefCount(void* handle); -int32_t transReleaseCliHandle(void* handle); -int32_t transReleaseSrvHandle(void* handle); +int32_t transReleaseCliHandle2(void* handle, int32_t status); +int32_t transReleaseSrvHandle2(void* handle, int32_t status); +#else int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); -int32_t transSendRequest2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs); + int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); int32_t transFreeConnById(void* pInit, int64_t transpointId); int32_t transSendResponse(STransMsg* msg); -int32_t transSendResponse2(STransMsg* msg); int32_t transRegisterMsg(const STransMsg* msg); int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle(void* handle); +void transUnrefSrvHandle(void* handle); + +void transRefCliHandle(void* handle); +void transUnrefCliHandle(void* handle); +int32_t transGetRefCount(void* handle); + +int32_t transReleaseCliHandle(void* handle); +int32_t transReleaseSrvHandle(void* handle); +#endif void transSockInfo2Str(struct sockaddr* sockname, char* dst); int32_t transAllocHandle(int64_t* refId); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 41e9d10de567..a95cf9e82c2c 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -257,19 +257,17 @@ void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOf transInitServer2, transInitClient2}; void (*taosCloseHandle[])(void* arg) = {transCloseServer2, transCloseClient2}; +int (*transReleaseHandle[])(void* handle, int32_t status) = {transReleaseSrvHandle2, transReleaseCliHandle2}; #else void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { transInitServer, transInitClient}; void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; -#endif - -void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; -void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; - int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; +#endif + static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; } typedef struct { char* lablset; @@ -436,55 +434,93 @@ void* rpcReallocCont(void* ptr, int64_t contLen) { } int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { -#ifndef TD_ACORE - return transSendRequest(shandle, pEpSet, pMsg, NULL); -#else +#ifdef TD_ACORE return transSendRequest2(shandle, pEpSet, pMsg, NULL); +#else + return transSendRequest(shandle, pEpSet, pMsg, NULL); #endif } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { +#ifdef TD_ACORE + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest2(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId2(shandle, pEpSet, pMsg, pRid); + } +#else if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { return transSendRequest(shandle, pEpSet, pMsg, pCtx); } else { return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); } +#endif } int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { +#ifdef TD_ACORE + return transSendRequestWithId2(shandle, pEpSet, pReq, transpointId); +#else return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +#endif } int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { +#ifdef TD_ACORE + return transSendRecv2(shandle, pEpSet, pMsg, pRsp); +#else return transSendRecv(shandle, pEpSet, pMsg, pRsp); +#endif } int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs) { +#ifdef TD_ACORE + return transSendRecvWithTimeout2(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#else return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#endif +} +int32_t rpcFreeConnById(void* shandle, int64_t connId) { +#ifdef TD_ACORE + return transFreeConnById2(shandle, connId); +#else + return transFreeConnById(shandle, connId); +#endif } -int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); } int32_t rpcSendResponse(SRpcMsg* pMsg) { -#ifndef TD_ACORE - return transSendResponse(pMsg); -#else +#ifdef TD_ACORE return transSendResponse2(pMsg); +#else + return transSendResponse(pMsg); #endif } -void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } - -void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } - -int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } -int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } +int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { +#ifdef TD_ACORE + return transRegisterMsg2(msg); +#else + return transRegisterMsg(msg); +#endif +} +int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle, 0); } // client only int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { // later +#ifdef TD_ACORE + return transSetDefaultAddr2(thandle, ip, fqdn); +#else return transSetDefaultAddr(thandle, ip, fqdn); +#endif } // server only -int32_t rpcSetIpWhite(void* thandle, void* arg) { return transSetIpWhiteList(thandle, arg, NULL); } +int32_t rpcSetIpWhite(void* thandle, void* arg) { +#ifdef TD_ACORE + return transSetIpWhiteList2(thandle, arg, NULL); +#else + return transSetIpWhiteList(thandle, arg, NULL); +#endif +} int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle(refId); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3e5863e63b31..b7a7ec91d50e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3847,235 +3847,235 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn* pConn, char* key) { return 1; } -static FORCE_INLINE bool filterToDebug(void* e, void* arg) { - SCliReq* pReq = QUEUE_DATA(e, SCliReq, q); - STraceId* trace = &pReq->msg.info.traceId; - tGWarn("%s is sent to, and no resp from server", TMSG_INFO(pReq->msg.msgType)); - return false; -} -static FORCE_INLINE void logConnMissHit(SCliConn* pConn) { - // queue set; - // QUEUE_INIT(&set); - SCliThrd* pThrd = pConn->hostThrd; - STrans* pInst = pThrd->pInst; - pConn->heapMissHit++; - tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, - transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), - pInst->shareConnLimit); - // if (transQueueSize(&pConn->reqsSentOut) >= pInst->shareConnLimit) { - // transQueueRemoveByFilter(&pConn->reqsSentOut, filterToDebug, NULL, &set, 1); - // } -} -static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key) { - int code = 0; - SHeap* pHeap = NULL; - SCliConn* pConn = NULL; - code = getOrCreateHeap(pConnHeapCache, key, &pHeap); - if (code != 0) { - tTrace("failed to get conn heap from cache for key:%s", key); - return NULL; + static FORCE_INLINE bool filterToDebug(void* e, void* arg) { + SCliReq* pReq = QUEUE_DATA(e, SCliReq, q); + STraceId* trace = &pReq->msg.info.traceId; + tGWarn("%s is sent to, and no resp from server", TMSG_INFO(pReq->msg.msgType)); + return false; } - code = transHeapGet(pHeap, &pConn); - if (code != 0) { - tTrace("failed to get conn from heap cache for key:%s", key); - return NULL; - } else { - tTrace("conn %p get conn from heap cache for key:%s", pConn, key); - if (shouldSWitchToOtherConn(pConn, key)) { - SCliConn* pNewConn = NULL; - code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); - if (code == 1) { - tTrace("conn %p start to handle reqs", pNewConn); - return pNewConn; - } + static FORCE_INLINE void logConnMissHit(SCliConn * pConn) { + // queue set; + // QUEUE_INIT(&set); + SCliThrd* pThrd = pConn->hostThrd; + STrans* pInst = pThrd->pInst; + pConn->heapMissHit++; + tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), + pInst->shareConnLimit); + // if (transQueueSize(&pConn->reqsSentOut) >= pInst->shareConnLimit) { + // transQueueRemoveByFilter(&pConn->reqsSentOut, filterToDebug, NULL, &set, 1); + // } + } + static SCliConn* getConnFromHeapCache(SHashObj * pConnHeapCache, char* key) { + int code = 0; + SHeap* pHeap = NULL; + SCliConn* pConn = NULL; + code = getOrCreateHeap(pConnHeapCache, key, &pHeap); + if (code != 0) { + tTrace("failed to get conn heap from cache for key:%s", key); return NULL; } - } - - return pConn; -} -static int32_t addConnToHeapCache(SHashObj* pConnHeapCacahe, SCliConn* pConn) { - SHeap* p = NULL; - int32_t code = 0; - - if (pConn->heap != NULL) { - p = pConn->heap; - tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, - pConn->inHeap, pConn->reqRefCnt); - } else { - code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + code = transHeapGet(pHeap, &pConn); if (code != 0) { - return code; - } - if (pConn->connnected == 0) { - int64_t now = taosGetTimestampMs(); - if (now - p->lastConnFailTs < 3000) { - return TSDB_CODE_RPC_NETWORK_UNAVAIL; + tTrace("failed to get conn from heap cache for key:%s", key); + return NULL; + } else { + tTrace("conn %p get conn from heap cache for key:%s", pConn, key); + if (shouldSWitchToOtherConn(pConn, key)) { + SCliConn* pNewConn = NULL; + code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); + if (code == 1) { + tTrace("conn %p start to handle reqs", pNewConn); + return pNewConn; + } + return NULL; } } + + return pConn; } + static int32_t addConnToHeapCache(SHashObj * pConnHeapCacahe, SCliConn * pConn) { + SHeap* p = NULL; + int32_t code = 0; - code = transHeapInsert(p, pConn); - tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, - pConn->reqRefCnt); - return code; -} + if (pConn->heap != NULL) { + p = pConn->heap; + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, + pConn->inHeap, pConn->reqRefCnt); + } else { + code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + if (code != 0) { + return code; + } + if (pConn->connnected == 0) { + int64_t now = taosGetTimestampMs(); + if (now - p->lastConnFailTs < 3000) { + return TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + } + } -static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) { - if (pConn->heap != NULL) { - tTrace("conn %p try to delete from heap cache direct", pConn); - return transHeapDelete(pConn->heap, pConn); + code = transHeapInsert(p, pConn); + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, + pConn->reqRefCnt); + return code; } - SHeap* p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); - if (p == NULL) { - tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); - return 0; - } - int32_t code = transHeapDelete(p, pConn); - if (code != 0) { - tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); + static int32_t delConnFromHeapCache(SHashObj * pConnHeapCache, SCliConn * pConn) { + if (pConn->heap != NULL) { + tTrace("conn %p try to delete from heap cache direct", pConn); + return transHeapDelete(pConn->heap, pConn); + } + + SHeap* p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); + if (p == NULL) { + tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); + return 0; + } + int32_t code = transHeapDelete(p, pConn); + if (code != 0) { + tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); + } + return code; } - return code; -} -static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn) { - SCliThrd* pThrd = pConn->hostThrd; - STrans* pInst = pThrd->pInst; - SCliConn* pTopConn = NULL; - if (pConn->heap != NULL && pConn->inHeap != 0) { - TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); - if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { - int32_t curReqs = REQS_ON_CONN(pConn); - int32_t topReqs = REQS_ON_CONN(pTopConn); - if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { - *pNewConn = pTopConn; - return 1; + static int8_t balanceConnHeapCache(SHashObj * pConnHeapCache, SCliConn * pConn, SCliConn * *pNewConn) { + SCliThrd* pThrd = pConn->hostThrd; + STrans* pInst = pThrd->pInst; + SCliConn* pTopConn = NULL; + if (pConn->heap != NULL && pConn->inHeap != 0) { + TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); + if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { + int32_t curReqs = REQS_ON_CONN(pConn); + int32_t topReqs = REQS_ON_CONN(pTopConn); + if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { + *pNewConn = pTopConn; + return 1; + } } } - } - return 0; -} -// conn heap -int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) { - SCliConn* args1 = container_of(a, SCliConn, node); - SCliConn* args2 = container_of(b, SCliConn, node); - - int32_t totalReq1 = REQS_ON_CONN(args1); - int32_t totalReq2 = REQS_ON_CONN(args2); - if (totalReq1 > totalReq2) { return 0; } - return 1; -} -int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b)) { - heap->heap = heapCreate(cmpFunc); - if (heap->heap == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + // conn heap + int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) { + SCliConn* args1 = container_of(a, SCliConn, node); + SCliConn* args2 = container_of(b, SCliConn, node); - heap->cmpFunc = cmpFunc; - return 0; -} -void transHeapDestroy(SHeap* heap) { - if (heap != NULL) { - heapDestroy(heap->heap); + int32_t totalReq1 = REQS_ON_CONN(args1); + int32_t totalReq2 = REQS_ON_CONN(args2); + if (totalReq1 > totalReq2) { + return 0; + } + return 1; } -} -int32_t transHeapGet(SHeap* heap, SCliConn** p) { - if (heapSize(heap->heap) == 0) { - *p = NULL; - return -1; + int32_t transHeapInit(SHeap * heap, int32_t(*cmpFunc)(const HeapNode* a, const HeapNode* b)) { + heap->heap = heapCreate(cmpFunc); + if (heap->heap == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + heap->cmpFunc = cmpFunc; + return 0; } - HeapNode* minNode = heapMin(heap->heap); - if (minNode == NULL) { - *p = NULL; - return -1; + void transHeapDestroy(SHeap * heap) { + if (heap != NULL) { + heapDestroy(heap->heap); + } } - *p = container_of(minNode, SCliConn, node); - return 0; -} -int32_t transHeapInsert(SHeap* heap, SCliConn* p) { - // impl later - p->reqRefCnt++; - if (p->inHeap == 1) { - tTrace("failed to insert conn %p since already in heap", p); - return TSDB_CODE_DUP_KEY; + int32_t transHeapGet(SHeap * heap, SCliConn * *p) { + if (heapSize(heap->heap) == 0) { + *p = NULL; + return -1; + } + HeapNode* minNode = heapMin(heap->heap); + if (minNode == NULL) { + *p = NULL; + return -1; + } + *p = container_of(minNode, SCliConn, node); + return 0; } + int32_t transHeapInsert(SHeap * heap, SCliConn * p) { + // impl later + p->reqRefCnt++; + if (p->inHeap == 1) { + tTrace("failed to insert conn %p since already in heap", p); + return TSDB_CODE_DUP_KEY; + } - heapInsert(heap->heap, &p->node); - p->inHeap = 1; - p->lastAddHeapTime = taosGetTimestampMs(); - p->heap = heap; - return 0; -} -int32_t transHeapDelete(SHeap* heap, SCliConn* p) { - // impl later - if (p->connnected == 0) { - TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); + heapInsert(heap->heap, &p->node); + p->inHeap = 1; + p->lastAddHeapTime = taosGetTimestampMs(); + p->heap = heap; + return 0; } + int32_t transHeapDelete(SHeap * heap, SCliConn * p) { + // impl later + if (p->connnected == 0) { + TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); + } - if (p->inHeap == 0) { - tTrace("failed to del conn %p since not in heap", p); - return 0; - } else { - int64_t now = taosGetTimestampMs(); - if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { - tTrace("conn %p not added/delete to heap frequently", p); - return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + if (p->inHeap == 0) { + tTrace("failed to del conn %p since not in heap", p); + return 0; + } else { + int64_t now = taosGetTimestampMs(); + if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { + tTrace("conn %p not added/delete to heap frequently", p); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } } - } - p->inHeap = 0; - p->reqRefCnt--; - if (p->reqRefCnt == 0) { - heapRemove(heap->heap, &p->node); - tTrace("conn %p delete from heap", p); - } else if (p->reqRefCnt < 0) { - tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); - } else { - tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + p->inHeap = 0; + p->reqRefCnt--; + if (p->reqRefCnt == 0) { + heapRemove(heap->heap, &p->node); + tTrace("conn %p delete from heap", p); + } else if (p->reqRefCnt < 0) { + tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); + } else { + tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + } + return 0; } - return 0; -} -int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p) { - heap->lastConnFailTs = taosGetTimestampMs(); - return 0; -} -int32_t transHeapMayBalance(SHeap* heap, SCliConn* p) { - if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + int32_t transHeapUpdateFailTs(SHeap * heap, SCliConn * p) { + heap->lastConnFailTs = taosGetTimestampMs(); return 0; } - SCliThrd* pThrd = p->hostThrd; - STrans* pInst = pThrd->pInst; - int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; + int32_t transHeapMayBalance(SHeap * heap, SCliConn * p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + SCliThrd* pThrd = p->hostThrd; + STrans* pInst = pThrd->pInst; + int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; - SCliConn* topConn = NULL; - int32_t code = transHeapGet(heap, &topConn); - if (code != 0) { - return code; - } + SCliConn* topConn = NULL; + int32_t code = transHeapGet(heap, &topConn); + if (code != 0) { + return code; + } - if (topConn == p) return code; + if (topConn == p) return code; - int32_t reqsOnTop = REQS_ON_CONN(topConn); - int32_t reqsOnCur = REQS_ON_CONN(p); + int32_t reqsOnTop = REQS_ON_CONN(topConn); + int32_t reqsOnCur = REQS_ON_CONN(p); - if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { - TAOS_UNUSED(transHeapBalance(heap, p)); + if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { + TAOS_UNUSED(transHeapBalance(heap, p)); + } + return code; } - return code; -} -int32_t transHeapBalance(SHeap* heap, SCliConn* p) { - if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + int32_t transHeapBalance(SHeap * heap, SCliConn * p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + heapRemove(heap->heap, &p->node); + heapInsert(heap->heap, &p->node); return 0; } - heapRemove(heap->heap, &p->node); - heapInsert(heap->heap, &p->node); - return 0; -} #else void transRefCliHandle(void* handle) { return; } @@ -4138,9 +4138,6 @@ int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* p return 0; } -int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - return transSendRecvWithTimeout(shandle, (SEpSet*)pEpSet, pReq, pRsp, NULL, 0); -} int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; } int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs) { @@ -4172,6 +4169,9 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, taosReleaseRef(transGetInstMgt(), (int64_t)shandle); return 0; } +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { + return transSendRecvWithTimeout(shandle, (SEpSet*)pEpSet, pReq, pRsp, NULL, 0); +} int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return 0; } int32_t transAllocHandle(int64_t* refId) { return 0; } int32_t transFreeConnById(void* shandle, int64_t transpointId) { return 0; } diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 742b82e9f9ab..02ce81756290 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1147,6 +1147,50 @@ static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { transQueuePush(&pConn->resps, &pResp->q); return evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } + +static void evtSvrHandleResp(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = evtSvrHandleSendResp(pThrd, pResp); + if (code != 0) { + tError("failed to send resp since %s", tstrerror(code)); + } +} +static void evtSvrHandleUpdate(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + tDebug("conn %p recv update, do nothing", pResp->pConn); + taosMemoryFree(pResp); +} + +static void evtSvrHandleRegiter(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = 0; + SSvrConn *pConn = pResp->pConn; + + int64_t qid = pResp->msg.info.qId; + SSvrRegArg *p = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (p != NULL) { + transFreeMsg(p->msg.pCont); + } + + SSvrRegArg arg = {.notifyCount = 0, .msg = pResp->msg}; + code = taosHashPut(pConn->pQTable, &arg.msg.info.qId, sizeof(arg.msg.info.qId), &arg, sizeof(arg)); + if (code != 0) { + tError("failed to put qid to hash since %s", tstrerror(code)); + return; + } + tDebug("conn %p success to register sid:%" PRId64 "", pConn, arg.msg.info.qId); + taosMemoryFree(pResp); +} +static void evtSvrHandleQuit(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + pThrd->quit = 1; + taosMemoryFree(pResp); +} +static void evtSvrHandleRelease(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = 0; + taosMemoryFree(pResp); + return; +} + +static void (*transAsyncFuncs[])(SSvrRespMsg *, SWorkThrd2 *) = { + evtSvrHandleResp, evtSvrHandleQuit, evtSvrHandleRelease, evtSvrHandleRegiter, evtSvrHandleUpdate}; + void evtSvrHandleAyncCb(void *async, int32_t status) { int32_t code = 0; @@ -1168,21 +1212,28 @@ void evtSvrHandleAyncCb(void *async, int32_t status) { QUEUE_REMOVE(el); SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); - - STransMsg transMsg = pResp->msg; - SExHandle *exh1 = transMsg.info.handle; - int64_t refId = transMsg.info.refId; - SExHandle *exh2 = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); - if (exh1 != exh2) { - tError("failed to acquire handle since %s", tstrerror(TSDB_CODE_REF_INVALID_ID)); + if (pResp == NULL) { continue; } - pResp->seqNum = transMsg.info.seq; - pResp->pConn = exh2->handle; + if (pResp->type == Quit || pResp->type == Update) { + transAsyncFuncs[pResp->type](pResp, pThrd); + } else { + STransMsg transMsg = pResp->msg; + SExHandle *exh1 = transMsg.info.handle; + int64_t refId = transMsg.info.refId; + SExHandle *exh2 = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); + if (exh1 != exh2) { + tError("failed to acquire handle since %s", tstrerror(TSDB_CODE_REF_INVALID_ID)); + continue; + } + + pResp->seqNum = transMsg.info.seq; + pResp->pConn = exh2->handle; - transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); - code = evtSvrHandleSendResp(handle->hostThrd, pResp); + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); + transAsyncFuncs[pResp->type](pResp, pThrd); + } } return; @@ -1314,11 +1365,102 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads return NULL; } -// int32_t transReleaseSrvHandle(void *handle) { return 0; } -// void transRefSrvHandle(void *handle) { return; } +int32_t transReleaseSrvHandle2(void *handle, int32_t status) { + int32_t code = 0; + SRpcHandleInfo *info = handle; + SExHandle *exh = info->handle; + int64_t qId = info->qId; + int64_t refId = info->refId; + + ASYNC_CHECK_HANDLE(info->refIdMgt, refId, exh); + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); -// void transUnrefSrvHandle(void *handle) { return; } + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .code = status, + .info.handle = exh, + .info.ahandle = NULL, + .info.refId = refId, + .info.qId = qId, + .info.traceId = info->traceId}; + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + + m->msg = tmsg; + m->type = Normal; + + tDebug("%s conn %p start to send %s, sid:%" PRId64 "", transLabel(pThrd->pInst), exh->handle, TMSG_INFO(tmsg.msgType), + qId); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(info->refIdMgt, refId); + return code; + } + + transReleaseExHandle(info->refIdMgt, refId); + return 0; +_return1: + tDebug("handle %p failed to send to release handle", exh); + transReleaseExHandle(info->refIdMgt, refId); + return code; +_return2: + tDebug("handle %p failed to send to release handle", exh); + return code; +} +int32_t transRegisterMsg2(const STransMsg *msg) { + int32_t code = 0; + + SExHandle *exh = msg->info.handle; + int64_t refId = msg->info.refId; + ASYNC_CHECK_HANDLE(msg->info.refIdMgt, refId, exh); + + STransMsg tmsg = *msg; + tmsg.info.noResp = 1; + + tmsg.info.qId = msg->info.qId; + tmsg.info.seq = msg->info.seq; + tmsg.info.refId = refId; + tmsg.info.refIdMgt = msg->info.refIdMgt; + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + + m->msg = tmsg; + m->type = Register; + + STrans *pInst = pThrd->pInst; + tDebug("%s conn %p start to register brokenlink callback", transLabel(pInst), exh->handle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; + } + + transReleaseExHandle(msg->info.refIdMgt, refId); + return 0; + +_return1: + tDebug("handle %p failed to register brokenlink", exh); + rpcFreeCont(msg->pCont); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; +_return2: + tDebug("handle %p failed to register brokenlink", exh); + rpcFreeCont(msg->pCont); + return code; +} +int32_t transSetIpWhiteList2(void *thandle, void *arg, FilteFunc *func) { return 0; } int32_t transSendResponse2(STransMsg *msg) { int32_t code = 0; @@ -1890,7 +2032,7 @@ static FORCE_INLINE SCliThrd2 *transGetWorkThrdFromHandle(STrans *trans, int64_t return pThrd; } -SCliThrd2 *transGetWorkThrd(STrans *trans, int64_t handle) { +static SCliThrd2 *transGetWorkThrd(STrans *trans, int64_t handle) { if (handle == 0) { int idx = cliRBChoseIdx(trans); if (idx < 0) return NULL; @@ -1962,6 +2104,48 @@ static FORCE_INLINE void destroyReq(void *arg) { transFreeMsg(pReq->msg.pCont); taosMemoryFree(pReq); } + +int32_t transReleaseCliHandle2(void *handle, int32_t status) { + int32_t code = 0; + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle); + if (pThrd == NULL) { + return TSDB_CODE_RPC_BROKEN_LINK; + } + + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .info.handle = handle, + .info.ahandle = (void *)0, + .info.qId = (int64_t)handle, + code = status}; + + TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + return terrno; + } + pCtx->ahandle = tmsg.info.ahandle; + SCliReq *cmsg = taosMemoryCalloc(1, sizeof(SCliReq)); + + if (cmsg == NULL) { + taosMemoryFree(pCtx); + return terrno; + } + cmsg->msg = tmsg; + cmsg->st = taosGetTimestampUs(); + cmsg->type = Normal; + cmsg->ctx = pCtx; + + STraceId *trace = &tmsg.info.traceId; + tGDebug("send release request at thread:%08" PRId64 ", malloc memory:%p", pThrd->pid, cmsg); + + if ((code = evtAsyncSend(pThrd->asyncHandle, &cmsg->q)) != 0) { + destroyReq(cmsg); + return code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code; + } + return code; +} + int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx) { STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); if (pInst == NULL) { @@ -2004,6 +2188,437 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, } return code; } +static bool isReqExceedLimit(STransMsg *pReq) { + if (pReq != NULL && pReq->contLen >= TRANS_MSG_LIMIT) { + return true; + } + return false; +} +int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, int64_t *transpointId) { + if (transpointId == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (isReqExceedLimit(pReq)) { + return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; + } + int32_t code = 0; + int8_t transIdInited = 0; + + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + + TAOS_CHECK_GOTO(transAllocHandle(transpointId), NULL, _exception); + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, *transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception); + } + + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), *transpointId); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + transIdInited = 1; + + pReq->info.handle = (void *)(*transpointId); + pReq->info.qId = *transpointId; + + SCliReq *pCliMsg = NULL; + TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, NULL, &pCliMsg), NULL, _exception); + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &(pCliMsg->q))) != 0) { + destroyReq(pCliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + } + + transReleaseExHandle(transGetRefMgt(), *transpointId); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return 0; + +_exception: + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + if (transIdInited) transReleaseExHandle(transGetRefMgt(), *transpointId); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + + tError("failed to send request since %s", tstrerror(code)); + return code; +} +int32_t transSendRecv2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransMsg *pRsp) { + if (isReqExceedLimit(pReq)) { + return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; + } + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + int32_t code = 0; + SCliReq *pCliReq = NULL; + SReqCtx *pCtx = NULL; + + STransMsg *pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); + if (pTransRsp == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _RETURN1); + } + + tsem_t *sem = taosMemoryCalloc(1, sizeof(tsem_t)); + if (sem == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + code = tsem_init(sem, 0, 0); + if (code != 0) { + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_UNUSED(tsem_destroy(sem)); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(code, NULL, _RETURN1); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(code, NULL, _RETURN1); + } + + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + pCtx->pSem = sem; + pCtx->pRsp = pTransRsp; + + pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCliReq == NULL) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pCtx->epSet), EPSET_GET_INUSE_PORT(pCtx->epSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); + if (code != 0) { + destroyReq(pReq); + TAOS_CHECK_GOTO((code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code), NULL, _RETURN); + } + TAOS_UNUSED(tsem_wait(sem)); + + memcpy(pRsp, pTransRsp, sizeof(STransMsg)); + +_RETURN: + tsem_destroy(sem); + taosMemoryFree(sem); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + taosMemoryFree(pTransRsp); + return code; +_RETURN1: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + taosMemoryFree(pTransRsp); + taosMemoryFree(pReq->pCont); + pReq->pCont = NULL; + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + return code; +} +static int32_t transCreateSyncMsg(STransMsg *pTransMsg, int64_t *refId) { + int32_t code = 0; + tsem2_t *sem = taosMemoryCalloc(1, sizeof(tsem2_t)); + if (sem == NULL) { + return terrno; + } + + if (tsem2_init(sem, 0, 0) != 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _EXIT); + } + + STransSyncMsg *pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); + if (pSyncMsg == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _EXIT); + } + + taosInitRWLatch(&pSyncMsg->latch); + pSyncMsg->inited = 0; + pSyncMsg->pRsp = pTransMsg; + pSyncMsg->pSem = sem; + pSyncMsg->hasEpSet = 0; + + int64_t id = taosAddRef(transGetSyncMsgMgt(), pSyncMsg); + if (id < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _EXIT); + } else { + *refId = id; + } + return 0; + +_EXIT: + TAOS_UNUSED(tsem2_destroy(sem)); + taosMemoryFree(sem); + taosMemoryFree(pSyncMsg); + return code; +} + +int32_t transSendRecvWithTimeout2(void *pInstRef, SEpSet *pEpSet, STransMsg *pReq, STransMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs) { + int32_t code = 0; + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + + STransMsg *pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); + if (pTransMsg == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _RETURN2); + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + taosMemoryFreeClear(pCtx->epSet); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + taosMemoryFreeClear(pCtx->epSet); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + + if ((code = transCreateSyncMsg(pTransMsg, &pCtx->syncMsgRef)) != 0) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + + int64_t ref = pCtx->syncMsgRef; + STransSyncMsg *pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), ref); + if (pSyncMsg == NULL) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _RETURN2); + } + + SCliReq *pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pReq == NULL) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pCtx->epSet), EPSET_GET_INUSE_PORT(pCtx->epSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); + if (code != 0) { + destroyReq(pReq); + TAOS_CHECK_GOTO(code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code, NULL, _RETURN); + goto _RETURN; + } + + code = tsem2_timewait(pSyncMsg->pSem, timeoutMs); + if (code != 0) { + pRsp->code = code; + } else { + memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + pSyncMsg->pRsp->pCont = NULL; + if (pSyncMsg->hasEpSet == 1) { + epsetAssign(pEpSet, &pSyncMsg->epSet); + *epUpdated = 1; + } + } +_RETURN: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), ref)); + TAOS_UNUSED(taosRemoveRef(transGetSyncMsgMgt(), ref)); + return code; +_RETURN2: + transFreeMsg(pReq->pCont); + + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + pReq->pCont = NULL; + taosMemoryFree(pTransMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} + +int32_t transSetDefaultAddr2(void *pInstRef, const char *ip, const char *fqdn) { + if (ip == NULL || fqdn == NULL) return TSDB_CODE_INVALID_PARA; + + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + + SCvtAddr cvtAddr = {0}; + tstrncpy(cvtAddr.ip, ip, sizeof(cvtAddr.ip)); + tstrncpy(cvtAddr.fqdn, fqdn, sizeof(cvtAddr.fqdn)); + cvtAddr.cvt = true; + + int32_t code = 0; + for (int8_t i = 0; i < pInst->numOfThreads; i++) { + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + code = terrno; + break; + } + + pCtx->pCvtAddr = (SCvtAddr *)taosMemoryCalloc(1, sizeof(SCvtAddr)); + if (pCtx->pCvtAddr == NULL) { + taosMemoryFree(pCtx); + code = terrno; + break; + } + + memcpy(pCtx->pCvtAddr, &cvtAddr, sizeof(SCvtAddr)); + + SCliReq *pReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pReq == NULL) { + taosMemoryFree(pCtx->pCvtAddr); + taosMemoryFree(pCtx); + code = terrno; + break; + } + + pReq->ctx = pCtx; + pReq->type = Update; + + SCliThrd2 *thrd = ((SCliObj2 *)pInst->tcphandle)->pThreadObj[i]; + tDebug("%s update epset at thread:%08" PRId64, pInst->label, thrd->pid); + + if ((code = evtAsyncSend(thrd->asyncHandle, &(pReq->q))) != 0) { + taosMemoryFree(pCtx->pCvtAddr); + destroyReq(pReq); + if (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT) { + code = TSDB_CODE_RPC_MODULE_QUIT; + } + break; + } + } + + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} +int32_t transAllocHandle2(int64_t *refId) { + SExHandle *exh = taosMemoryCalloc(1, sizeof(SExHandle)); + if (exh == NULL) { + return terrno; + } + + exh->refId = transAddExHandle(transGetRefMgt(), exh); + if (exh->refId < 0) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + SExHandle *self = transAcquireExHandle(transGetRefMgt(), exh->refId); + if (exh != self) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + QUEUE_INIT(&exh->q); + taosInitRWLatch(&exh->latch); + tDebug("trans alloc sid:%" PRId64 ", malloc:%p", exh->refId, exh); + *refId = exh->refId; + return 0; +} + +int32_t transFreeConnById2(void *pInstRef, int64_t transpointId) { + int32_t code = 0; + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + if (transpointId == 0) { + tDebug("not free by refId:%" PRId64 "", transpointId); + TAOS_CHECK_GOTO(0, NULL, _exception); + } + + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(pInst, transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); + } + + SCliReq *pCli = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCli == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + pCli->type = Normal; + + STransMsg msg = {.msgType = TDMT_SCH_TASK_RELEASE, .info.handle = (void *)transpointId}; + TRACE_SET_MSGID(&msg.info.traceId, tGenIdPI64()); + msg.info.qId = transpointId; + pCli->msg = msg; + + STraceId *trace = &pCli->msg.info.traceId; + tGDebug("%s start to free conn sid:%" PRId64 "", pInst->label, transpointId); + + code = evtAsyncSend(pThrd->asyncHandle, &pCli->q); + if (code != 0) { + taosMemoryFreeClear(pCli); + TAOS_CHECK_GOTO(code, NULL, _exception); + } + +_exception: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} static int32_t evtCliRecycleConn(SCliConn *pConn) { int32_t code = 0; @@ -3007,10 +3622,4 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { TAOS_UNUSED(transHeapBalance(heap, p)); } return code; -} -// static int32_t cliGetConnOrCreateConn(SCliThrd2 *pThrd, SCliConn **ppConn) { -// int32_t code = 0; -// SCliConn *pConn = NULL; - -// return code; -// } \ No newline at end of file +} \ No newline at end of file From 184731c536136fff2d4af597e93a98c09ff9ebcf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 27 Dec 2024 10:54:24 +0800 Subject: [PATCH 56/76] refactor code --- source/libs/transport/src/transImpl2.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 02ce81756290..f0a1238fa654 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -741,10 +741,6 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { TAOS_CHECK_GOTO(code, &lino, _end); } - // if ((code = transQueueInit(&pConn->resps, uvDestroyResp)) != 0) { - // TAOS_CHECK_GOTO(code, &lino, _end); - //} - pConn->broken = false; pConn->status = ConnNormal; @@ -777,9 +773,7 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { } QUEUE_PUSH(&pThrd->conn, &pConn->queue); transQueueInit(&pConn->resps, NULL); - // code = initWQ(&pConn->wq); - // TAOS_CHECK_GOTO(code, &lino, _end); - // wqInited = 1; + pConn->hostThrd = pThrd; return pConn; _end: @@ -831,11 +825,6 @@ bool connMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen return false; } -static int32_t evtConnHandleReleaseReq(SSvrConn *pConn, STransMsgHead *phead) { - int32_t code = 0; - return code; -} - static int32_t evtSvrHandleReleaseReq(SSvrConn *pConn, STransMsgHead *pHead) { int32_t code = 0; STrans *pInst = pConn->pInst; @@ -3484,6 +3473,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads pThrd->thrdInited = 1; cli->pThreadObj[i] = pThrd; } + return cli; _exit: From 8be009a944936bd1637f61b69b97abd33c58072d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 27 Dec 2024 11:46:22 +0800 Subject: [PATCH 57/76] refactor code --- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/src/transImpl2.c | 174 ++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a0ed95f67341..2244236be1de 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -475,12 +475,13 @@ void transQueueDestroy(STransQueue* queue); /* * delay queue based on uv loop and uv timer, and only used in retry */ + +#ifndef TD_ACORE + typedef struct STaskArg { void* param1; void* param2; } STaskArg; - -#ifndef TD_ACORE typedef struct SDelayTask { void (*func)(void* arg); void* arg; diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index f0a1238fa654..e4a9409239d1 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -57,6 +57,26 @@ typedef struct { queue q; } SFdQueue; +typedef struct STaskArg { + void *param1; + void *param2; +} STaskArg; +typedef struct SDelayTask { + void (*func)(void *arg); + void *arg; + uint64_t execTime; + HeapNode node; +} SDelayTask; + +typedef struct { + Heap *heap; + void *mgt; +} SDelayQueue; + +int32_t transDQCreate(void *loop, SDelayQueue **queue); +void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); +SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); +void transDQCancel(SDelayQueue *queue, SDelayTask *task); typedef struct { int notifyCount; // int init; // init or not @@ -227,6 +247,9 @@ typedef struct { int32_t fd[2048]; int32_t fdIdx; + void *arg; + int32_t (*timeoutFunc)(void *arg); + int32_t (*caclTimeout)(void *arg); } SEvtMgt; static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); @@ -276,6 +299,15 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { return code; } +static int32_t evtMgtAddTimoutFunc(SEvtMgt *pOpt, void *arg, int32_t (*timeoutFunc)(void *arg), + int32_t (*caclTimeout)(void *arg)) { + int32_t code = 0; + + pOpt->arg = arg; + pOpt->timeoutFunc = timeoutFunc; + pOpt->caclTimeout = caclTimeout; + return code; +} static int32_t evtBufInit(SEvtBuf *evtBuf) { int32_t code = 0; @@ -413,9 +445,33 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { } return evtMgtHandleImpl(pOpt, pArg, res); } + +static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { + int32_t code = 0; + int32_t timeout = 0; + if (pOpt->caclTimeout) { + timeout = pOpt->caclTimeout(pOpt->arg); + if (timeout <= 0) { + tv->tv_sec = 5; + tv->tv_usec = 0; + } + } + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout % 1000) * 1000; + return code; +} static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; + if (pOpt->timeoutFunc != NULL) { + code = pOpt->timeoutFunc(pOpt->arg); + if (code != 0) { + tError("failed to handle timeout since %s", tstrerror(code)); + } else { + tInfo("succ to handle timeout since %s", tstrerror(code)); + } + } + if (pOpt->resizeOutSets) { fd_set *readSetOut = NULL, *writeSetOut = NULL; int32_t sz = pOpt->evtFdsSize; @@ -1262,6 +1318,13 @@ void *transWorkerThread(void *arg) { while (!pThrd->quit) { struct timeval tv = {30, 0}; + code = evtCaclNextTimeout(pOpt, &tv); + if (code != 0) { + tError("%s failed to cacl next timeout since %s", pInst->label, tstrerror(code)); + } else { + tDebug("%s succ to cacl next timeout", pInst->label); + } + code = evtMgtDispath(pOpt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); @@ -3612,4 +3675,113 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { TAOS_UNUSED(transHeapBalance(heap, p)); } return code; -} \ No newline at end of file +} +static FORCE_INLINE int32_t timeCompare(const HeapNode *a, const HeapNode *b) { + SDelayTask *arg1 = container_of(a, SDelayTask, node); + SDelayTask *arg2 = container_of(b, SDelayTask, node); + if (arg1->execTime > arg2->execTime) { + return 0; + } else { + return 1; + } +} +int32_t transDQCreate(void *loop, SDelayQueue **queue) { + int32_t code = 0; + Heap *heap = NULL; + // uv_timer_t* timer = NULL; + SDelayQueue *q = NULL; + + // timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + // if (timer == NULL) { + // return terrno; + // } + + heap = heapCreate(timeCompare); + if (heap == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _return1); + } + + q = taosMemoryCalloc(1, sizeof(SDelayQueue)); + if (q == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _return1); + } + q->heap = heap; + q->mgt = loop; + + *queue = q; + return 0; + +_return1: + heapDestroy(heap); + taosMemoryFree(q); + return TSDB_CODE_OUT_OF_MEMORY; +} +void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)) { + int32_t code = 0; + while (heapSize(queue->heap) > 0) { + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return; + } + heapRemove(queue->heap, minNode); + + SDelayTask *task = container_of(minNode, SDelayTask, node); + + STaskArg *arg = task->arg; + if (freeFunc) freeFunc(arg); + taosMemoryFree(arg); + + taosMemoryFree(task); + } + heapDestroy(queue->heap); + taosMemoryFree(queue); + return; +} +SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs) { + uint64_t now = taosGetTimestampMs(); + SDelayTask *task = taosMemoryCalloc(1, sizeof(SDelayTask)); + if (task == NULL) { + return NULL; + } + + task->func = func; + task->arg = arg; + task->execTime = now + timeoutMs; + + HeapNode *minNode = heapMin(queue->heap); + if (minNode) { + SDelayTask *minTask = container_of(minNode, SDelayTask, node); + if (minTask->execTime < task->execTime) { + timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; + } + } + + tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); + heapInsert(queue->heap, &task->node); + // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0)); + return NULL; +} +void transDQCancel(SDelayQueue *queue, SDelayTask *task) { + // TAOS_UNUSED(uv_timer_stop(queue->timer)); + + if (heapSize(queue->heap) <= 0) { + taosMemoryFree(task->arg); + taosMemoryFree(task); + return; + } + heapRemove(queue->heap, &task->node); + + taosMemoryFree(task->arg); + taosMemoryFree(task); + + if (heapSize(queue->heap) != 0) { + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) return; + + uint64_t now = taosGetTimestampMs(); + SDelayTask *task = container_of(minNode, SDelayTask, node); + uint64_t timeout = now > task->execTime ? now - task->execTime : 0; + + // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); + } +} From 65e3a9928c52374d081ccc22e0dfac51c544d0fe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 27 Dec 2024 16:08:55 +0800 Subject: [PATCH 58/76] refactor code --- source/libs/transport/src/transImpl2.c | 284 ++++++++++++++---- .../libs/transport/test/transImpl2_test.cpp | 2 +- 2 files changed, 225 insertions(+), 61 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index e4a9409239d1..3ea07f915901 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -77,6 +77,8 @@ int32_t transDQCreate(void *loop, SDelayQueue **queue); void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); void transDQCancel(SDelayQueue *queue, SDelayTask *task); +int32_t transDQHandleTimeout(SDelayQueue *queue); +int32_t transDQGetNextTimeout(SDelayQueue *queue); typedef struct { int notifyCount; // int init; // init or not @@ -247,13 +249,14 @@ typedef struct { int32_t fd[2048]; int32_t fdIdx; + char label[128]; void *arg; int32_t (*timeoutFunc)(void *arg); - int32_t (*caclTimeout)(void *arg); + int32_t (*caclTimeout)(void *arg, int64_t *timeout); } SEvtMgt; static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); -static int32_t evtMgtCreate(SEvtMgt **pOpt); +static int32_t evtMgtCreate(SEvtMgt **pOpt, char *label); static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv); static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); @@ -273,7 +276,7 @@ static int32_t evtInitPipe(int32_t dst[2]) { dst[1] = fd[1]; return code; } -static int32_t evtMgtCreate(SEvtMgt **pOpt) { +static int32_t evtMgtCreate(SEvtMgt **pOpt, char *label) { int32_t code = 0; SEvtMgt *pRes = taosMemoryCalloc(1, sizeof(SEvtMgt)); if (pRes == NULL) { @@ -289,6 +292,7 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { pRes->evtWriteSetOut = NULL; pRes->pFdTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pRes->fdIdx = 0; + memcpy(pRes->label, label, strlen(label)); code = evtMgtResize(pRes, (32 + 1)); if (code != 0) { @@ -300,10 +304,10 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt) { return code; } static int32_t evtMgtAddTimoutFunc(SEvtMgt *pOpt, void *arg, int32_t (*timeoutFunc)(void *arg), - int32_t (*caclTimeout)(void *arg)) { + int32_t (*caclTimeout)(void *arg, int64_t *timeout)) { int32_t code = 0; - pOpt->arg = arg; + // pOpt->arg = arg; pOpt->timeoutFunc = timeoutFunc; pOpt->caclTimeout = caclTimeout; return code; @@ -383,7 +387,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { SEvtBuf *pBuf = &pArg->buf; code = evtBufInit(pBuf); if (code != 0) { - tError("failed to init buf since %s", tstrerror(code)); + tError("%s failed to init buf since %s", pOpt->label, tstrerror(code)); return code; } @@ -400,13 +404,13 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { SEvtBuf *pBuf = &pArg->sendBuf; code = evtBufInit(pBuf); if (code != 0) { - tError("failed to init wbuf since %s", tstrerror(code)); + tError("%s failed to init wbuf since %s", pOpt->label, tstrerror(code)); return code; } code = pArg->sendCb(pArg, pBuf, 0); if (code != 0) { - tError("failed to build send buf since %s", tstrerror(code)); + tError("%s failed to build send buf since %s", pOpt->label, tstrerror(code)); } int32_t total = pBuf->offset; int32_t offset = 0; @@ -427,9 +431,9 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { evtBufClear(&pArg->sendBuf); pArg->sendFinishCb(pArg, code); if (code != 0) { - tError("failed to send buf since %s", tstrerror(code)); + tError("%s failed to send buf since %s", pOpt->label, tstrerror(code)); } else { - tDebug("succ to send buf"); + tDebug("%s succ to send buf", pOpt->label); } return code; } @@ -448,16 +452,23 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0; - int32_t timeout = 0; + int64_t timeout = 0; if (pOpt->caclTimeout) { - timeout = pOpt->caclTimeout(pOpt->arg); - if (timeout <= 0) { - tv->tv_sec = 5; + code = pOpt->caclTimeout(pOpt->arg, &timeout); + if (timeout == 0) { + tv->tv_sec = 0; + tv->tv_usec = 100 * 1000; + } else if (timeout < 0) { + tv->tv_sec = 30; tv->tv_usec = 0; + } else { + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout % 1000) * 1000; } + } else { + tv->tv_sec = 30; + tv->tv_usec = 0; } - tv->tv_sec = timeout / 1000; - tv->tv_usec = (timeout % 1000) * 1000; return code; } static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { @@ -466,9 +477,9 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (pOpt->timeoutFunc != NULL) { code = pOpt->timeoutFunc(pOpt->arg); if (code != 0) { - tError("failed to handle timeout since %s", tstrerror(code)); + tError("%s failed to handle timeout since %s", pOpt->label, tstrerror(code)); } else { - tInfo("succ to handle timeout since %s", tstrerror(code)); + tInfo("%s succ to handle timeout since %s", pOpt->label, tstrerror(code)); } } @@ -500,7 +511,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); } else if (active_Fds == 0) { - tDebug("select timeout occurred"); + tDebug("%s select timeout occurred", pOpt->label); return code; } @@ -520,9 +531,9 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { active_Fds--; code = evtMgtHandle(pOpt, res, fd); if (code != 0) { - tError("failed to handle fd %d since %s", fd, tstrerror(code)); + tError("%s failed to handle fd %d since %s", pOpt->label, fd, tstrerror(code)); } else { - tDebug("success to handle fd %d", fd); + tDebug("%s success to handle fd %d", pOpt->label, fd); } } } @@ -634,7 +645,7 @@ static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg } SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); if (pArg == NULL) { - tError("%s failed to get fd %d since %s", __func__, fd, tstrerror(TAOS_SYSTEM_ERROR(EBADF))); + tError("%s %s failed to get fd %d since %s", pOpt->label, __func__, fd, tstrerror(TAOS_SYSTEM_ERROR(EBADF))); return 0; } pArg->rwRef -= rwRef; @@ -666,7 +677,7 @@ static void evtMgtDestroy(SEvtMgt *pOpt) { if (pOpt->evtWriteSetOut) { taosMemoryFree(pOpt->evtWriteSetOut); } - + tDebug("%s succ to destroy evt mgt", pOpt->label); taosHashCleanup(pOpt->pFdTable); taosMemoryFree(pOpt); } @@ -1294,7 +1305,7 @@ void *transWorkerThread(void *arg) { SEvtMgt *pOpt = NULL; - code = evtMgtCreate(&pOpt); + code = evtMgtCreate(&pOpt, pInst->label); if (code != 0) { tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); @@ -1720,8 +1731,48 @@ typedef struct SCliThrd2 { int32_t pipe_queue_fd[2]; SEvtMgt *pEvtMgt; SAsyncHandle *asyncHandle; + int32_t connLimit; } SCliThrd2; +typedef struct { + SDelayQueue *queue; + void *pEvtMgt; +} STimeoutArg; + +int32_t createTimeoutArg(void *pEvtMgt, STimeoutArg **pArg) { + int32_t code = 0; + int32_t line = 0; + + STimeoutArg *arg = taosMemoryCalloc(1, sizeof(STimeoutArg)); + if (arg == NULL) { + code = terrno; + TAOS_CHECK_GOTO(code, &line, _end); + } + SDelayQueue *queue = NULL; + transDQCreate(arg, &queue); + + arg->queue = queue; + arg->pEvtMgt = pEvtMgt; + *pArg = arg; + return code; +_end: + if (code != 0) { + tError("%s failed to create timeout arg at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} + +void destroyTimeoutArg(STimeoutArg *arg) { + if (arg == NULL) { + return; + } + + SDelayQueue *queue = arg->queue; + transDQDestroy(queue, NULL); + taosMemoryFree(arg); + return; +} + typedef struct SCliObj2 { char label[TSDB_LABEL_LEN]; int32_t index; @@ -1889,7 +1940,7 @@ static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppC } if (QUEUE_IS_EMPTY(&plist->conns)) { - if (plist->totalSize >= pInst->connLimitNum) { + if (plist->totalSize >= pThrd->connLimit) { return TSDB_CODE_RPC_MAX_SESSIONS; } return TSDB_CODE_RPC_NETWORK_BUSY; @@ -1928,6 +1979,9 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon *ppConn = pConn; return 0; } + if (code == TSDB_CODE_RPC_MAX_SESSIONS) { + return code; + } pConn = taosMemoryCalloc(1, sizeof(SCliConn)); if (pConn == NULL) { @@ -3239,6 +3293,16 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { return code; } +static FORCE_INLINE void evtLogConnMissHit(SCliConn *pConn) { + // queue set; + // QUEUE_INIT(&set); + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + pConn->heapMissHit++; + tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), + pThrd->connLimit); +} static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { int32_t code = 0; size_t klen = strlen(key); @@ -3270,14 +3334,16 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { SCliThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; - tDebug("get conn %p from heap cache for key:%s, status:%d, refCnt:%d", pConn, key, pConn->inHeap, pConn->reqRefCnt); int32_t reqsNum = transQueueSize(&pConn->reqsToSend); int32_t reqsSentOut = transQueueSize(&pConn->reqsSentOut); int32_t stateNum = taosHashGetSize(pConn->pQTable); int32_t totalReqs = reqsNum + reqsSentOut; - if (totalReqs >= pInst->shareConnLimit) { - // logConnMissHit(pConn); + tDebug("get conn %p from heap cache for key:%s, status:%d, refCnt:%d, totalReqs:%d", pConn, key, pConn->inHeap, + pConn->reqRefCnt, totalReqs); + + if (totalReqs >= pThrd->connLimit) { + evtLogConnMissHit(pConn); if (pConn->list == NULL && pConn->dstAddr != NULL) { pConn->list = taosHashGet((SHashObj *)pThrd->pool, pConn->dstAddr, strlen(pConn->dstAddr)); @@ -3285,9 +3351,9 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { tTrace("conn %p get list %p from pool for key:%s", pConn, pConn->list, key); } } - if (pConn->list && pConn->list->totalSize >= pInst->connLimitNum / 4) { + if (pConn->list && pConn->list->totalSize >= pThrd->connLimit / 4) { tWarn("%s conn %p try to remove timeout msg since too many conn created", transLabel(pInst), pConn); - + pThrd->connLimit = pThrd->connLimit * 4; // TODO // if (cliConnRemoveTimeoutMsg(pConn)) { // tWarn("%s conn %p succ to remove timeout msg", transLabel(pInst), pConn); @@ -3384,7 +3450,7 @@ static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SC if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { int32_t curReqs = REQS_ON_CONN(pConn); int32_t topReqs = REQS_ON_CONN(pTopConn); - if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { + if (curReqs > topReqs && topReqs < pThrd->connLimit) { *pNewConn = pTopConn; return 1; } @@ -3392,19 +3458,56 @@ static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SC } return 0; } -static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { - int32_t code = 0; +FORCE_INLINE int32_t evtCliBuildExceptResp(SCliThrd2 *pThrd, SCliReq *pReq, STransMsg *pResp) { + if (pReq == NULL) return -1; + STrans *pInst = pThrd->pInst; - char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); - int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); - char addr[TSDB_FQDN_LEN + 64] = {0}; + + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + pResp->msgType = pReq ? pReq->msg.msgType + 1 : 0; + pResp->info.cliVer = pInst->compatibilityVer; + pResp->info.ahandle = pCtx ? pCtx->ahandle : 0; + if (pReq) { + pResp->info.traceId = pReq->msg.info.traceId; + } + + // handle noresp and inter manage msg + if (pCtx == NULL || pReq->msg.info.noResp == 1) { + return TSDB_CODE_RPC_NO_STATE; + } + if (pResp->code == 0) { + pResp->code = TSDB_CODE_RPC_BROKEN_LINK; + } + + return 0; +} +static int32_t evtCliHandleExcept(void *thrd, SCliReq *pReq, STransMsg *pResp) { + SCliThrd2 *pThrd = thrd; + STrans *pInst = pThrd->pInst; + int32_t code = evtCliBuildExceptResp(pThrd, pReq, pResp); + if (code != 0) { + destroyReq(pReq); + return code; + } + pInst->cfp(pInst->parent, pResp, NULL); + destroyReq(pReq); + return code; +} +static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { + int32_t lino = 0; + int32_t code = 0; + STransMsg resp = {0}; + STrans *pInst = pThrd->pInst; + char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); + int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); + char addr[TSDB_FQDN_LEN + 64] = {0}; snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); SCliConn *pConn = NULL; code = getOrCreateConn(pThrd, fqdn, port, &pConn); if (code != 0) { tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); - return code; + TAOS_CHECK_GOTO(code, &lino, _end); } else { transQueuePush(&pConn->reqsToSend, &req->q); STraceId *trace = &req->msg.info.traceId; @@ -3421,6 +3524,14 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); } return code; +_end: + resp.code = code; + STraceId *trace = &req->msg.info.traceId; + if (code != 0) { + tGError("%s failed to handle req since %s", pInst->label, tstrerror(code)); + } + evtCliHandleExcept(pThrd, req, &resp); + return code; } static void evtCliHandleAsyncCb(void *arg, int32_t status) { int32_t code = 0; @@ -3456,6 +3567,22 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { } } +static int32_t evtCliTimeoutFunc(void *arg) { + if (arg == 0) return 0; + + int32_t code = 0; + STimeoutArg *pArg = arg; + transDQHandleTimeout(pArg->queue); + return code; +} +static int32_t evtCliCalcTimeout(void *arg, int64_t *timeout) { + if (arg == 0) return 0; + STimeoutArg *pArg = arg; + int32_t code = 0; + + *timeout = transDQGetNextTimeout(pArg->queue); + return code; +} static void *cliWorkThread2(void *arg) { int32_t line = 0; int32_t code = 0; @@ -3469,10 +3596,8 @@ static void *cliWorkThread2(void *arg) { TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); setThreadName(threadName); - code = evtMgtCreate(&pThrd->pEvtMgt); - if (code != 0) { - TAOS_CHECK_GOTO(code, &line, _end); - } + code = evtMgtCreate(&pThrd->pEvtMgt, pInst->label); + TAOS_CHECK_GOTO(code, &line, _end); pThrd->pEvtMgt->hostThrd = pThrd; @@ -3480,8 +3605,18 @@ static void *cliWorkThread2(void *arg) { (void *)pThrd); TAOS_CHECK_GOTO(code, &line, _end); + createTimeoutArg(pThrd->pEvtMgt, (STimeoutArg **)&pThrd->pEvtMgt->arg); + TAOS_CHECK_GOTO(code, &line, _end); + + code = evtMgtAddTimoutFunc(pThrd->pEvtMgt, NULL, evtCliTimeoutFunc, evtCliCalcTimeout); + while (!pThrd->quit) { struct timeval tv = {30, 0}; + code = evtCaclNextTimeout(pThrd->pEvtMgt, &tv); + if (code != 0) { + } else { + tDebug("%s success to calc next timeout", pInst->label); + } code = evtMgtDispath(pThrd->pEvtMgt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); @@ -3527,7 +3662,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads code = createThrdObj(arg, &pThrd); TAOS_CHECK_EXIT(code); pThrd->pInst = pInst; - + pThrd->connLimit = pInst->connLimitNum; code = evtInitPipe(pThrd->pipe_queue_fd); TAOS_CHECK_EXIT(code); @@ -3658,7 +3793,7 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { } SCliThrd2 *pThrd = p->hostThrd; STrans *pInst = pThrd->pInst; - int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; + int32_t balanceLimit = pThrd->connLimit >= 4 ? pThrd->connLimit / 2 : 2; SCliConn *topConn = NULL; int32_t code = transHeapGet(heap, &topConn); @@ -3747,19 +3882,18 @@ SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, task->func = func; task->arg = arg; task->execTime = now + timeoutMs; + // HeapNode *minNode = heapMin(queue->heap); + // if (minNode) { + // SDelayTask *minTask = container_of(minNode, SDelayTask, node); + // if (minTask->execTime < task->execTime) { + // timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; + // } + // } - HeapNode *minNode = heapMin(queue->heap); - if (minNode) { - SDelayTask *minTask = container_of(minNode, SDelayTask, node); - if (minTask->execTime < task->execTime) { - timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; - } - } - - tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); + // tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); heapInsert(queue->heap, &task->node); // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0)); - return NULL; + return task; } void transDQCancel(SDelayQueue *queue, SDelayTask *task) { // TAOS_UNUSED(uv_timer_stop(queue->timer)); @@ -3774,14 +3908,44 @@ void transDQCancel(SDelayQueue *queue, SDelayTask *task) { taosMemoryFree(task->arg); taosMemoryFree(task); - if (heapSize(queue->heap) != 0) { - HeapNode *minNode = heapMin(queue->heap); - if (minNode == NULL) return; + // if (heapSize(queue->heap) != 0) { + // HeapNode *minNode = heapMin(queue->heap); + // if (minNode == NULL) return; - uint64_t now = taosGetTimestampMs(); - SDelayTask *task = container_of(minNode, SDelayTask, node); - uint64_t timeout = now > task->execTime ? now - task->execTime : 0; + // uint64_t now = taosGetTimestampMs(); + // SDelayTask *task = container_of(minNode, SDelayTask, node); + // uint64_t timeout = now > task->execTime ? now - task->execTime : 0; + + // } +} - // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); +int32_t transDQHandleTimeout(SDelayQueue *queue) { + int32_t code = 0; + uint64_t now = taosGetTimestampMs(); + while (heapSize(queue->heap) > 0) { + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return 0; + } + SDelayTask *task = container_of(minNode, SDelayTask, node); + if (task->execTime > now) { + break; + } + heapRemove(queue->heap, minNode); + task->func(task->arg); + taosMemoryFree(task); + } + return code; +} +int32_t transDQGetNextTimeout(SDelayQueue *queue) { + if (heapSize(queue->heap) <= 0) { + return -1; } + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return -1; + } + SDelayTask *task = container_of(minNode, SDelayTask, node); + uint64_t now = taosGetTimestampMs(); + return task->execTime > now ? task->execTime - now : 0; } diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index df4861c1c37a..bb48360beb25 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -303,7 +303,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndReq) { - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 10000; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 1; req.pCont = rpcMallocCont(10); From 23313f77dfae5fa6c88df00e1b33af11ef54cfc1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 27 Dec 2024 22:03:13 +0800 Subject: [PATCH 59/76] refactor code --- source/libs/transport/src/transImpl2.c | 73 +++++++++++++++++++------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 3ea07f915901..d6dee2c937a5 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -479,7 +479,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (code != 0) { tError("%s failed to handle timeout since %s", pOpt->label, tstrerror(code)); } else { - tInfo("%s succ to handle timeout since %s", pOpt->label, tstrerror(code)); + // tInfo("%s succ to handle timeout", pOpt->label); } } @@ -607,6 +607,10 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar if (arg == NULL) { return TSDB_CODE_INVALID_MSG; } + if (pOpt->fdIdx >= 1024) { + return TSDB_CODE_INVALID_MSG; + // taosClose(fd); + } if (fd != 0) { if (events & EVT_READ) { FD_SET(fd, pOpt->evtReadSetIn); @@ -621,7 +625,6 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); } } - return 0; } @@ -1731,7 +1734,7 @@ typedef struct SCliThrd2 { int32_t pipe_queue_fd[2]; SEvtMgt *pEvtMgt; SAsyncHandle *asyncHandle; - int32_t connLimit; + int32_t shareConnLimit; } SCliThrd2; typedef struct { @@ -1810,7 +1813,7 @@ static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn); static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key); static int32_t addConnToHeapCache(SHashObj *pConnHeap, SCliConn *pConn); -static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { +static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { int32_t code = 0; int32_t line = 0; int32_t sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1821,9 +1824,11 @@ static int32_t createSocket(char *ip, int32_t port, int32_t *fd) { memset(&server_addr, 0, sizeof(server_addr)); server_addr.sin_family = AF_INET; server_addr.sin_port = htons(port); - if (inet_pton(AF_INET, ip, &server_addr.sin_addr) <= 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); - } + server_addr.sin_addr.s_addr = ip; + + // if (inet_pton(AF_INET, ip, &server_addr.sin_addr) <= 0) { + // TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + // } if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); @@ -1940,7 +1945,7 @@ static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppC } if (QUEUE_IS_EMPTY(&plist->conns)) { - if (plist->totalSize >= pThrd->connLimit) { + if (plist->totalSize >= pInst->connLimitNum) { return TSDB_CODE_RPC_MAX_SESSIONS; } return TSDB_CODE_RPC_NETWORK_BUSY; @@ -1960,11 +1965,34 @@ static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppC *ppConn = conn; return 0; } +static int32_t evtCliGetIpFromFqdn(SHashObj *pTable, char *fqdn, uint32_t *ip) { + int32_t code = 0; + uint32_t addr = 0; + size_t len = strlen(fqdn); + uint32_t *v = taosHashGet(pTable, fqdn, len); + if (v == NULL) { + code = taosGetIpv4FromFqdn(fqdn, &addr); + if (code != 0) { + code = TSDB_CODE_RPC_FQDN_ERROR; + tError("failed to get ip from fqdn:%s since %s", fqdn, tstrerror(code)); + return code; + } + + if ((code = taosHashPut(pTable, fqdn, len, &addr, sizeof(addr)) != 0)) { + return code; + } + *ip = addr; + } else { + *ip = *v; + } + return 0; +} static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { int32_t code = 0; int32_t line = 0; STrans *pInst = pThrd->pInst; SCliConn *pConn = NULL; + uint32_t ipAddr; char addr[TSDB_FQDN_LEN + 64] = {0}; snprintf(addr, sizeof(addr), "%s:%d", ip, port); @@ -2010,7 +2038,10 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon transQueueInit(&pConn->reqsToSend, NULL); transQueueInit(&pConn->reqsSentOut, NULL); - TAOS_CHECK_GOTO(createSocket(ip, port, &pConn->fd), &line, _end); + + TAOS_CHECK_GOTO(evtCliGetIpFromFqdn(pThrd->fqdn2ipCache, ip, &ipAddr), &line, _end); + + TAOS_CHECK_GOTO(createSocket(ipAddr, port, &pConn->fd), &line, _end); TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); pConn->connnected = 1; @@ -2019,6 +2050,8 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon pConn->list = taosHashGet(pThrd->pool, addr, strlen(addr)); pConn->list->totalSize += 1; + tDebug("%s succ to create conn %p src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + *ppConn = pConn; return code; @@ -3301,7 +3334,7 @@ static FORCE_INLINE void evtLogConnMissHit(SCliConn *pConn) { pConn->heapMissHit++; tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), - pThrd->connLimit); + pThrd->shareConnLimit); } static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { int32_t code = 0; @@ -3339,10 +3372,10 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { int32_t stateNum = taosHashGetSize(pConn->pQTable); int32_t totalReqs = reqsNum + reqsSentOut; - tDebug("get conn %p from heap cache for key:%s, status:%d, refCnt:%d, totalReqs:%d", pConn, key, pConn->inHeap, - pConn->reqRefCnt, totalReqs); + tDebug("%s conn %p get from heap cache for key:%s, status:%d, refCnt:%d, totalReqs:%d", pInst->label, pConn, key, + pConn->inHeap, pConn->reqRefCnt, totalReqs); - if (totalReqs >= pThrd->connLimit) { + if (totalReqs >= pThrd->shareConnLimit) { evtLogConnMissHit(pConn); if (pConn->list == NULL && pConn->dstAddr != NULL) { @@ -3351,9 +3384,9 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { tTrace("conn %p get list %p from pool for key:%s", pConn, pConn->list, key); } } - if (pConn->list && pConn->list->totalSize >= pThrd->connLimit / 4) { + if (pConn->list && pConn->list->totalSize >= pInst->connLimitNum / 4) { tWarn("%s conn %p try to remove timeout msg since too many conn created", transLabel(pInst), pConn); - pThrd->connLimit = pThrd->connLimit * 4; + pThrd->shareConnLimit = pThrd->shareConnLimit * 2; // TODO // if (cliConnRemoveTimeoutMsg(pConn)) { // tWarn("%s conn %p succ to remove timeout msg", transLabel(pInst), pConn); @@ -3450,7 +3483,7 @@ static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SC if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { int32_t curReqs = REQS_ON_CONN(pConn); int32_t topReqs = REQS_ON_CONN(pTopConn); - if (curReqs > topReqs && topReqs < pThrd->connLimit) { + if (curReqs > topReqs && topReqs < pThrd->shareConnLimit) { *pNewConn = pTopConn; return 1; } @@ -3511,7 +3544,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { } else { transQueuePush(&pConn->reqsToSend, &req->q); STraceId *trace = &req->msg.info.traceId; - tGDebug("%s success to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tGDebug("%s success to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pConn, @@ -3647,7 +3680,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads pInst->connLimitNum = 512; } if (pInst->shareConnLimit <= 0) { - pInst->shareConnLimit = 32; + pInst->shareConnLimit = 64; } memcpy(cli->label, label, TSDB_LABEL_LEN); @@ -3662,7 +3695,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads code = createThrdObj(arg, &pThrd); TAOS_CHECK_EXIT(code); pThrd->pInst = pInst; - pThrd->connLimit = pInst->connLimitNum; + pThrd->shareConnLimit = pInst->shareConnLimit; code = evtInitPipe(pThrd->pipe_queue_fd); TAOS_CHECK_EXIT(code); @@ -3793,7 +3826,7 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { } SCliThrd2 *pThrd = p->hostThrd; STrans *pInst = pThrd->pInst; - int32_t balanceLimit = pThrd->connLimit >= 4 ? pThrd->connLimit / 2 : 2; + int32_t balanceLimit = pThrd->shareConnLimit >= 4 ? pThrd->shareConnLimit / 2 : 2; SCliConn *topConn = NULL; int32_t code = transHeapGet(heap, &topConn); From 8d190b2c53505a29dd86392913b816a5023df818 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 30 Dec 2024 13:47:48 +0800 Subject: [PATCH 60/76] refactor code --- source/libs/transport/src/transImpl2.c | 72 +++++++++++++++++++++----- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index d6dee2c937a5..c5fab1bda105 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -390,7 +390,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { tError("%s failed to init buf since %s", pOpt->label, tstrerror(code)); return code; } - + tDebug("%s handle read on fd:%d", pOpt->label, pArg->fd); nBytes = recv(pArg->fd, pBuf->buf, pBuf->len, 0); if (nBytes > 0) { code = pArg->readCb(pArg, pBuf, nBytes); @@ -407,6 +407,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { tError("%s failed to init wbuf since %s", pOpt->label, tstrerror(code)); return code; } + tDebug("%s handle write on fd:%d", pOpt->label, pArg->fd); code = pArg->sendCb(pArg, pBuf, 0); if (code != 0) { @@ -429,6 +430,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { } while (total > 0); evtBufClear(&pArg->sendBuf); + tDebug("%s handle write finish on fd:%d", pOpt->label, pArg->fd); pArg->sendFinishCb(pArg, code); if (code != 0) { tError("%s failed to send buf since %s", pOpt->label, tstrerror(code)); @@ -469,6 +471,11 @@ static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { tv->tv_sec = 30; tv->tv_usec = 0; } + + if (tv->tv_sec <= 0) { + if (tv->tv_usec <= 50 * 1000) tv->tv_usec = 50 * 1000; + } + tDebug("%s next timeout %ld, sec:%d, usec:%d", pOpt->label, timeout, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec); return code; } static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { @@ -944,6 +951,47 @@ static int32_t evtSvrHandleReleaseReq(SSvrConn *pConn, STransMsgHead *pHead) { return 0; } + +static void evtSvrPerfLog_receive(SSvrConn *pConn, STransMsgHead *pHead, STransMsg *pTransMsg) { + if (!(rpcDebugFlag & DEBUG_DEBUG)) { + return; + } + + STrans *pInst = pConn->pInst; + STraceId *trace = &pHead->traceId; + + int64_t cost = taosGetTimestampUs() - taosNtoh64(pHead->timestamp); + static int64_t EXCEPTION_LIMIT_US = 100 * 1000; + + if (pConn->status == ConnNormal && pHead->noResp == 0) { + if (cost >= EXCEPTION_LIMIT_US) { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception, seqNum:%" PRId64 + ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + (int)cost, pTransMsg->info.seq, pTransMsg->info.qId); + } else { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + (int)cost, pTransMsg->info.seq, pTransMsg->info.qId); + } + } else { + if (cost >= EXCEPTION_LIMIT_US) { + tGDebug( + "%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception, " + "seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + pHead->noResp, pTransMsg->code, (int)(cost), pTransMsg->info.seq, pTransMsg->info.qId); + } else { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, seqNum:%" PRId64 + ", " + "sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + pHead->noResp, pTransMsg->code, (int)(cost), pTransMsg->info.seq, pTransMsg->info.qId); + } + } + tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pInst), pTransMsg->info.handle, pConn, + pConn->refId); +} static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { SWorkThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; @@ -992,6 +1040,8 @@ static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { transMsg.info.qId = taosHton64(pHead->qid); transMsg.info.msgType = pHead->msgType; + evtSvrPerfLog_receive(pConn, pHead, &transMsg); + SRpcConnInfo *pConnInfo = &(transMsg.info.conn); pConnInfo->clientIp = pConn->clientIp; pConnInfo->clientPort = pConn->port; @@ -1155,7 +1205,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { taosMemoryFree(pArg); continue; } else { - tDebug("%s success to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tDebug("%s succ to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); } SFdCbArg arg = {.evtType = EVT_CONN_T, @@ -1332,13 +1382,6 @@ void *transWorkerThread(void *arg) { while (!pThrd->quit) { struct timeval tv = {30, 0}; - code = evtCaclNextTimeout(pOpt, &tv); - if (code != 0) { - tError("%s failed to cacl next timeout since %s", pInst->label, tstrerror(code)); - } else { - tDebug("%s succ to cacl next timeout", pInst->label); - } - code = evtMgtDispath(pOpt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); @@ -1535,6 +1578,10 @@ int32_t transSendResponse2(STransMsg *msg) { tTrace("no need send resp"); return 0; } + + STraceId *trace = &msg->info.traceId; + tGDebug("start to send resp %p", msg); + SExHandle *exh = msg->info.handle; if (exh == NULL) { @@ -1562,7 +1609,7 @@ int32_t transSendResponse2(STransMsg *msg) { m->type = Normal; - STraceId *trace = (STraceId *)&msg->info.traceId; + // STraceId *trace = (STraceId *)&msg->info.traceId; tGDebug("conn %p start to send resp (1/2)", exh->handle); if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { destroySmsg(m); @@ -1574,12 +1621,12 @@ int32_t transSendResponse2(STransMsg *msg) { return 0; _return1: - tDebug("handle %p failed to send resp", exh); + tGDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(msg->info.refIdMgt, refId); return code; _return2: - tDebug("handle %p failed to send resp", exh); + tGDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); return code; } @@ -2005,6 +2052,7 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon code = getConnFromPool(pThrd, addr, &pConn); if (pConn != NULL) { *ppConn = pConn; + addConnToHeapCache(pThrd->connHeapCache, pConn); return 0; } if (code == TSDB_CODE_RPC_MAX_SESSIONS) { From ff034b05a130b70c2e3d2068bf918a32de0f1ed0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 30 Dec 2024 17:00:53 +0800 Subject: [PATCH 61/76] refactor code --- source/libs/transport/src/transImpl2.c | 32 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index c5fab1bda105..dcc5033e7dc3 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -294,7 +294,7 @@ static int32_t evtMgtCreate(SEvtMgt **pOpt, char *label) { pRes->fdIdx = 0; memcpy(pRes->label, label, strlen(label)); - code = evtMgtResize(pRes, (32 + 1)); + code = evtMgtResize(pRes, 32); if (code != 0) { evtMgtDestroy(pRes); } else { @@ -368,18 +368,21 @@ static int32_t evtBufDestroy(SEvtBuf *evtBuf) { } int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { int32_t nBytes = 0; - char buf[2] = {0}; + char buf[512] = {0}; int32_t code = 0; if (pArg->evtType == EVT_NEW_CONN_T || pArg->evtType == EVT_ASYNC_T) { // handle new coming conn; if (res & EVT_READ) { SAsyncHandle *handle = pArg->arg; - nBytes = read(pArg->fd, buf, 2); - if (nBytes >= 1) { + + nBytes = read(pArg->fd, buf, sizeof(buf)); + if (nBytes == 1 && buf[0] == '1') { handle->cb(handle, 0); } + tDebug("%s handle async read on fd:%d", pOpt->label, pArg->fd); } if (res & EVT_WRITE) { + tDebug("%s handle async write on fd:%d", pOpt->label, pArg->fd); // handle err } } else if (pArg->evtType == EVT_CONN_T) { @@ -514,12 +517,15 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { nfds = pOpt->evtFds + 1; // TODO lock or not + tDebug("%s start to select, timeout:%d s: %d ms", pOpt->label, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec / 1000); active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); } else if (active_Fds == 0) { tDebug("%s select timeout occurred", pOpt->label); return code; + } else { + tDebug("%s select count %d", pOpt->label, active_Fds); } for (j = 0; j < nfds && active_Fds > 0; j++) { @@ -531,6 +537,9 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if ((fd > 0) && FD_ISSET(fd, pOpt->evtWriteSetOut)) { res |= EVT_WRITE; } + if (fd > 0) { + tDebug("%s start to handle fd %d", pOpt->label, fd); + } if (res == 0) { continue; @@ -540,7 +549,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (code != 0) { tError("%s failed to handle fd %d since %s", pOpt->label, fd, tstrerror(code)); } else { - tDebug("%s success to handle fd %d", pOpt->label, fd); + tDebug("%s succ to handle fd %d", pOpt->label, fd); } } } @@ -628,6 +637,10 @@ static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *ar FD_SET(fd, pOpt->evtWriteSetIn); rwRef++; } + arg->rwRef = rwRef; + if (arg->rwRef >= 2) { + arg->rwRef = 2; + } pOpt->fd[pOpt->fdIdx++] = fd; code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); } @@ -708,12 +721,12 @@ static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, SFdCbArg arg = {.evtType = evtType, .arg = pAsync, .fd = fd[0]}; arg.arg = pAsync; - code = evtMgtAdd(pOpt, fd[0], EVT_READ, &arg); if (code != 0) { taosMemoryFree(pAsync); return code; } + tDebug("%s succ to init async on fd:%d", pOpt->label, fd[0]); QUEUE_INIT(&pAsync->q); *async = pAsync; return code; @@ -1171,6 +1184,7 @@ static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { if (transQueueEmpty(&pConn->resps)) { // stop write evt + tDebug("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } return code; @@ -1379,9 +1393,10 @@ void *transWorkerThread(void *arg) { tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &line, _end); } - + int32_t count = 0; while (!pThrd->quit) { struct timeval tv = {30, 0}; + tDebug("%s-------------------- dispatch count:%d -----------------------------", pInst->label, count++); code = evtMgtDispath(pOpt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); @@ -1884,7 +1899,7 @@ static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { return code; _end: if (code != 0) { - tError("%s failed to connect to %s:%d at line %d since %s", __func__, ip, port, line, tstrerror(code)); + tError("%s failed to connect to %d:%d at line %d since %s", __func__, ip, port, line, tstrerror(code)); } return code; } @@ -3314,6 +3329,7 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { tDebug("%s success to send out request", pInst->label); } if (transQueueEmpty(&pConn->reqsToSend)) { + tDebug("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } From 4cb784cf335cbd5126348f57e9a023a89ba59c2d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 08:59:15 +0800 Subject: [PATCH 62/76] refactor code --- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/src/trans.c | 2 +- source/libs/transport/src/transComm.c | 2 +- source/libs/transport/src/transImpl2.c | 113 +++++++++++++++++++------ 4 files changed, 91 insertions(+), 31 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2244236be1de..38b701132814 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -380,16 +380,17 @@ int32_t transReleaseCliHandle(void* handle); int32_t transReleaseSrvHandle(void* handle); #endif -void transSockInfo2Str(struct sockaddr* sockname, char* dst); -int32_t transAllocHandle(int64_t* refId); #ifndef TD_ACORE +void transSockInfo2Str(struct sockaddr* sockname, char* dst); +int32_t transAllocHandle(int64_t* refId); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void transCloseClient(void* arg); void transCloseServer(void* arg); #else +int32_t transAllocHandle2(int64_t* refId); void* transInitClient2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* transInitServer2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void transCloseClient2(void* arg); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index a95cf9e82c2c..b78909ab1460 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -522,7 +522,7 @@ int32_t rpcSetIpWhite(void* thandle, void* arg) { #endif } -int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle(refId); } +int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle2(refId); } int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c74c69428b73..b8cfaef4ef62 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -1197,7 +1197,7 @@ void transCtxInit(STransCtx* ctx) { ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); } void transCtxCleanup(STransCtx* ctx) { - if (ctx->args == NULL) { + if (ctx == NULL || ctx->args == NULL) { return; } diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index dcc5033e7dc3..1281f4934d8c 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1163,6 +1163,7 @@ static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { if (j++ > batchLimit) { break; } + destroySmsg(pResp); } return code; } @@ -2411,7 +2412,7 @@ int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); } - TAOS_CHECK_GOTO(transAllocHandle(transpointId), NULL, _exception); + TAOS_CHECK_GOTO(transAllocHandle2(transpointId), NULL, _exception); SCliThrd2 *pThrd = transGetWorkThrd(pInst, *transpointId); if (pThrd == NULL) { @@ -3134,6 +3135,26 @@ int32_t cliHandleStateMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { } return 0; } +int32_t cliHandleStateMayUpdateState(SCliConn *pConn, SCliReq *pReq) { + SCliThrd2 *pThrd = pConn->hostThrd; + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + if (qid == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + + SReqState state = {.conn = pConn, .arg = NULL}; + code = taosHashPut(pThrd->pIdConnTable, &qid, sizeof(qid), &state, sizeof(state)); + if (code != 0) { + tDebug("%s conn %p failed to statue, sid:%" PRId64 " since %s", transLabel(pThrd->pInst), pConn, qid, + tstrerror(code)); + } else { + tDebug("%s conn %p succ to add statue, sid:%" PRId64 " (1)", transLabel(pThrd->pInst), pConn, qid); + } + + TAOS_UNUSED(cliHandleStateMayUpdateStateCtx(pConn, pReq)); + return code; +} static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { int32_t code = 0; @@ -3157,6 +3178,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { if (evtCliRecycleConn(pConn)) { return code; } + return 0; } code = cliGetReqBySeq(pConn, seq, pHead->msgType, &pReq); @@ -3375,7 +3397,7 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { p->len -= msgLen; code = evtCliHandleResp(pConn, pMsg, msgLen); - TAOS_CHECK_GOTO(terrno, &line, _end); + TAOS_CHECK_GOTO(code, &line, _end); } else { break; } @@ -3590,40 +3612,81 @@ static int32_t evtCliHandleExcept(void *thrd, SCliReq *pReq, STransMsg *pResp) { destroyReq(pReq); return code; } + +static int32_t evtCliMayGetStateByQid(SCliThrd2 *pThrd, SCliReq *pReq, SCliConn **pConn) { + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + if (qid == 0) { + return TSDB_CODE_RPC_NO_STATE; + } else { + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), qid); + if (exh == NULL) { + return TSDB_CODE_RPC_STATE_DROPED; + } + + SReqState *pState = taosHashGet(pThrd->pIdConnTable, &qid, sizeof(qid)); + + if (pState == NULL) { + if (pReq->ctx == NULL) { + transReleaseExHandle(transGetRefMgt(), qid); + return TSDB_CODE_RPC_STATE_DROPED; + } + tDebug("%s conn %p failed to get statue, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + transReleaseExHandle(transGetRefMgt(), qid); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } else { + *pConn = pState->conn; + tDebug("%s conn %p succ to get conn of statue, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } + transReleaseExHandle(transGetRefMgt(), qid); + return 0; + } +} static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { int32_t lino = 0; int32_t code = 0; STransMsg resp = {0}; STrans *pInst = pThrd->pInst; - char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); - int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); - char addr[TSDB_FQDN_LEN + 64] = {0}; - snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); + STraceId *trace = &req->msg.info.traceId; SCliConn *pConn = NULL; - code = getOrCreateConn(pThrd, fqdn, port, &pConn); - if (code != 0) { - tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _end); - } else { - transQueuePush(&pConn->reqsToSend, &req->q); - STraceId *trace = &req->msg.info.traceId; - tGDebug("%s success to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); - SFdCbArg arg = {.evtType = EVT_CONN_T, - .arg = pConn, - .fd = pConn->fd, - .readCb = evtCliReadResp, - .sendCb = evtCliPreSendReq, - .sendFinishCb = evtCliSendCb, - .data = pConn}; + code = evtCliMayGetStateByQid(pThrd, req, &pConn); + if (code == 0) { + cliHandleStateMayUpdateStateCtx(pConn, req); + } else if (code == TSDB_CODE_RPC_STATE_DROPED) { + TAOS_CHECK_GOTO(code, &lino, _end); + } else if (code == TSDB_CODE_RPC_NO_STATE || code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); + int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); - code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); + code = getOrCreateConn(pThrd, fqdn, port, &pConn); + if (code != 0) { + tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); + TAOS_CHECK_GOTO(code, &lino, _end); + } else { + } + cliHandleStateMayUpdateState(pConn, req); } + + transQueuePush(&pConn->reqsToSend, &req->q); + tGDebug("%s success to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + + SFdCbArg arg = {.evtType = EVT_CONN_T, + .arg = pConn, + .fd = pConn->fd, + .readCb = evtCliReadResp, + .sendCb = evtCliPreSendReq, + .sendFinishCb = evtCliSendCb, + .data = pConn}; + + code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); + return code; _end: resp.code = code; - STraceId *trace = &req->msg.info.traceId; if (code != 0) { tGError("%s failed to handle req since %s", pInst->label, tstrerror(code)); } @@ -3656,10 +3719,6 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { if (pReq == NULL) { continue; } - STraceId *trace = &pReq->msg.info.traceId; - tGDebug("%s handle request at thread:%08" PRId64 ", dst:%s:%d, app:%p", pInst->label, pThrd->pid, - pReq->ctx->epSet->eps[0].fqdn, pReq->ctx->epSet->eps[0].port, pReq->msg.info.ahandle); - code = evtHandleCliReq(pThrd, pReq); } } From b3971fb73bb649115724411535e5c053b13094e5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 09:50:35 +0800 Subject: [PATCH 63/76] refactor code --- source/libs/transport/src/transImpl2.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 1281f4934d8c..5dad25d142d2 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -379,10 +379,10 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { if (nBytes == 1 && buf[0] == '1') { handle->cb(handle, 0); } - tDebug("%s handle async read on fd:%d", pOpt->label, pArg->fd); + tTrace("%s handle async read on fd:%d", pOpt->label, pArg->fd); } if (res & EVT_WRITE) { - tDebug("%s handle async write on fd:%d", pOpt->label, pArg->fd); + tTrace("%s handle async write on fd:%d", pOpt->label, pArg->fd); // handle err } } else if (pArg->evtType == EVT_CONN_T) { @@ -393,7 +393,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { tError("%s failed to init buf since %s", pOpt->label, tstrerror(code)); return code; } - tDebug("%s handle read on fd:%d", pOpt->label, pArg->fd); + tTrace("%s handle read on fd:%d", pOpt->label, pArg->fd); nBytes = recv(pArg->fd, pBuf->buf, pBuf->len, 0); if (nBytes > 0) { code = pArg->readCb(pArg, pBuf, nBytes); @@ -410,7 +410,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { tError("%s failed to init wbuf since %s", pOpt->label, tstrerror(code)); return code; } - tDebug("%s handle write on fd:%d", pOpt->label, pArg->fd); + tTrace("%s handle write on fd:%d", pOpt->label, pArg->fd); code = pArg->sendCb(pArg, pBuf, 0); if (code != 0) { @@ -433,12 +433,12 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { } while (total > 0); evtBufClear(&pArg->sendBuf); - tDebug("%s handle write finish on fd:%d", pOpt->label, pArg->fd); + tTrace("%s handle write finish on fd:%d", pOpt->label, pArg->fd); pArg->sendFinishCb(pArg, code); if (code != 0) { tError("%s failed to send buf since %s", pOpt->label, tstrerror(code)); } else { - tDebug("%s succ to send buf", pOpt->label); + tTrace("%s succ to send buf", pOpt->label); } return code; } @@ -478,7 +478,7 @@ static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { if (tv->tv_sec <= 0) { if (tv->tv_usec <= 50 * 1000) tv->tv_usec = 50 * 1000; } - tDebug("%s next timeout %ld, sec:%d, usec:%d", pOpt->label, timeout, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec); + tTrace("%s next timeout %ld, sec:%d, usec:%d", pOpt->label, timeout, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec); return code; } static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { @@ -489,7 +489,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (code != 0) { tError("%s failed to handle timeout since %s", pOpt->label, tstrerror(code)); } else { - // tInfo("%s succ to handle timeout", pOpt->label); + tTrace("%s succ to handle timeout", pOpt->label); } } @@ -517,15 +517,15 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { nfds = pOpt->evtFds + 1; // TODO lock or not - tDebug("%s start to select, timeout:%d s: %d ms", pOpt->label, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec / 1000); + tTrace("%s start to select, timeout:%d s: %d ms", pOpt->label, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec / 1000); active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); if (active_Fds < 0) { return TAOS_SYSTEM_ERROR(errno); } else if (active_Fds == 0) { - tDebug("%s select timeout occurred", pOpt->label); + tTrace("%s select timeout occurred", pOpt->label); return code; } else { - tDebug("%s select count %d", pOpt->label, active_Fds); + tTrace("%s select count %d", pOpt->label, active_Fds); } for (j = 0; j < nfds && active_Fds > 0; j++) { @@ -538,7 +538,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { res |= EVT_WRITE; } if (fd > 0) { - tDebug("%s start to handle fd %d", pOpt->label, fd); + tTrace("%s start to handle fd %d", pOpt->label, fd); } if (res == 0) { @@ -549,7 +549,7 @@ static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { if (code != 0) { tError("%s failed to handle fd %d since %s", pOpt->label, fd, tstrerror(code)); } else { - tDebug("%s succ to handle fd %d", pOpt->label, fd); + tTrace("%s succ to handle fd %d", pOpt->label, fd); } } } From 6ca5dccab3ce4a8a3bfd52b27472af7c6f79fba1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 09:57:38 +0800 Subject: [PATCH 64/76] refactor code --- source/libs/transport/src/transImpl2.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 5dad25d142d2..19c94d120f36 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -726,7 +726,7 @@ static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, taosMemoryFree(pAsync); return code; } - tDebug("%s succ to init async on fd:%d", pOpt->label, fd[0]); + tTrace("%s succ to init async on fd:%d", pOpt->label, fd[0]); QUEUE_INIT(&pAsync->q); *async = pAsync; return code; @@ -1185,7 +1185,7 @@ static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { if (transQueueEmpty(&pConn->resps)) { // stop write evt - tDebug("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); + tTrace("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); } return code; @@ -1220,7 +1220,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { taosMemoryFree(pArg); continue; } else { - tDebug("%s succ to create conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tDebug("%s succ to create svr conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->dst, pConn->src); } SFdCbArg arg = {.evtType = EVT_CONN_T, @@ -1299,7 +1299,7 @@ static void evtSvrHandleRegiter(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { tError("failed to put qid to hash since %s", tstrerror(code)); return; } - tDebug("conn %p success to register sid:%" PRId64 "", pConn, arg.msg.info.qId); + tDebug("conn %p succ to register sid:%" PRId64 "", pConn, arg.msg.info.qId); taosMemoryFree(pResp); } static void evtSvrHandleQuit(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { @@ -1397,7 +1397,7 @@ void *transWorkerThread(void *arg) { int32_t count = 0; while (!pThrd->quit) { struct timeval tv = {30, 0}; - tDebug("%s-------------------- dispatch count:%d -----------------------------", pInst->label, count++); + tTrace("%s-------------------- dispatch count:%d -----------------------------", pInst->label, count++); code = evtMgtDispath(pOpt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); @@ -2114,7 +2114,7 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon pConn->list = taosHashGet(pThrd->pool, addr, strlen(addr)); pConn->list->totalSize += 1; - tDebug("%s succ to create conn %p src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tDebug("%s succ to create cli conn %p src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); *ppConn = pConn; return code; @@ -3348,7 +3348,7 @@ static int32_t evtCliSendCb(void *arg, int32_t status) { tError("failed to send request since %s", tstrerror(code)); return code; } else { - tDebug("%s success to send out request", pInst->label); + tDebug("%s succ to send out request", pInst->label); } if (transQueueEmpty(&pConn->reqsToSend)) { tDebug("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); @@ -3402,7 +3402,7 @@ static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { break; } } - tDebug("%s conn %p success to read resp", pInst->label, pConn); + tDebug("%s conn %p succ to read resp", pInst->label, pConn); return code; _end: if (code != 0) { @@ -3672,7 +3672,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { } transQueuePush(&pConn->reqsToSend, &req->q); - tGDebug("%s success to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tGDebug("%s succ to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pConn, @@ -3770,15 +3770,16 @@ static void *cliWorkThread2(void *arg) { struct timeval tv = {30, 0}; code = evtCaclNextTimeout(pThrd->pEvtMgt, &tv); if (code != 0) { + tTrace("%s failed to calc next timeout", pInst->label); } else { - tDebug("%s success to calc next timeout", pInst->label); + tTrace("%s succ to calc next timeout", pInst->label); } code = evtMgtDispath(pThrd->pEvtMgt, &tv); if (code != 0) { tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); continue; } else { - tDebug("%s success to dispatch", pInst->label); + tTrace("%s succe to dispatch", pInst->label); } } From b2411b991a212873fad53ed4ff648f200acaef2e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 13:46:28 +0800 Subject: [PATCH 65/76] refactor code --- source/libs/transport/src/transImpl2.c | 78 +++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 19c94d120f36..e111ff767507 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -48,6 +48,7 @@ typedef struct { queue q; TdThreadMutex mutex; void *hostThrd; + int8_t stop; } SAsyncHandle; typedef struct { queue q; @@ -1858,6 +1859,9 @@ typedef struct { int64_t lastConnFailTs; } SHeap; +static FORCE_INLINE void destroyReq(void *arg); +static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req); + static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); static void transHeapDestroy(SHeap *heap); @@ -1876,6 +1880,41 @@ static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn); static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key); static int32_t addConnToHeapCache(SHashObj *pConnHeap, SCliConn *pConn); +static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleQuit(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleRelease(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleRegiter(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd); +static void (*transCliAsyncFuncs[])(SCliReq *, SCliThrd2 *) = {evtCliHandlReq, evtCliHandleQuit, evtCliHandleRelease, + evtCliHandleRelease, evtCliHandleUpdate}; + +static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd) { + int32_t code = evtHandleCliReq(pThrd, pReq); + if (code != 0) { + tDebug("failed to handle req"); + } +} +static void evtCliHandleQuit(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv quit, set quit flag"); + pThrd->quit = 1; + destroyReq(pReq); + return; +} +static void evtCliHandleRelease(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv release, do nothing"); + destroyReq(pReq); + return; +} +static void evtCliHandleRegiter(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv register, do nothing"); + destroyReq(pReq); + return; +} +static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv update, do nothing"); + destroyReq(pReq); + return; +} static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { int32_t code = 0; int32_t line = 0; @@ -3719,7 +3758,11 @@ static void evtCliHandleAsyncCb(void *arg, int32_t status) { if (pReq == NULL) { continue; } - code = evtHandleCliReq(pThrd, pReq); + if (pReq->type == Quit) { + transCliAsyncFuncs[pReq->type](pReq, pThrd); + break; + } + transCliAsyncFuncs[pReq->type](pReq, pThrd); } } @@ -3846,8 +3889,39 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads terrno = code; return NULL; } +static int32_t transSendQuit(SCliThrd2 *pThrd) { + if (pThrd->thrdInited == 0) { + return 0; + } + int32_t code = 0; + SCliReq *msg = taosMemoryCalloc(1, sizeof(SCliReq)); + if (msg == NULL) { + return terrno; + } + + msg->type = Quit; + if ((code = evtAsyncSend(pThrd->asyncHandle, &msg->q)) != 0) { + code = (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + taosMemoryFree(msg); + return code; + } + + atomic_store_8(&pThrd->asyncHandle->stop, 1); + return 0; +} void transCloseClient2(void *arg) { - int32_t code = 0; + int32_t code = 0; + SCliObj2 *cli = arg; + for (int i = 0; i < cli->numOfThreads; i++) { + code = transSendQuit(cli->pThreadObj[i]); + if (code != 0) { + tError("failed to send quit to thread:%d since %s", i, tstrerror(code)); + } + + destroyThrdObj(cli->pThreadObj[i]); + } + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); return; } From 0059728b342ef351b4b754efacacd2333013d86d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 15:01:27 +0800 Subject: [PATCH 66/76] refactor code --- source/libs/transport/src/transImpl2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index e111ff767507..f32e5509cb77 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -3711,7 +3711,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { } transQueuePush(&pConn->reqsToSend, &req->q); - tGDebug("%s succ to get conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + tGDebug("%s conn %p get from pool, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); SFdCbArg arg = {.evtType = EVT_CONN_T, .arg = pConn, From eb52ae388d6621fe51befbb7b63620e73e902495 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 15:07:34 +0800 Subject: [PATCH 67/76] refactor code --- source/libs/transport/src/transImpl2.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index f32e5509cb77..67fc7d93e647 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -36,7 +36,6 @@ #define EVT_SIGNAL 0x08 #define EVT_ASYNC 0x10 -/* How many bytes to allocate for N fds? */ #define SELECT_ALLOC_SIZE(n) (DIV_ROUNDUP(n, NFDBITS) * sizeof(fd_mask)) typedef void (*AsyncCb)(void *async, int32_t status); From 30dffc7ce8b984fbcadcb329e600578e6a604675 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Dec 2024 15:14:50 +0800 Subject: [PATCH 68/76] refactor code --- source/libs/transport/src/transImpl2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 67fc7d93e647..c07654331c12 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1951,7 +1951,7 @@ static int32_t cliConnGetSockInfo(SCliConn *pConn) { char ip_str[INET_ADDRSTRLEN]; int port; - // 获取对端地址 + // get peer address if (getpeername(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); } @@ -1959,7 +1959,7 @@ static int32_t cliConnGetSockInfo(SCliConn *pConn) { port = ntohs(addr.sin_port); snprintf(pConn->dst, sizeof(pConn->dst), "%s:%d", ip_str, port); - // 获取本地地址 + // get local address if (getsockname(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); } From 35bc68f9f1bcdd2eb7a4ee5e2723db5fefb34d0f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Jan 2025 14:51:46 +0800 Subject: [PATCH 69/76] add timeout --- source/libs/transport/src/transImpl2.c | 286 +++++++++++++++++++++++-- 1 file changed, 273 insertions(+), 13 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index c07654331c12..2a3e78844549 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -462,7 +462,7 @@ static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { code = pOpt->caclTimeout(pOpt->arg, &timeout); if (timeout == 0) { tv->tv_sec = 0; - tv->tv_usec = 100 * 1000; + tv->tv_usec = 10 * 1000; } else if (timeout < 0) { tv->tv_sec = 30; tv->tv_usec = 0; @@ -2931,7 +2931,7 @@ static bool cliIsEpsetUpdated(int32_t code, SReqCtx *pCtx) { if (code != 0) return false; return transReqEpsetIsEqual(pCtx->epSet, pCtx->origEpSet) ? false : true; } -static int32_t cliNotifyImplCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { +static int32_t evtCliNotifyImplCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { int32_t code = 0; SCliThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; @@ -2994,6 +2994,266 @@ static int32_t cliNotifyImplCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) return code; } +FORCE_INLINE bool evtCliTryUpdateEpset(SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SReqCtx *ctx = pReq->ctx; + + if ((pResp == NULL || pResp->info.hasEpSet == 0)) { + return false; + } + // rebuild resp msg + SEpSet epset; + if ((code = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset)) < 0) { + tError("failed to deserialize epset, code:%d", code); + return false; + } + SEpSet tepset; + int32_t tlen = tSerializeSEpSet(NULL, 0, &tepset); + + char *buf = NULL; + int32_t len = pResp->contLen - tlen; + if (len != 0) { + buf = rpcMallocCont(len); + if (buf == NULL) { + pResp->code = TSDB_CODE_OUT_OF_MEMORY; + return false; + } + // TODO: check buf + memcpy(buf, (char *)pResp->pCont + tlen, len); + } + rpcFreeCont(pResp->pCont); + + pResp->pCont = buf; + pResp->contLen = len; + + pResp->info.hasEpSet = 1; + + if (transCreateReqEpsetFromUserEpset(&epset, &ctx->epSet) != 0) { + return false; + } + return true; +} +static void evtCliMayResetRespCode(SCliReq *pReq, STransMsg *pResp) { + SReqCtx *pCtx = pReq->ctx; + if (pCtx->retryCode != TSDB_CODE_SUCCESS) { + int32_t code = pResp->code; + // return internal code app + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { + pResp->code = pCtx->retryCode; + } + } + + // check whole vnodes is offline on this vgroup + if (((pCtx->epSet != NULL) && pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) || pCtx->retryStep > 0) { + if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; + } + } +} +void cliRetryMayInitCtx(STrans *pInst, SCliReq *pReq) { + SReqCtx *pCtx = pReq->ctx; + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pInst->retryMinInterval; + pCtx->retryMaxInterval = pInst->retryMaxInterval; + pCtx->retryStepFactor = pInst->retryStepFactor; + pCtx->retryMaxTimeout = pInst->retryMaxTimeout; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 0; + pCtx->retryInit = 1; + pCtx->retryCode = TSDB_CODE_SUCCESS; + pReq->msg.info.handle = 0; + } +} +int32_t cliRetryIsTimeout(STrans *pInst, SCliReq *pReq) { + SReqCtx *pCtx = pReq->ctx; + if (pCtx->retryMaxTimeout != -1 && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return 1; + } + return 0; +} + +int8_t cliRetryShouldRetry(STrans *pInst, STransMsg *pResp) { + bool retry = pInst->retry != NULL ? pInst->retry(pResp->code, pResp->msgType - 1) : false; + return retry == false ? 0 : 1; +} + +void cliRetryUpdateRule(SReqCtx *pCtx, int8_t noDelay) { + if (noDelay == false) { + pCtx->epsetRetryCnt = 1; + pCtx->retryStep++; + + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; + } +} +bool cliResetEpset(SReqCtx *pCtx, STransMsg *pResp, bool hasEpSet) { + bool noDelay = true; + if (hasEpSet == false) { + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else if (pResp->contLen != 0) { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else { + if (!transCompareReqAndUserEpset(pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset1"); + transPrintEpSet((SEpSet *)pCtx->epSet); + transPrintEpSet(&epSet); + + if (transCreateReqEpsetFromUserEpset(&epSet, &pCtx->epSet) != 0) { + tDebug("failed to create req epset from user epset"); + } + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } + } + } + } else { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else { + if (!transCompareReqAndUserEpset(pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset2"); + transPrintEpSet((SEpSet *)pCtx->epSet); + transPrintEpSet(&epSet); + if (transCreateReqEpsetFromUserEpset(&epSet, &pCtx->epSet) != 0) { + tError("failed to create req epset from user epset"); + } + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } + } + } + return noDelay; +} + +static void doDelayTask(void *param) { + STaskArg *arg = param; + if (arg && arg->param1) { + SCliReq *pReq = arg->param1; + pReq->inRetry = 1; + } + evtCliHandlReq((SCliReq *)arg->param1, (SCliThrd2 *)arg->param2); + taosMemoryFree(arg); +} +static int32_t evtCliDoSched(SCliReq *pReq, SCliThrd2 *pThrd) { + int32_t code = 0; + STrans *pInst = pThrd->pInst; + SReqCtx *pCtx = pReq->ctx; + + STaskArg *arg = taosMemoryMalloc(sizeof(STaskArg)); + if (arg == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + arg->param1 = pReq; + arg->param2 = pThrd; + + SDelayTask *pTask = transDQSched(pThrd->pEvtMgt->arg, doDelayTask, arg, pCtx->retryNextInterval); + if (pTask == NULL) { + taosMemoryFree(arg); + return TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} +static bool evtCliMayRetry(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq->ctx; + int32_t code = pResp->code; + + if (pReq && pReq->msg.info.qId != 0) { + return false; + } + cliRetryMayInitCtx(pInst, pReq); + + if (!cliRetryShouldRetry(pInst, pResp)) { + return false; + } + + if (cliRetryIsTimeout(pInst, pReq)) { + return false; + } + + bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + transFreeMsg(pResp->pCont); + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING || code == TSDB_CODE_VND_STOPPED) { + tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, true); + transFreeMsg(pResp->pCont); + } else if (code == TSDB_CODE_SYN_RESTORING) { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, true); + transFreeMsg(pResp->pCont); + } else { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + transFreeMsg(pResp->pCont); + } + pResp->pCont = NULL; + pResp->info.hasEpSet = 0; + if (code != TSDB_CODE_RPC_BROKEN_LINK && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && code != TSDB_CODE_SUCCESS) { + // save one internal code + pCtx->retryCode = code; + } + + cliRetryUpdateRule(pCtx, noDelay); + + pReq->sent = 0; + pReq->seq = 0; + + code = evtCliDoSched(pReq, pThrd); + if (code != 0) { + pResp->code = code; + tError("failed to sched msg to next node since %s", tstrerror(code)); + return false; + } + return true; +} static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { int32_t code = 0; SCliThrd2 *pThrd = pConn->hostThrd; @@ -3001,17 +3261,17 @@ static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { if (pReq != NULL) { removeReqFromSendQ(pReq); - // if (pResp->code != TSDB_CODE_SUCCESS) { - // if (cliMayRetry(pConn, pReq, pResp)) { - // return TSDB_CODE_RPC_ASYNC_IN_PROCESS; - // } - // cliMayResetRespCode(pReq, pResp); - // } - // if (cliTryUpdateEpset(pReq, pResp)) { - // cliPerfLog_epset(pConn, pReq); - // } - } - return cliNotifyImplCb(pConn, pReq, pResp); + if (pResp->code != TSDB_CODE_SUCCESS) { + if (evtCliMayRetry(pConn, pReq, pResp)) { + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + evtCliMayResetRespCode(pReq, pResp); + } + if (evtCliTryUpdateEpset(pReq, pResp)) { + // cliPerfLog_epset(pConn, pReq); + } + } + return evtCliNotifyImplCb(pConn, pReq, pResp); } static int32_t evtCliBuildRespFromCont(SCliReq *pReq, STransMsg *pResp, STransMsgHead *pHead) { pResp->contLen = transContLenFromMsg(pHead->msgLen); From 2b2f78543fc36bb317d28da27fcafb05715c18b5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Jan 2025 15:20:45 +0800 Subject: [PATCH 70/76] add timeout --- source/libs/transport/src/transImpl2.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 2a3e78844549..75525362af71 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1683,7 +1683,7 @@ typedef struct SCliConn { // SCliBatch *pBatch; - // SDelayTask *task; + SDelayTask *task; HeapNode node; // for heap int8_t inHeap; @@ -1999,6 +1999,19 @@ static int32_t getOrCreateConnList(SCliThrd2 *pThrd, const char *key, SConnList } return 0; } +static void evtCliCloseIdleConn(void *param) { + STaskArg *arg = param; + SCliConn *conn = arg->param1; + SCliThrd2 *thrd = arg->param2; + tDebug("%s conn %p idle, close it", thrd->pInst->label, conn); + conn->task = NULL; + taosMemoryFree(arg); + + // int32_t ref = transUnrefCliHandle(conn); + // if (ref <= 0) { + // return; + // } +} static void addConnToPool(void *pool, SCliConn *conn) { if (conn->status == ConnInPool) { return; @@ -2029,8 +2042,7 @@ static void addConnToPool(void *pool, SCliConn *conn) { arg->param2 = thrd; STrans *pInst = thrd->pInst; - // conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, (10 * CONN_PERSIST_TIME(pInst->idleTime) / - // 3)); + conn->task = transDQSched(thrd->pEvtMgt->arg, evtCliCloseIdleConn, arg, (1000 * (pInst->idleTime))); } } @@ -2061,6 +2073,12 @@ static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppC QUEUE_INIT(&conn->q); conn->list = plist; + if (conn->task != NULL) { + SDelayTask *task = conn->task; + conn->task = NULL; + transDQCancel(((SCliThrd2 *)conn->hostThrd)->pEvtMgt->arg, task); + } + tDebug("conn %p get from pool, pool size:%d, dst:%s", conn, conn->list->size, conn->dstAddr); *ppConn = conn; From ee7cf22b5e5ddc4c5d22f5ff68fb2410e83915ab Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Jan 2025 19:23:23 +0800 Subject: [PATCH 71/76] add timeout --- source/libs/transport/src/transImpl2.c | 216 ++++++++++++++++++++++++- 1 file changed, 212 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 75525362af71..35c9e5f7d74c 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -1887,6 +1887,12 @@ static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd); static void (*transCliAsyncFuncs[])(SCliReq *, SCliThrd2 *) = {evtCliHandlReq, evtCliHandleQuit, evtCliHandleRelease, evtCliHandleRelease, evtCliHandleUpdate}; +static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq); + +static void transRefCliHandle(void *handle); +static int32_t transUnrefCliHandle(void *handle); +// static int32_t transGetRefCount(void *handle); + static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd) { int32_t code = evtHandleCliReq(pThrd, pReq); if (code != 0) { @@ -2007,10 +2013,11 @@ static void evtCliCloseIdleConn(void *param) { conn->task = NULL; taosMemoryFree(arg); - // int32_t ref = transUnrefCliHandle(conn); - // if (ref <= 0) { - // return; - // } + // taosCloseSocketNoCheck1(conn->fd); + int32_t ref = transUnrefCliHandle(conn); + if (ref <= 0) { + return; + } } static void addConnToPool(void *pool, SCliConn *conn) { if (conn->status == ConnInPool) { @@ -2191,6 +2198,207 @@ static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { taosMemoryFree(ctx); } } + +static FORCE_INLINE bool filterAllReq(void *e, void *arg) { return 1; } + +static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp); +static void notifyAndDestroyReq(SCliConn *pConn, SCliReq *pReq, int32_t code) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + STransMsg resp = {0}; + resp.code = (pConn->connnected ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL); + if (code != 0) { + resp.code = code; + } + + resp.msgType = pReq ? pReq->msg.msgType + 1 : 0; + resp.info.cliVer = pInst->compatibilityVer; + resp.info.ahandle = pCtx ? pCtx->ahandle : 0; + resp.info.handle = pReq->msg.info.handle; + if (pReq) { + resp.info.traceId = pReq->msg.info.traceId; + } + + STraceId *trace = &resp.info.traceId; + tDebug("%s conn %p notify user and destroy msg %s since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); + + // handle noresp and inter manage msg + if (pCtx == NULL || pReq->msg.info.noResp) { + tDebug("%s conn %p destroy %s msg directly since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); + destroyReq(pReq); + return; + } + + pReq->seq = 0; + code = cliNotifyCb(pConn, pReq, &resp); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + return; + } else { + // already notify user + destroyReq(pReq); + } +} + +static FORCE_INLINE void destroyReqInQueue(SCliConn *conn, queue *set, int32_t code) { + while (!QUEUE_IS_EMPTY(set)) { + queue *el = QUEUE_HEAD(set); + QUEUE_REMOVE(el); + + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + removeReqFromSendQ(pReq); + notifyAndDestroyReq(conn, pReq, code); + } +} +static FORCE_INLINE int32_t destroyAllReqs(SCliConn *conn) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + STrans *pInst = pThrd->pInst; + queue set; + QUEUE_INIT(&set); + // TODO + // 1. from qId from thread table + // 2. not itera to all reqs + transQueueRemoveByFilter(&conn->reqsSentOut, filterAllReq, NULL, &set, -1); + transQueueRemoveByFilter(&conn->reqsToSend, filterAllReq, NULL, &set, -1); + + destroyReqInQueue(conn, &set, 0); + return 0; +} +static void evtCliDestroyAllQidFromThrd(SCliConn *conn) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + STrans *pInst = pThrd->pInst; + + void *pIter = taosHashIterate(conn->pQTable, NULL); + while (pIter != NULL) { + int64_t *qid = taosHashGetKey(pIter, NULL); + + code = taosHashRemove(pThrd->pIdConnTable, qid, sizeof(*qid)); + if (code != 0) { + tDebug("%s conn %p failed to remove state %" PRId64 " since %s", pInst->label, conn, *qid, tstrerror(code)); + } else { + tDebug("%s conn %p destroy sid::%" PRId64 "", pInst->label, conn, *qid); + } + + STransCtx *ctx = pIter; + transCtxCleanup(ctx); + + transReleaseExHandle(transGetRefMgt(), *qid); + transRemoveExHandle(transGetRefMgt(), *qid); + + pIter = taosHashIterate(conn->pQTable, pIter); + } + taosHashCleanup(conn->pQTable); + conn->pQTable = NULL; +} +static void evtCliDestroy(SCliConn *pConn) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + // cliResetConnTimer(conn); + + tDebug("%s conn %p try to destroy", pInst->label, pConn); + + code = destroyAllReqs(pConn); + if (code != 0) { + tDebug("%s conn %p failed to all reqs since %s", pInst->label, pConn, tstrerror(code)); + } + + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code != 0) { + tDebug("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); + } + + taosMemoryFree(pConn->dstAddr); + // taosMemoryFree(conn->stream); + taosMemoryFree(pConn->ipStr); + // cliDestroyAllQidFromThrd(conn); + + if (pConn->pInitUserReq) { + taosMemoryFree(pConn->pInitUserReq); + pConn->pInitUserReq = NULL; + } + + // taosMemoryFree(conn->buf); + // destroyWQ(&conn->wq); + + taosCloseSocketNoCheck1(pConn->fd); + transDestroyBuffer(&pConn->readBuf); + + tTrace("%s conn %p destroy successfully", pInst->label, pConn); + + taosMemoryFree(pConn); +} +static void evtCliDestroyConn(SCliConn *pConn, bool force) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + // cliResetConnTimer(conn); + code = destroyAllReqs(pConn); + if (code != 0) { + tError("%s conn %p failed to destroy all reqs on conn since %s", pInst->label, pConn, tstrerror(code)); + } + + evtCliDestroyAllQidFromThrd(pConn); + if (pThrd->quit == false && pConn->list) { + QUEUE_REMOVE(&pConn->q); + pConn->list->totalSize -= 1; + pConn->list = NULL; + } + + if (pConn->task != NULL) { + transDQCancel(((SCliThrd2 *)pConn->hostThrd)->pEvtMgt->arg, pConn->task); + pConn->task = NULL; + } + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code != 0) { + tError("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); + } + + if (pConn->registered) { + int8_t ref = pConn->ref; + if (ref == 0) { + // uv_close((uv_handle_t *)conn->stream, cliDestroy); + } + } + return; +} +static void transRefCliHandle(void *handle) { + int32_t ref = 0; + if (handle == NULL) { + return; + } + SCliConn *conn = (SCliConn *)handle; + SCliThrd2 *thrd = conn->hostThrd; + conn->ref++; + + tTrace("%s conn %p ref %d", thrd->pInst->label, conn, conn->ref); +} +static int32_t transUnrefCliHandle(void *handle) { + if (handle == NULL) { + return 0; + } + int32_t ref = 0; + SCliConn *conn = (SCliConn *)handle; + SCliThrd2 *thrd = conn->hostThrd; + conn->ref--; + ref = conn->ref; + + tTrace("%s conn %p ref:%d", thrd->pInst->label, conn, conn->ref); + if (conn->ref == 0) { + evtCliDestroyConn(conn, false); + } + return ref; +} +// static int32_t transGet + static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq) { if (pReq == NULL || pReq->inSendQ == 0) { return; From 3c59ee200b9aa38f56ea08a0a57710f997262b7f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Jan 2025 13:46:41 +0800 Subject: [PATCH 72/76] refactor code --- include/os/osFile.h | 2 ++ source/libs/transport/src/transImpl2.c | 50 +++++++++----------------- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 1c397f304275..aad8cbf95f11 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -24,6 +24,7 @@ extern "C" { // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following sectio +#ifndef TD_ACORE #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID @@ -37,6 +38,7 @@ extern "C" { #define getline GETLINE_FUNC_TAOS_FORBID // #define fflush FFLUSH_FUNC_TAOS_FORBID #endif +#endif #ifndef PATH_MAX #define PATH_MAX 256 diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 35c9e5f7d74c..106221f3934c 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -73,12 +73,12 @@ typedef struct { void *mgt; } SDelayQueue; -int32_t transDQCreate(void *loop, SDelayQueue **queue); -void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); -SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); -void transDQCancel(SDelayQueue *queue, SDelayTask *task); -int32_t transDQHandleTimeout(SDelayQueue *queue); -int32_t transDQGetNextTimeout(SDelayQueue *queue); +static int32_t transDQCreate(void *loop, SDelayQueue **queue); +static void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); +SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); +void transDQCancel(SDelayQueue *queue, SDelayTask *task); +int32_t transDQHandleTimeout(SDelayQueue *queue); +int32_t transDQGetNextTimeout(SDelayQueue *queue); typedef struct { int notifyCount; // int init; // init or not @@ -1622,10 +1622,8 @@ int32_t transSendResponse2(STransMsg *msg) { goto _return1; } m->msg = tmsg; - m->type = Normal; - // STraceId *trace = (STraceId *)&msg->info.traceId; tGDebug("conn %p start to send resp (1/2)", exh->handle); if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { destroySmsg(m); @@ -1646,8 +1644,6 @@ int32_t transSendResponse2(STransMsg *msg) { rpcFreeCont(msg->pCont); return code; } -// int32_t transRegisterMsg(const STransMsg *msg) { return 0; } -// int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } void transCloseServer2(void *arg) { // impl later @@ -1663,10 +1659,6 @@ typedef struct SConnList { typedef struct SCliConn { int32_t ref; - // uv_connect_t connReq; - // uv_stream_t *stream; - - // uv_timer_t *timer; // read timer, forbidden void *hostThrd; @@ -1681,8 +1673,6 @@ typedef struct SCliConn { bool broken; // link broken or not ConnStatus status; // - // SCliBatch *pBatch; - SDelayTask *task; HeapNode node; // for heap @@ -1760,19 +1750,12 @@ typedef struct SCliThrd2 { TdThread thread; // tid int64_t pid; // pid - // uv_loop_t *loop; - // SAsyncPool *asyncPool; - void *pool; // conn pool - // timer handles - SArray *timerList; - // msg queue + void *pool; // conn pool + SArray *timerList; queue msg; TdThreadMutex msgMtx; - // SDelayQueue *delayQueue; - // SDelayQueue *timeoutQueue; - // SDelayQueue *waitConnQueue; - uint64_t nextTimeout; // next timeout - STrans *pInst; // + uint64_t nextTimeout; // next timeout + STrans *pInst; // void (*destroyAhandleFp)(void *ahandle); SHashObj *fqdn2ipCache; @@ -1933,10 +1916,6 @@ static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { server_addr.sin_port = htons(port); server_addr.sin_addr.s_addr = ip; - // if (inet_pton(AF_INET, ip, &server_addr.sin_addr) <= 0) { - // TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); - // } - if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); } @@ -2365,6 +2344,7 @@ static void evtCliDestroyConn(SCliConn *pConn, bool force) { if (pConn->registered) { int8_t ref = pConn->ref; if (ref == 0) { + close(pConn->fd); // uv_close((uv_handle_t *)conn->stream, cliDestroy); } } @@ -2457,6 +2437,9 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); } + pThrd->pInst = pInst; + pThrd->shareConnLimit = pInst->shareConnLimit; + *ppThrd = pThrd; return code; _end: @@ -4344,15 +4327,16 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads for (int i = 0; i < cli->numOfThreads; i++) { SCliThrd2 *pThrd = NULL; + code = createThrdObj(arg, &pThrd); TAOS_CHECK_EXIT(code); - pThrd->pInst = pInst; - pThrd->shareConnLimit = pInst->shareConnLimit; + code = evtInitPipe(pThrd->pipe_queue_fd); TAOS_CHECK_EXIT(code); code = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); TAOS_CHECK_EXIT(code); + pThrd->thrdInited = 1; cli->pThreadObj[i] = pThrd; } From 5516bc01b36aba3cbc36eac106ce084fd1cc14aa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Jan 2025 16:18:38 +0800 Subject: [PATCH 73/76] refactor code --- source/libs/transport/src/transImpl2.c | 447 +++++++++++++------------ 1 file changed, 236 insertions(+), 211 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 106221f3934c..45a6ee693b44 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -73,12 +73,13 @@ typedef struct { void *mgt; } SDelayQueue; -static int32_t transDQCreate(void *loop, SDelayQueue **queue); -static void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); -SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); -void transDQCancel(SDelayQueue *queue, SDelayTask *task); -int32_t transDQHandleTimeout(SDelayQueue *queue); -int32_t transDQGetNextTimeout(SDelayQueue *queue); +static int32_t transDQCreate2(void *loop, SDelayQueue **queue); +static void transDQDestroy2(SDelayQueue *queue, void (*freeFunc)(void *arg)); +static void transDQCancel2(SDelayQueue *queue, SDelayTask *task); +static int32_t transDQHandleTimeout2(SDelayQueue *queue); +static int32_t transDQGetNextTimeout2(SDelayQueue *queue); +static int32_t transDQSched2(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs, + SDelayTask **pTask); typedef struct { int notifyCount; // int init; // init or not @@ -455,7 +456,7 @@ int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { return evtMgtHandleImpl(pOpt, pArg, res); } -static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { +static int32_t evtCalcNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { int32_t code = 0; int64_t timeout = 0; if (pOpt->caclTimeout) { @@ -745,7 +746,7 @@ static int32_t evtAsyncSend(SAsyncHandle *async, queue *q) { return code; } -void *transAcceptThread(void *arg) { +void *evtSvrAcceptThread(void *arg) { int32_t code = 0; setThreadName("trans-accept-work"); @@ -780,11 +781,11 @@ void *transAcceptThread(void *arg) { } int32_t evtSvrGetConnRefOfThrd(SWorkThrd2 *thrd) { return thrd ? thrd->connRefMgt : -1; } -void uvDestroyResp(void *e) { +void evtSvrDestroyResp(void *e) { SSvrRespMsg *pMsg = QUEUE_DATA(e, SSvrRespMsg, q); destroySmsg(pMsg); } -static int32_t connGetSockInfo(SSvrConn *pConn) { +static int32_t evtSvrConnGetSockInfo(SSvrConn *pConn) { int32_t code = 0; struct sockaddr_in addr; @@ -812,7 +813,25 @@ static int32_t connGetSockInfo(SSvrConn *pConn) { return code; } -static SSvrConn *createConn(void *tThrd, int32_t fd) { +static void evtSvrDestroyConn(SSvrConn *pConn) { + if (pConn == NULL) { + return; + } + if (pConn->refId > 0) { + SWorkThrd2 *pThrd = pConn->hostThrd; + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); + transRemoveExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); + } + if (pConn->fd > 0) { + close(pConn->fd); + pConn->fd = -1; + } + + transDestroyBuffer(&pConn->readBuf); + taosHashCleanup(pConn->pQTable); + taosMemoryFree(pConn); +} +static SSvrConn *evtSvrCreateConn(void *tThrd, int32_t fd) { int32_t code = 0; int32_t lino = 0; SWorkThrd2 *pThrd = tThrd; @@ -820,11 +839,13 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { if (pConn == NULL) { return NULL; } + pConn->hostThrd = pThrd; + pConn->fd = fd; pConn->pInst = pThrd->pInst; QUEUE_INIT(&pConn->queue); - code = connGetSockInfo(pConn); + code = evtSvrConnGetSockInfo(pConn); TAOS_CHECK_GOTO(code, &lino, _end); if ((code = transInitBuffer(&pConn->readBuf)) != 0) { @@ -864,28 +885,17 @@ static SSvrConn *createConn(void *tThrd, int32_t fd) { QUEUE_PUSH(&pThrd->conn, &pConn->queue); transQueueInit(&pConn->resps, NULL); - pConn->hostThrd = pThrd; return pConn; _end: if (code != 0) { tError("failed to create conn since %s", tstrerror(code)); - // destroryConn(pConn); + evtSvrDestroyConn(pConn); } return NULL; } -static void destroryConn(SSvrConn *pConn) { - if (pConn == NULL) { - return; - } - // transDestroyBuffer(&pConn->readBuf); - // transQueueCleanup(&pConn->resps); - taosHashCleanup(pConn->pQTable); - QUEUE_REMOVE(&pConn->queue); - taosMemoryFree(pConn); -} static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp); -bool connMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { +bool evtSvrConnMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { if (pConn->userInited) { return false; } @@ -1012,7 +1022,7 @@ static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { int32_t code = 0; int32_t msgLen = len; STransMsgHead *pHead = (STransMsgHead *)req; - if (connMayGetUserInfo(pConn, &pHead, &msgLen) == true) { + if (evtSvrConnMayGetUserInfo(pConn, &pHead, &msgLen) == true) { tDebug("%s conn %p get user info", transLabel(pInst), pConn); } else { if (pConn->userInited == 0) { @@ -1190,7 +1200,7 @@ static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { } return code; } -void evtNewConnNotifyCb(void *async, int32_t status) { +void evtSvrNewConnNotifyCb(void *async, int32_t status) { int32_t code = 0; SAsyncHandle *handle = async; @@ -1214,7 +1224,7 @@ void evtNewConnNotifyCb(void *async, int32_t status) { SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); - SSvrConn *pConn = createConn(pEvtMgt->hostThrd, pArg->acceptFd); + SSvrConn *pConn = evtSvrCreateConn(pEvtMgt->hostThrd, pArg->acceptFd); if (pConn == NULL) { tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); taosMemoryFree(pArg); @@ -1363,7 +1373,7 @@ void evtSvrHandleAyncCb(void *async, int32_t status) { return; } -void *transWorkerThread(void *arg) { +void *evtSvrWorkerThread(void *arg) { int32_t code = 0; int32_t line = 0; @@ -1382,7 +1392,7 @@ void *transWorkerThread(void *arg) { pThrd->pEvtMgt = pOpt; pOpt->hostThrd = pThrd; - code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_NEW_CONN_T, + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtSvrNewConnNotifyCb, EVT_NEW_CONN_T, (void *)pThrd); if (code != 0) { tError("%s failed to create evt since %s", pInst->label, tstrerror(code)); @@ -1411,7 +1421,7 @@ void *transWorkerThread(void *arg) { evtMgtDestroy(pOpt); return NULL; } -static int32_t addHandleToAcceptloop(void *arg) { +static int32_t evtSvrAddHandleToAcceptLoop(void *arg) { // impl later int32_t code = 0; SServerObj2 *srv = arg; @@ -1436,7 +1446,7 @@ static int32_t addHandleToAcceptloop(void *arg) { if (listen(server_fd, 128) < 0) { return TAOS_SYSTEM_ERROR(errno); } - code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); + code = taosThreadCreate(&srv->thread, NULL, evtSvrAcceptThread, srv); return code; } @@ -1474,13 +1484,13 @@ void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads TAOS_CHECK_EXIT(evtInitPipe(thrd->pipe_queue_fd)); QUEUE_INIT(&thrd->conn); - code = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); + code = taosThreadCreate(&(thrd->thread), NULL, evtSvrWorkerThread, (void *)(thrd)); TAOS_CHECK_EXIT(code); thrd->inited = 1; thrd->quit = false; srv->pThreadObj[i] = thrd; } - code = addHandleToAcceptloop(srv); + code = evtSvrAddHandleToAcceptLoop(srv); TAOS_CHECK_EXIT(code); return NULL; _exit: @@ -1798,7 +1808,7 @@ int32_t createTimeoutArg(void *pEvtMgt, STimeoutArg **pArg) { TAOS_CHECK_GOTO(code, &line, _end); } SDelayQueue *queue = NULL; - transDQCreate(arg, &queue); + transDQCreate2(arg, &queue); arg->queue = queue; arg->pEvtMgt = pEvtMgt; @@ -1817,7 +1827,7 @@ void destroyTimeoutArg(STimeoutArg *arg) { } SDelayQueue *queue = arg->queue; - transDQDestroy(queue, NULL); + transDQDestroy2(queue, NULL); taosMemoryFree(arg); return; } @@ -1841,7 +1851,7 @@ typedef struct { int64_t lastConnFailTs; } SHeap; -static FORCE_INLINE void destroyReq(void *arg); +static FORCE_INLINE void evtCliDestroyReq(void *arg); static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req); static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); @@ -1870,7 +1880,7 @@ static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd); static void (*transCliAsyncFuncs[])(SCliReq *, SCliThrd2 *) = {evtCliHandlReq, evtCliHandleQuit, evtCliHandleRelease, evtCliHandleRelease, evtCliHandleUpdate}; -static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq); +static FORCE_INLINE void evtCliRemoveReqFromSendQ(SCliReq *pReq); static void transRefCliHandle(void *handle); static int32_t transUnrefCliHandle(void *handle); @@ -1885,25 +1895,25 @@ static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd) { static void evtCliHandleQuit(SCliReq *pReq, SCliThrd2 *pThrd) { tDebug("recv quit, set quit flag"); pThrd->quit = 1; - destroyReq(pReq); + evtCliDestroyReq(pReq); return; } static void evtCliHandleRelease(SCliReq *pReq, SCliThrd2 *pThrd) { tDebug("recv release, do nothing"); - destroyReq(pReq); + evtCliDestroyReq(pReq); return; } static void evtCliHandleRegiter(SCliReq *pReq, SCliThrd2 *pThrd) { tDebug("recv register, do nothing"); - destroyReq(pReq); + evtCliDestroyReq(pReq); return; } static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd) { tDebug("recv update, do nothing"); - destroyReq(pReq); + evtCliDestroyReq(pReq); return; } -static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { +static int32_t evtCliCreateSocket(uint32_t ip, int32_t port, int32_t *fd) { int32_t code = 0; int32_t line = 0; int32_t sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -1923,11 +1933,14 @@ static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { return code; _end: if (code != 0) { + if (sockfd > 0) { + close(sockfd); + } tError("%s failed to connect to %d:%d at line %d since %s", __func__, ip, port, line, tstrerror(code)); } return code; } -static int32_t cliConnGetSockInfo(SCliConn *pConn) { +static int32_t evtCliConnGetSockInfo(SCliConn *pConn) { int32_t code = 0; int32_t line = 0; struct sockaddr_in addr; @@ -1959,7 +1972,7 @@ static int32_t cliConnGetSockInfo(SCliConn *pConn) { return code; } -static int32_t getOrCreateConnList(SCliThrd2 *pThrd, const char *key, SConnList **ppList) { +static int32_t evtCliGetOrCreateConnList(SCliThrd2 *pThrd, const char *key, SConnList **ppList) { int32_t code = 0; void *pool = pThrd->pool; size_t klen = strlen(key); @@ -1998,7 +2011,8 @@ static void evtCliCloseIdleConn(void *param) { return; } } -static void addConnToPool(void *pool, SCliConn *conn) { +static void evtCliAddConnToPool(void *pool, SCliConn *conn) { + int32_t code = 0; if (conn->status == ConnInPool) { return; } @@ -2028,17 +2042,21 @@ static void addConnToPool(void *pool, SCliConn *conn) { arg->param2 = thrd; STrans *pInst = thrd->pInst; - conn->task = transDQSched(thrd->pEvtMgt->arg, evtCliCloseIdleConn, arg, (1000 * (pInst->idleTime))); + code = transDQSched2(thrd->pEvtMgt->arg, evtCliCloseIdleConn, arg, (1000 * (pInst->idleTime)), &conn->task); + if (code != 0) { + taosMemoryFree(arg); + tError("failed to schedule idle conn %p since %s", tstrerror(code)); + } } } -static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppConn) { +static int32_t evtCliGetConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppConn) { int32_t code = 0; void *pool = pThrd->pool; STrans *pInst = pThrd->pInst; SConnList *plist = NULL; - code = getOrCreateConnList(pThrd, key, &plist); + code = evtCliGetOrCreateConnList(pThrd, key, &plist); if (code != 0) { return code; } @@ -2062,7 +2080,7 @@ static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppC if (conn->task != NULL) { SDelayTask *task = conn->task; conn->task = NULL; - transDQCancel(((SCliThrd2 *)conn->hostThrd)->pEvtMgt->arg, task); + transDQCancel2(((SCliThrd2 *)conn->hostThrd)->pEvtMgt->arg, task); } tDebug("conn %p get from pool, pool size:%d, dst:%s", conn, conn->list->size, conn->dstAddr); @@ -2092,7 +2110,24 @@ static int32_t evtCliGetIpFromFqdn(SHashObj *pTable, char *fqdn, uint32_t *ip) { } return 0; } -static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { +static void evtCliDestroySocket(SCliConn *pConn) { + if (pConn == NULL) return; + + if (pConn->fd > 0) { + taosCloseSocketNoCheck1(pConn->fd); + pConn->fd = -1; + } + taosMemoryFreeClear(pConn->dstAddr); + taosMemoryFreeClear(pConn->ipStr); + transDestroyBuffer(&pConn->readBuf); + + if (pConn->pQTable) { + taosHashCleanup(pConn->pQTable); + } + + taosMemoryFree(pConn); +} +static int32_t evtCliCreateNewSocket(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { int32_t code = 0; int32_t line = 0; STrans *pInst = pThrd->pInst; @@ -2101,21 +2136,6 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon char addr[TSDB_FQDN_LEN + 64] = {0}; snprintf(addr, sizeof(addr), "%s:%d", ip, port); - pConn = getConnFromHeapCache(pThrd->connHeapCache, addr); - if (pConn != NULL) { - *ppConn = pConn; - return code; - } - - code = getConnFromPool(pThrd, addr, &pConn); - if (pConn != NULL) { - *ppConn = pConn; - addConnToHeapCache(pThrd->connHeapCache, pConn); - return 0; - } - if (code == TSDB_CODE_RPC_MAX_SESSIONS) { - return code; - } pConn = taosMemoryCalloc(1, sizeof(SCliConn)); if (pConn == NULL) { @@ -2147,8 +2167,8 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon TAOS_CHECK_GOTO(evtCliGetIpFromFqdn(pThrd->fqdn2ipCache, ip, &ipAddr), &line, _end); - TAOS_CHECK_GOTO(createSocket(ipAddr, port, &pConn->fd), &line, _end); - TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); + TAOS_CHECK_GOTO(evtCliCreateSocket(ipAddr, port, &pConn->fd), &line, _end); + TAOS_CHECK_GOTO(evtCliConnGetSockInfo(pConn), &line, _end); pConn->connnected = 1; addConnToHeapCache(pThrd->connHeapCache, pConn); @@ -2160,17 +2180,49 @@ static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliCon *ppConn = pConn; return code; - _end: if (code != 0) { // TODO, delete conn mem + tError("%s failed to create new conn at line %d since %s", __func__, line, tstrerror(code)); + evtCliDestroySocket(pConn); + } + return code; +} +static int32_t evtCliGetOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { + int32_t code = 0; + int32_t line = 0; + STrans *pInst = pThrd->pInst; + SCliConn *pConn = NULL; + uint32_t ipAddr; + + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", ip, port); + pConn = getConnFromHeapCache(pThrd->connHeapCache, addr); + if (pConn != NULL) { + *ppConn = pConn; + return code; + } + + code = evtCliGetConnFromPool(pThrd, addr, &pConn); + if (pConn != NULL) { + *ppConn = pConn; + addConnToHeapCache(pThrd->connHeapCache, pConn); + return 0; + } + if (code == TSDB_CODE_RPC_MAX_SESSIONS) { + return code; + } + + code = evtCliCreateNewSocket(pThrd, ip, port, &pConn); + +_end: + if (code != 0) { tError("%s failed to create conn at line %d since %s", __func__, line, tstrerror(code)); - taosMemoryFree(pConn); } return code; } -static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { +static FORCE_INLINE void evtCliDestroyReqCtx(SReqCtx *ctx) { if (ctx) { taosMemoryFree(ctx->epSet); taosMemoryFree(ctx->origEpSet); @@ -2178,61 +2230,61 @@ static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { } } -static FORCE_INLINE bool filterAllReq(void *e, void *arg) { return 1; } - +static bool filterAllReq(void *e, void *arg) { return 1; } static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp); -static void notifyAndDestroyReq(SCliConn *pConn, SCliReq *pReq, int32_t code) { - SCliThrd2 *pThrd = pConn->hostThrd; - STrans *pInst = pThrd->pInst; - SReqCtx *pCtx = pReq ? pReq->ctx : NULL; - STransMsg resp = {0}; - resp.code = (pConn->connnected ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL); - if (code != 0) { - resp.code = code; +static void evtCliNotifyAndDestroyReq(SCliConn *pConn, SCliReq *pReq, int32_t code) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + STransMsg resp = {0}; + resp.code = (pConn->connnected ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL); + if (code != 0) { + resp.code = code; } - resp.msgType = pReq ? pReq->msg.msgType + 1 : 0; - resp.info.cliVer = pInst->compatibilityVer; - resp.info.ahandle = pCtx ? pCtx->ahandle : 0; - resp.info.handle = pReq->msg.info.handle; - if (pReq) { - resp.info.traceId = pReq->msg.info.traceId; + resp.msgType = pReq ? pReq->msg.msgType + 1 : 0; + resp.info.cliVer = pInst->compatibilityVer; + resp.info.ahandle = pCtx ? pCtx->ahandle : 0; + resp.info.handle = pReq->msg.info.handle; + if (pReq) { + resp.info.traceId = pReq->msg.info.traceId; } - STraceId *trace = &resp.info.traceId; - tDebug("%s conn %p notify user and destroy msg %s since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), - tstrerror(resp.code)); + STraceId *trace = &resp.info.traceId; + tDebug("%s conn %p notify user and destroy msg %s since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); - // handle noresp and inter manage msg - if (pCtx == NULL || pReq->msg.info.noResp) { - tDebug("%s conn %p destroy %s msg directly since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), - tstrerror(resp.code)); - destroyReq(pReq); - return; + // handle noresp and inter manage msg + if (pCtx == NULL || pReq->msg.info.noResp) { + tDebug("%s conn %p destroy %s msg directly since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); + evtCliDestroyReq(pReq); + return; } - pReq->seq = 0; - code = cliNotifyCb(pConn, pReq, &resp); - if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { - return; + pReq->seq = 0; + code = cliNotifyCb(pConn, pReq, &resp); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + return; } else { - // already notify user - destroyReq(pReq); + // already notify user + evtCliDestroyReq(pReq); } } -static FORCE_INLINE void destroyReqInQueue(SCliConn *conn, queue *set, int32_t code) { +static FORCE_INLINE void evtCliDestroyReqInQueue(SCliConn *conn, queue *set, int32_t code) { while (!QUEUE_IS_EMPTY(set)) { queue *el = QUEUE_HEAD(set); QUEUE_REMOVE(el); SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); - removeReqFromSendQ(pReq); - notifyAndDestroyReq(conn, pReq, code); + evtCliRemoveReqFromSendQ(pReq); + evtCliNotifyAndDestroyReq(conn, pReq, code); } } -static FORCE_INLINE int32_t destroyAllReqs(SCliConn *conn) { +static FORCE_INLINE int32_t evtCliDestroyAllReqs(SCliConn *conn) { int32_t code = 0; SCliThrd2 *pThrd = conn->hostThrd; STrans *pInst = pThrd->pInst; @@ -2244,7 +2296,7 @@ static FORCE_INLINE int32_t destroyAllReqs(SCliConn *conn) { transQueueRemoveByFilter(&conn->reqsSentOut, filterAllReq, NULL, &set, -1); transQueueRemoveByFilter(&conn->reqsToSend, filterAllReq, NULL, &set, -1); - destroyReqInQueue(conn, &set, 0); + evtCliDestroyReqInQueue(conn, &set, 0); return 0; } static void evtCliDestroyAllQidFromThrd(SCliConn *conn) { @@ -2278,11 +2330,10 @@ static void evtCliDestroy(SCliConn *pConn) { int32_t code = 0; SCliThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; - // cliResetConnTimer(conn); tDebug("%s conn %p try to destroy", pInst->label, pConn); - code = destroyAllReqs(pConn); + code = evtCliDestroyAllReqs(pConn); if (code != 0) { tDebug("%s conn %p failed to all reqs since %s", pInst->label, pConn, tstrerror(code)); } @@ -2293,19 +2344,13 @@ static void evtCliDestroy(SCliConn *pConn) { tDebug("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); } - taosMemoryFree(pConn->dstAddr); - // taosMemoryFree(conn->stream); - taosMemoryFree(pConn->ipStr); - // cliDestroyAllQidFromThrd(conn); + taosMemoryFreeClear(pConn->dstAddr); + taosMemoryFreeClear(pConn->ipStr); if (pConn->pInitUserReq) { - taosMemoryFree(pConn->pInitUserReq); - pConn->pInitUserReq = NULL; + taosMemoryFreeClear(pConn->pInitUserReq); } - // taosMemoryFree(conn->buf); - // destroyWQ(&conn->wq); - taosCloseSocketNoCheck1(pConn->fd); transDestroyBuffer(&pConn->readBuf); @@ -2318,13 +2363,13 @@ static void evtCliDestroyConn(SCliConn *pConn, bool force) { SCliThrd2 *pThrd = pConn->hostThrd; STrans *pInst = pThrd->pInst; - // cliResetConnTimer(conn); - code = destroyAllReqs(pConn); + code = evtCliDestroyAllReqs(pConn); if (code != 0) { tError("%s conn %p failed to destroy all reqs on conn since %s", pInst->label, pConn, tstrerror(code)); } evtCliDestroyAllQidFromThrd(pConn); + if (pThrd->quit == false && pConn->list) { QUEUE_REMOVE(&pConn->q); pConn->list->totalSize -= 1; @@ -2332,7 +2377,7 @@ static void evtCliDestroyConn(SCliConn *pConn, bool force) { } if (pConn->task != NULL) { - transDQCancel(((SCliThrd2 *)pConn->hostThrd)->pEvtMgt->arg, pConn->task); + transDQCancel2(((SCliThrd2 *)pConn->hostThrd)->pEvtMgt->arg, pConn->task); pConn->task = NULL; } pConn->forceDelFromHeap = 1; @@ -2341,11 +2386,10 @@ static void evtCliDestroyConn(SCliConn *pConn, bool force) { tError("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); } - if (pConn->registered) { + if (pConn->connnected) { int8_t ref = pConn->ref; if (ref == 0) { - close(pConn->fd); - // uv_close((uv_handle_t *)conn->stream, cliDestroy); + evtCliDestroy(pConn); } } return; @@ -2379,7 +2423,7 @@ static int32_t transUnrefCliHandle(void *handle) { } // static int32_t transGet -static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq) { +static FORCE_INLINE void evtCliRemoveReqFromSendQ(SCliReq *pReq) { if (pReq == NULL || pReq->inSendQ == 0) { return; } @@ -2395,7 +2439,7 @@ static void destroyThrdObj(SCliThrd2 *pThrd) { } taosMemoryFree(pThrd); } -static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { +static int32_t evtCliCreateThrdObj(void *trans, SCliThrd2 **ppThrd) { int32_t line = 0; int32_t code = 0; @@ -2450,7 +2494,7 @@ static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { return code; } -FORCE_INLINE int cliRBChoseIdx(STrans *pInst) { +FORCE_INLINE int evtCliRBCloseIdx(STrans *pInst) { int32_t index = pInst->index; if (pInst->numOfThreads == 0) { return -1; @@ -2471,7 +2515,7 @@ static FORCE_INLINE SCliThrd2 *transGetWorkThrdFromHandle(STrans *trans, int64_t } taosWLockLatch(&exh->latch); if (exh->pThrd == NULL && trans != NULL) { - int idx = cliRBChoseIdx(trans); + int idx = evtCliRBCloseIdx(trans); if (idx < 0) return NULL; exh->pThrd = ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; } @@ -2484,14 +2528,14 @@ static FORCE_INLINE SCliThrd2 *transGetWorkThrdFromHandle(STrans *trans, int64_t } static SCliThrd2 *transGetWorkThrd(STrans *trans, int64_t handle) { if (handle == 0) { - int idx = cliRBChoseIdx(trans); + int idx = evtCliRBCloseIdx(trans); if (idx < 0) return NULL; return ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; } SCliThrd2 *pThrd = transGetWorkThrdFromHandle(trans, handle); return pThrd; } -static int32_t transInitMsg(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx, SCliReq **pCliMsg) { +static int32_t evtCliInitMsg(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx, SCliReq **pCliMsg) { int32_t code = 0; if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -2538,18 +2582,18 @@ static int32_t transInitMsg(void *pInstRef, const SEpSet *pEpSet, STransMsg *pRe return code; } -static FORCE_INLINE void destroyReq(void *arg) { +static FORCE_INLINE void evtCliDestroyReq(void *arg) { SCliReq *pReq = arg; if (pReq == NULL) { return; } - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); STraceId *trace = &pReq->msg.info.traceId; tGDebug("free memory:%p, free ctx: %p", pReq, pReq->ctx); if (pReq->ctx) { - destroyReqCtx(pReq->ctx); + evtCliDestroyReqCtx(pReq->ctx); } transFreeMsg(pReq->msg.pCont); taosMemoryFree(pReq); @@ -2590,7 +2634,7 @@ int32_t transReleaseCliHandle2(void *handle, int32_t status) { tGDebug("send release request at thread:%08" PRId64 ", malloc memory:%p", pThrd->pid, cmsg); if ((code = evtAsyncSend(pThrd->asyncHandle, &cmsg->q)) != 0) { - destroyReq(cmsg); + evtCliDestroyReq(cmsg); return code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code; } return code; @@ -2613,7 +2657,7 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, pReq->info.qId = handle; SCliReq *pCliMsg = NULL; - TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, ctx, &pCliMsg), NULL, _exception); + TAOS_CHECK_GOTO(evtCliInitMsg(pInstRef, pEpSet, pReq, ctx, &pCliMsg), NULL, _exception); STraceId *trace = &pReq->info.traceId; tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, @@ -2621,7 +2665,7 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, code = evtAsyncSend(pThrd->asyncHandle, &pCliMsg->q); if (code != 0) { - destroyReq(pCliMsg); + evtCliDestroyReq(pCliMsg); transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); } @@ -2638,7 +2682,7 @@ int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, } return code; } -static bool isReqExceedLimit(STransMsg *pReq) { +static bool evtCliIsReqExceedLimit(STransMsg *pReq) { if (pReq != NULL && pReq->contLen >= TRANS_MSG_LIMIT) { return true; } @@ -2648,7 +2692,7 @@ int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg if (transpointId == NULL) { return TSDB_CODE_INVALID_PARA; } - if (isReqExceedLimit(pReq)) { + if (evtCliIsReqExceedLimit(pReq)) { return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; } int32_t code = 0; @@ -2676,13 +2720,13 @@ int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg pReq->info.qId = *transpointId; SCliReq *pCliMsg = NULL; - TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, NULL, &pCliMsg), NULL, _exception); + TAOS_CHECK_GOTO(evtCliInitMsg(pInstRef, pEpSet, pReq, NULL, &pCliMsg), NULL, _exception); STraceId *trace = &pReq->info.traceId; tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); if ((code = evtAsyncSend(pThrd->asyncHandle, &(pCliMsg->q))) != 0) { - destroyReq(pCliMsg); + evtCliDestroyReq(pCliMsg); transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); } @@ -2701,7 +2745,7 @@ int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg return code; } int32_t transSendRecv2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransMsg *pRsp) { - if (isReqExceedLimit(pReq)) { + if (evtCliIsReqExceedLimit(pReq)) { return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; } STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); @@ -2781,7 +2825,7 @@ int32_t transSendRecv2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, ST code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); if (code != 0) { - destroyReq(pReq); + evtCliDestroyReq(pReq); TAOS_CHECK_GOTO((code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code), NULL, _RETURN); } TAOS_UNUSED(tsem_wait(sem)); @@ -2912,7 +2956,7 @@ int32_t transSendRecvWithTimeout2(void *pInstRef, SEpSet *pEpSet, STransMsg *pRe code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); if (code != 0) { - destroyReq(pReq); + evtCliDestroyReq(pReq); TAOS_CHECK_GOTO(code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code, NULL, _RETURN); goto _RETURN; } @@ -2993,7 +3037,7 @@ int32_t transSetDefaultAddr2(void *pInstRef, const char *ip, const char *fqdn) { if ((code = evtAsyncSend(thrd->asyncHandle, &(pReq->q))) != 0) { taosMemoryFree(pCtx->pCvtAddr); - destroyReq(pReq); + evtCliDestroyReq(pReq); if (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT) { code = TSDB_CODE_RPC_MODULE_QUIT; } @@ -3094,7 +3138,7 @@ static int32_t evtCliRecycleConn(SCliConn *pConn) { return 0; } } - addConnToPool(pThrd->pool, pConn); + evtCliAddConnToPool(pThrd->pool, pConn); return 1; } else if ((transQueueSize(&pConn->reqsToSend) == 0) && (transQueueSize(&pConn->reqsSentOut) == 0) && (taosHashGetSize(pConn->pQTable) != 0)) { @@ -3111,7 +3155,7 @@ static bool filteBySeq(void *key, void *arg) { SFiterArg *targ = arg; SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); if (pReq->seq == targ->seq && pReq->msg.msgType + 1 == targ->msgType) { - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); return true; } else { return false; @@ -3395,11 +3439,13 @@ static int32_t evtCliDoSched(SCliReq *pReq, SCliThrd2 *pThrd) { arg->param1 = pReq; arg->param2 = pThrd; - SDelayTask *pTask = transDQSched(pThrd->pEvtMgt->arg, doDelayTask, arg, pCtx->retryNextInterval); + SDelayTask *pTask = NULL; + code = transDQSched2(pThrd->pEvtMgt->arg, doDelayTask, arg, pCtx->retryNextInterval, &pTask); if (pTask == NULL) { taosMemoryFree(arg); return TSDB_CODE_OUT_OF_MEMORY; } + return code; } static bool evtCliMayRetry(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { @@ -3469,7 +3515,7 @@ static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { STrans *pInst = pThrd->pInst; if (pReq != NULL) { - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); if (pResp->code != TSDB_CODE_SUCCESS) { if (evtCliMayRetry(pConn, pReq, pResp)) { return TSDB_CODE_RPC_ASYNC_IN_PROCESS; @@ -3505,14 +3551,14 @@ static bool filterByQid(void *key, void *arg) { SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); if (pReq->msg.info.qId == *qid) { - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); return true; } else { return false; } } -static FORCE_INLINE void destroyReqWrapper(void *arg, void *param) { +static FORCE_INLINE void evtCliDestroyReqWrapper(void *arg, void *param) { if (arg == NULL) return; SCliReq *pReq = arg; @@ -3523,18 +3569,18 @@ static FORCE_INLINE void destroyReqWrapper(void *arg, void *param) { (*pThrd->destroyAhandleFp)(pReq->ctx->ahandle); } } - destroyReq(pReq); + evtCliDestroyReq(pReq); } -static FORCE_INLINE void destroyReqAndAhanlde(void *param) { +static FORCE_INLINE void evtCliDestroyReqAndAhanlde(void *param) { if (param == NULL) return; STaskArg *arg = param; SCliReq *pReq = arg->param1; SCliThrd2 *pThrd = arg->param2; - destroyReqWrapper(pReq, pThrd); + evtCliDestroyReqWrapper(pReq, pThrd); } -int32_t cliHandleStateMayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { +int32_t evtCliMayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { int32_t code = 0; SCliThrd2 *pThrd = conn->hostThrd; if (pHead->msgType == TDMT_SCH_TASK_RELEASE || pHead->msgType == TDMT_SCH_TASK_RELEASE + 1) { @@ -3573,10 +3619,10 @@ int32_t cliHandleStateMayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) queue *el = QUEUE_HEAD(&set); QUEUE_REMOVE(el); SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); STraceId *trace = &pReq->msg.info.traceId; tGDebug("start to free msg %p", pReq); - destroyReqWrapper(pReq, pThrd); + evtCliDestroyReqWrapper(pReq, pThrd); } taosMemoryFree(pHead); return 1; @@ -3603,7 +3649,7 @@ int32_t cliHandleStateMayCreateAhandle(SCliConn *conn, STransMsgHead *pHead, STr return 0; } -static FORCE_INLINE void cliConnClearInitUserMsg(SCliConn *conn) { +static FORCE_INLINE void evtCliConnClearInitUserMsg(SCliConn *conn) { if (conn->pInitUserReq) { taosMemoryFree(conn->pInitUserReq); conn->pInitUserReq = NULL; @@ -3620,7 +3666,7 @@ int32_t cliHandleStateMayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { return 0; } -int32_t cliHandleStateMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { +int32_t evtCliMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { int32_t code = 0; int64_t qid = pReq->msg.info.qId; SReqCtx *pCtx = pReq->ctx; @@ -3642,7 +3688,7 @@ int32_t cliHandleStateMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { } return 0; } -int32_t cliHandleStateMayUpdateState(SCliConn *pConn, SCliReq *pReq) { +int32_t evtCliMayUpdateState(SCliConn *pConn, SCliReq *pReq) { SCliThrd2 *pThrd = pConn->hostThrd; int32_t code = 0; int64_t qid = pReq->msg.info.qId; @@ -3659,7 +3705,7 @@ int32_t cliHandleStateMayUpdateState(SCliConn *pConn, SCliReq *pReq) { tDebug("%s conn %p succ to add statue, sid:%" PRId64 " (1)", transLabel(pThrd->pInst), pConn, qid); } - TAOS_UNUSED(cliHandleStateMayUpdateStateCtx(pConn, pReq)); + TAOS_UNUSED(evtCliMayUpdateStateCtx(pConn, pReq)); return code; } @@ -3679,9 +3725,9 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { STransMsg resp = {0}; - cliConnClearInitUserMsg(pConn); + evtCliConnClearInitUserMsg(pConn); - if (cliHandleStateMayHandleReleaseResp(pConn, pHead)) { + if (evtCliMayHandleReleaseResp(pConn, pHead)) { if (evtCliRecycleConn(pConn)) { return code; } @@ -3715,7 +3761,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { tstrerror(code)); } } - removeReqFromSendQ(pReq); + evtCliRemoveReqFromSendQ(pReq); code = evtCliBuildRespFromCont(pReq, &resp, pHead); STraceId *trace = &resp.info.traceId; @@ -3726,7 +3772,7 @@ static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tGWarn("%s msg need retry", CONN_GET_INST_LABEL(pConn)); } else { - destroyReq(pReq); + evtCliDestroyReq(pReq); } if (evtCliRecycleConn(pConn)) { @@ -4112,11 +4158,11 @@ static int32_t evtCliHandleExcept(void *thrd, SCliReq *pReq, STransMsg *pResp) { STrans *pInst = pThrd->pInst; int32_t code = evtCliBuildExceptResp(pThrd, pReq, pResp); if (code != 0) { - destroyReq(pReq); + evtCliDestroyReq(pReq); return code; } pInst->cfp(pInst->parent, pResp, NULL); - destroyReq(pReq); + evtCliDestroyReq(pReq); return code; } @@ -4160,7 +4206,7 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { code = evtCliMayGetStateByQid(pThrd, req, &pConn); if (code == 0) { - cliHandleStateMayUpdateStateCtx(pConn, req); + evtCliMayUpdateStateCtx(pConn, req); } else if (code == TSDB_CODE_RPC_STATE_DROPED) { TAOS_CHECK_GOTO(code, &lino, _end); } else if (code == TSDB_CODE_RPC_NO_STATE || code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { @@ -4169,13 +4215,13 @@ static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { char addr[TSDB_FQDN_LEN + 64] = {0}; snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); - code = getOrCreateConn(pThrd, fqdn, port, &pConn); + code = evtCliGetOrCreateConn(pThrd, fqdn, port, &pConn); if (code != 0) { tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); TAOS_CHECK_GOTO(code, &lino, _end); } else { } - cliHandleStateMayUpdateState(pConn, req); + evtCliMayUpdateState(pConn, req); } transQueuePush(&pConn->reqsToSend, &req->q); @@ -4239,7 +4285,7 @@ static int32_t evtCliTimeoutFunc(void *arg) { int32_t code = 0; STimeoutArg *pArg = arg; - transDQHandleTimeout(pArg->queue); + transDQHandleTimeout2(pArg->queue); return code; } static int32_t evtCliCalcTimeout(void *arg, int64_t *timeout) { @@ -4247,7 +4293,7 @@ static int32_t evtCliCalcTimeout(void *arg, int64_t *timeout) { STimeoutArg *pArg = arg; int32_t code = 0; - *timeout = transDQGetNextTimeout(pArg->queue); + *timeout = transDQGetNextTimeout2(pArg->queue); return code; } static void *cliWorkThread2(void *arg) { @@ -4279,7 +4325,7 @@ static void *cliWorkThread2(void *arg) { while (!pThrd->quit) { struct timeval tv = {30, 0}; - code = evtCaclNextTimeout(pThrd->pEvtMgt, &tv); + code = evtCalcNextTimeout(pThrd->pEvtMgt, &tv); if (code != 0) { tTrace("%s failed to calc next timeout", pInst->label); } else { @@ -4328,7 +4374,7 @@ void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads for (int i = 0; i < cli->numOfThreads; i++) { SCliThrd2 *pThrd = NULL; - code = createThrdObj(arg, &pThrd); + code = evtCliCreateThrdObj(arg, &pThrd); TAOS_CHECK_EXIT(code); code = evtInitPipe(pThrd->pipe_queue_fd); @@ -4491,12 +4537,12 @@ static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { return 0; } + SCliConn *topConn = NULL; SCliThrd2 *pThrd = p->hostThrd; STrans *pInst = pThrd->pInst; int32_t balanceLimit = pThrd->shareConnLimit >= 4 ? pThrd->shareConnLimit / 2 : 2; - SCliConn *topConn = NULL; - int32_t code = transHeapGet(heap, &topConn); + int32_t code = transHeapGet(heap, &topConn); if (code != 0) { return code; } @@ -4520,16 +4566,11 @@ static FORCE_INLINE int32_t timeCompare(const HeapNode *a, const HeapNode *b) { return 1; } } -int32_t transDQCreate(void *loop, SDelayQueue **queue) { +int32_t transDQCreate2(void *loop, SDelayQueue **queue) { int32_t code = 0; Heap *heap = NULL; - // uv_timer_t* timer = NULL; - SDelayQueue *q = NULL; - // timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); - // if (timer == NULL) { - // return terrno; - // } + SDelayQueue *q = NULL; heap = heapCreate(timeCompare); if (heap == NULL) { @@ -4551,7 +4592,7 @@ int32_t transDQCreate(void *loop, SDelayQueue **queue) { taosMemoryFree(q); return TSDB_CODE_OUT_OF_MEMORY; } -void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)) { +void transDQDestroy2(SDelayQueue *queue, void (*freeFunc)(void *arg)) { int32_t code = 0; while (heapSize(queue->heap) > 0) { HeapNode *minNode = heapMin(queue->heap); @@ -4572,32 +4613,26 @@ void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)) { taosMemoryFree(queue); return; } -SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs) { + +int32_t transDQSched2(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs, SDelayTask **pTask) { + int32_t code = 0; uint64_t now = taosGetTimestampMs(); SDelayTask *task = taosMemoryCalloc(1, sizeof(SDelayTask)); if (task == NULL) { - return NULL; + return terrno; } task->func = func; task->arg = arg; task->execTime = now + timeoutMs; - // HeapNode *minNode = heapMin(queue->heap); - // if (minNode) { - // SDelayTask *minTask = container_of(minNode, SDelayTask, node); - // if (minTask->execTime < task->execTime) { - // timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; - // } - // } - - // tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); + heapInsert(queue->heap, &task->node); - // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0)); - return task; -} -void transDQCancel(SDelayQueue *queue, SDelayTask *task) { - // TAOS_UNUSED(uv_timer_stop(queue->timer)); + tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); + *pTask = task; + return code; +} +void transDQCancel2(SDelayQueue *queue, SDelayTask *task) { if (heapSize(queue->heap) <= 0) { taosMemoryFree(task->arg); taosMemoryFree(task); @@ -4607,19 +4642,9 @@ void transDQCancel(SDelayQueue *queue, SDelayTask *task) { taosMemoryFree(task->arg); taosMemoryFree(task); - - // if (heapSize(queue->heap) != 0) { - // HeapNode *minNode = heapMin(queue->heap); - // if (minNode == NULL) return; - - // uint64_t now = taosGetTimestampMs(); - // SDelayTask *task = container_of(minNode, SDelayTask, node); - // uint64_t timeout = now > task->execTime ? now - task->execTime : 0; - - // } } -int32_t transDQHandleTimeout(SDelayQueue *queue) { +int32_t transDQHandleTimeout2(SDelayQueue *queue) { int32_t code = 0; uint64_t now = taosGetTimestampMs(); while (heapSize(queue->heap) > 0) { @@ -4637,7 +4662,7 @@ int32_t transDQHandleTimeout(SDelayQueue *queue) { } return code; } -int32_t transDQGetNextTimeout(SDelayQueue *queue) { +int32_t transDQGetNextTimeout2(SDelayQueue *queue) { if (heapSize(queue->heap) <= 0) { return -1; } From 81ee4c50c7411fb7277abd11408c37c7c7b2ad50 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Jan 2025 17:15:13 +0800 Subject: [PATCH 74/76] refactor code --- source/libs/transport/src/transImpl2.c | 4 ++-- source/libs/transport/test/transImpl2_test.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c index 45a6ee693b44..57922cc11b70 100644 --- a/source/libs/transport/src/transImpl2.c +++ b/source/libs/transport/src/transImpl2.c @@ -377,7 +377,7 @@ int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { SAsyncHandle *handle = pArg->arg; nBytes = read(pArg->fd, buf, sizeof(buf)); - if (nBytes == 1 && buf[0] == '1') { + if (nBytes >= 1 && buf[0] == '1') { handle->cb(handle, 0); } tTrace("%s handle async read on fd:%d", pOpt->label, pArg->fd); @@ -2213,7 +2213,7 @@ static int32_t evtCliGetOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, S return code; } - code = evtCliCreateNewSocket(pThrd, ip, port, &pConn); + code = evtCliCreateNewSocket(pThrd, ip, port, ppConn); _end: if (code != 0) { diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp index bb48360beb25..9db14b37c21a 100644 --- a/source/libs/transport/test/transImpl2_test.cpp +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -303,7 +303,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndReq) { - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 2; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 1; req.pCont = rpcMallocCont(10); @@ -312,5 +312,5 @@ TEST_F(TransEnv, 01sendAndReq) { //taosMsleep(100000); assert(resp.code == 0); } - taosMsleep(20000); + taosSsleep(2000000); } From 76db889b09352add64e8f356fa3989325f653a06 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Jan 2025 18:28:40 +0800 Subject: [PATCH 75/76] add test --- source/libs/transport/test/CMakeLists.txt | 59 ++++++ source/libs/transport/test/cliBench2.c | 212 ++++++++++++++++++++++ source/libs/transport/test/svrBench2.c | 212 ++++++++++++++++++++++ 3 files changed, 483 insertions(+) create mode 100644 source/libs/transport/test/cliBench2.c create mode 100644 source/libs/transport/test/svrBench2.c diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 34c906295e44..ed30ae581a6e 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -5,6 +5,8 @@ add_executable(svrBench "") add_executable(cliBench "") add_executable(httpBench "") add_executable(transImpl2 "") +add_executable(cliBench2 "") +add_executable(svrBench2 "") target_sources(transUT PRIVATE @@ -24,10 +26,19 @@ target_sources(svrBench PRIVATE "svrBench.c" ) + +target_sources(svrBench2 + PRIVATE + "svrBench2.c" +) target_sources(cliBench PRIVATE "cliBench.c" ) +target_sources(cliBench2 + PRIVATE + "cliBench2.c" +) target_sources(httpBench PRIVATE "http_test.c" @@ -96,6 +107,19 @@ target_link_libraries(svrBench gtest_main transport ) +target_include_directories(svrBench2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(svrBench2 + os + util + common + gtest_main + transport +) target_include_directories(cliBench PUBLIC @@ -103,6 +127,18 @@ target_include_directories(cliBench "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(httpBench + PUBLIC + "${TD_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_include_directories(cliBench2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + target_include_directories(httpBench PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" @@ -131,6 +167,29 @@ target_link_libraries(cliBench transport ) +target_link_libraries(svrBench + os + util + common + gtest_main + transport +) + +target_link_libraries(cliBench2 + os + util + common + gtest_main + transport +) +target_link_libraries(svrBench2 + os + util + common + gtest_main + transport +) + target_link_libraries(httpBench os util diff --git a/source/libs/transport/test/cliBench2.c b/source/libs/transport/test/cliBench2.c new file mode 100644 index 000000000000..fdcf71d28e31 --- /dev/null +++ b/source/libs/transport/test/cliBench2.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef TD_ACORE +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "transLog.h" +#include "trpc.h" +#include "tutil.h" +#include "tversion.h" + +typedef struct { + int index; + SEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t *pOverSem; + TdThread thread; + void *pRpc; +} SInfo; + +void initLogEnv() { + const char *logDir = "/tmp/trans_cli"; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10000; + tsAsyncLog = 0; + // idxDebugFlag = 143; + strcpy(tsLogDir, (char *)logDir); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +static void processResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SInfo *pInfo = (SInfo *)pMsg->info.ahandle; + tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); + + rpcFreeCont(pMsg->pCont); + tsem_post(&pInfo->rspSem); +} + +static int tcount = 0; + +static void *sendRequest(void *param) { + SInfo *pInfo = (SInfo *)param; + SRpcMsg rpcMsg = {0}; + + tDebug("thread:%d, start to send request", pInfo->index); + + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { + pInfo->num++; + rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); + rpcMsg.contLen = pInfo->msgSize; + rpcMsg.info.ahandle = pInfo; + rpcMsg.info.noResp = 0; + rpcMsg.msgType = 1; + tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); + if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + tsem_wait(&pInfo->rspSem); + } + + tDebug("thread:%d, it is over", pInfo->index); + tcount++; + + return NULL; +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + SEpSet epSet; + int msgSize = 128; + int numOfReqs = 0; + int appThreads = 1; + char serverIp[40] = "127.0.0.1"; + struct timeval systemTime; + int64_t startTime, endTime; + + // server info + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.eps[0].port = 7000; + epSet.eps[1].port = 7000; + strcpy(epSet.eps[0].fqdn, serverIp); + strcpy(epSet.eps[1].fqdn, "192.168.0.1"); + + // client info + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "APP"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processResponse; + rpcInit.sessions = 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = "michael"; + + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.shareConnLimit = tsShareConnLimit; + rpcInit.supportBatch = 1; + rpcInit.compressSize = -1; + rpcDebugFlag = 143; + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { + numOfReqs = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { + appThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { + } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { + } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { + } else if (strcmp(argv[i], "-l") == 0 && i < argc - 1) { + rpcInit.shareConnLimit = atoi(argv[++i]); + } else if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { + rpcInit.compressSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + } else { + printf("\nusage: %s [options] \n", argv[0]); + printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); + printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); + printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-c compressSize]: compress size, default:%d\n", tsCompressMsgSize); + printf(" [-l shareConnLimit]: share conn limit, default:%d\n", tsShareConnLimit); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + initLogEnv(); + taosVersionStrToInt(td_version, &(rpcInit.compatibilityVer)); + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to initialize RPC"); + return terrno; + } + + tInfo("client is initialized"); + tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); + + int64_t now = taosGetTimestampUs(); + + SInfo **pInfo = (SInfo **)taosMemoryCalloc(1, sizeof(SInfo *) * appThreads); + for (int i = 0; i < appThreads; ++i) { + SInfo *p = taosMemoryCalloc(1, sizeof(SInfo)); + p->index = i; + p->epSet = epSet; + p->numOfReqs = numOfReqs; + p->msgSize = msgSize; + tsem_init(&p->rspSem, 0, 0); + p->pRpc = pRpc; + pInfo[i] = p; + + taosThreadCreate(&p->thread, NULL, sendRequest, pInfo[i]); + } + + do { + taosUsleep(1); + } while (tcount < appThreads); + + float usedTime = (taosGetTimestampUs() - now) / 1000.0f; + + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + + for (int i = 0; i < appThreads; i++) { + SInfo *p = pInfo[i]; + taosThreadJoin(p->thread, NULL); + taosMemoryFree(p); + } + taosMemoryFree(pInfo); + + // int ch = getchar(); + // UNUSED(ch); + + taosCloseLog(); + + return 0; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/test/svrBench2.c b/source/libs/transport/test/svrBench2.c new file mode 100644 index 000000000000..c8e318e636ba --- /dev/null +++ b/source/libs/transport/test/svrBench2.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// #define _DEFAULT_SOURCE + +#ifdef TD_ACORE +#include "os.h" +#include "tglobal.h" +#include "tqueue.h" +#include "transLog.h" +#include "trpc.h" +#include "tversion.h" + +int msgSize = 128; +int commit = 0; +TdFilePtr pDataFile = NULL; +STaosQueue *qhandle = NULL; +STaosQset *qset = NULL; + +int32_t balance = 0; + +typedef struct { + int32_t numOfThread; + STaosQueue **qhandle; + STaosQset **qset; + +} MultiThreadQhandle; + +typedef struct TThread { + TdThread thread; + int idx; +} TThread; + +MultiThreadQhandle *tMultiQ = NULL; + +void initLogEnv() { + const char *logDir = "/tmp/trans_svr"; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10000; + tsAsyncLog = 0; + // idxDebugFlag = 143; + strcpy(tsLogDir, logDir); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} +void *processShellMsg(void *arg) { + TThread *thread = (TThread *)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall *qall; + SRpcMsg *pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(tMultiQ->qset[idx], qall, &qinfo); + tDebug("%d shell msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.info = pRpcMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + taosFreeQitem(pRpcMsg); + } + + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} + +void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg *pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void **)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int32_t idx = balance % tMultiQ->numOfThread; + tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(tMultiQ->qhandle[idx], pTemp); + balance++; + if (balance >= tMultiQ->numOfThread) balance = 0; +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + char dataName[20] = "server.data"; + + taosBlockSIGPIPE(); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 7000; + memcpy(rpcInit.localFqdn, "localhost", strlen("localhost")); + rpcInit.label = "SER"; + rpcInit.numOfThreads = 10; + rpcInit.cfp = processRequestMsg; + rpcInit.idleTime = 2 * 1500; + + taosVersionStrToInt(td_version, &(rpcInit.compatibilityVer)); + rpcDebugFlag = 131; + rpcInit.compressSize = -1; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { + rpcInit.localPort = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { + rpcInit.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { + msgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { + rpcInit.sessions = atoi(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { + tsCompressMsgSize = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { + commit = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { + rpcDebugFlag = atoi(argv[++i]); + dDebugFlag = rpcDebugFlag; + uDebugFlag = rpcDebugFlag; + } else { + printf("\nusage:%s [options] \n", argv[0]); + printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); + printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); + printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); + printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); + printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); + printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); + printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); + printf(" [-h help]: print out this help\n\n"); + exit(0); + } + } + + rpcInit.connType = TAOS_CONN_SERVER; + + initLogEnv(); + taosVersionStrToInt(td_version, &(rpcInit.compatibilityVer)); + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + tError("failed to start RPC server"); + return -1; + } + // taosSsleep(5); + + tInfo("RPC server is running, ctrl-c to exit"); + + if (commit) { + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CREATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); + } + + int32_t numOfAthread = 1; + tMultiQ = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + tMultiQ->numOfThread = numOfAthread; + tMultiQ->qhandle = (STaosQueue **)taosMemoryMalloc(sizeof(STaosQueue *) * numOfAthread); + tMultiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&tMultiQ->qhandle[i]); + taosOpenQset(&tMultiQ->qset[i]); + taosAddIntoQset(tMultiQ->qset[i], tMultiQ->qhandle[i], NULL); + } + TThread *threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, processShellMsg, (void *)&threads[i]); + } + + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); + taosRemoveFile(dataName); + } + int ch = getchar(); + UNUSED(ch); + + return 0; +} + +#endif From 064204903d24a045cbe8fa223ec73fd47bd01df8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Jan 2025 08:43:14 +0800 Subject: [PATCH 76/76] add test --- source/libs/transport/test/svrBench2.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/test/svrBench2.c b/source/libs/transport/test/svrBench2.c index c8e318e636ba..4fbcf5f6e1dd 100644 --- a/source/libs/transport/test/svrBench2.c +++ b/source/libs/transport/test/svrBench2.c @@ -47,9 +47,10 @@ MultiThreadQhandle *tMultiQ = NULL; void initLogEnv() { const char *logDir = "/tmp/trans_svr"; - const char *defaultLogFileNamePrefix = "taoslog"; + const char *defaultLogFileNamePrefix = "taosdlog"; const int32_t maxLogFileNum = 10000; tsAsyncLog = 0; + rpcDebugFlag = 143; // idxDebugFlag = 143; strcpy(tsLogDir, logDir); taosRemoveDir(tsLogDir); @@ -125,12 +126,12 @@ int main(int argc, char *argv[]) { rpcInit.localPort = 7000; memcpy(rpcInit.localFqdn, "localhost", strlen("localhost")); rpcInit.label = "SER"; - rpcInit.numOfThreads = 10; + rpcInit.numOfThreads = 1; rpcInit.cfp = processRequestMsg; rpcInit.idleTime = 2 * 1500; taosVersionStrToInt(td_version, &(rpcInit.compatibilityVer)); - rpcDebugFlag = 131; + rpcDebugFlag = 143; rpcInit.compressSize = -1; for (int i = 1; i < argc; ++i) { @@ -173,7 +174,7 @@ int main(int argc, char *argv[]) { tError("failed to start RPC server"); return -1; } - // taosSsleep(5); + taosSsleep(5); tInfo("RPC server is running, ctrl-c to exit");