Merge branch 'develop' into hotfix/sangshuduo/TD-3147-insert-rate-mor…

…e-than-1s

.gitignore

@@ -27,7 +27,6 @@ tests/hdfs/
 nmake/
 sln/
 hdfs/
-c/
 taoshebei/
 taosdalipu/
 Target/

Jenkinsfile

@@ -227,6 +227,8 @@ pipeline {
               ./test-all.sh p4
               cd ${WKC}/tests
               ./test-all.sh full jdbc
+              cd ${WKC}/tests
+              ./test-all.sh full unit
               date'''
             }
           }

documentation20/cn/00.index/docs.md

@@ -31,6 +31,20 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
 * [创建超级表](/model#create-stable)：为同一类型的数据采集点创建一个超级表
 * [创建表](/model#create-table)：使用超级表做模板，为每一个具体的数据采集点单独建表
 
+## [TAOS SQL](/taos-sql)
+
+* [支持的数据类型](/taos-sql#data-type)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型
+* [数据库管理](/taos-sql#management)：添加、删除、查看数据库
+* [表管理](/taos-sql#table)：添加、删除、查看、修改表
+* [超级表管理](/taos-sql#super-table)：添加、删除、查看、修改超级表
+* [标签管理](/taos-sql#tags)：增加、删除、修改标签
+* [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入
+* [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等
+* [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等
+* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理
+* [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件
+* [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码
+
 ## [高效写入数据](/insert)
 
 * [SQL写入](/insert#sql)：使用SQL insert命令向一张或多张表写入单条或多条记录
@@ -94,20 +108,6 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
 * [文件目录结构](/administrator#directories)：TDengine数据文件、配置文件等所在目录
 * [参数限制与保留关键字](/administrator#keywords)：TDengine的参数限制与保留关键字列表
 
-## [TAOS SQL](/taos-sql)
-
-* [支持的数据类型](/taos-sql#data-type)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型
-* [数据库管理](/taos-sql#management)：添加、删除、查看数据库
-* [表管理](/taos-sql#table)：添加、删除、查看、修改表
-* [超级表管理](/taos-sql#super-table)：添加、删除、查看、修改超级表
-* [标签管理](/taos-sql#tags)：增加、删除、修改标签
-* [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入
-* [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等
-* [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等
-* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理
-* [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件
-* [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码
-
 ## TDengine的技术设计
 
 * [系统模块](/architecture/taosd)：taosd的功能和模块划分

packaging/docker/Dockerfile

@@ -13,9 +13,8 @@ WORKDIR /root/${dirName}/
 RUN /bin/bash install.sh -e no
 
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042
 CMD ["taosd"]
 VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]

src/client/inc/tscUtil.h

@@ -270,7 +270,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
 bool hasMoreVnodesToTry(SSqlObj *pSql);
 bool hasMoreClauseToTry(SSqlObj* pSql);
 
-void tscFreeQueryInfo(SSqlCmd* pCmd);
+void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta);
 
 void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);

src/client/inc/tsclient.h

@@ -442,6 +442,8 @@ void tscCloseTscObj(void *pObj);
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, TAOS **taos);
 TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res);
+TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param);
+
 void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
 
 void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);

src/client/src/tscAsync.c

@@ -74,20 +74,24 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
 
 // TODO return the correct error code to client in tscQueueAsyncError
 void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
+  taos_query_ra(taos, sqlstr, fp, param);
+}
+
+TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
     tscError("bug!!! pObj:%p", pObj);
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
-    return;
+    return NULL;
   }
 
   int32_t sqlLen = (int32_t)strlen(sqlstr);
   if (sqlLen > tsMaxSQLStringLen) {
     tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
     terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
     tscQueueAsyncError(fp, param, terrno);
-    return;
+    return NULL;
   }
 
   nPrintTsc("%s", sqlstr);
@@ -96,12 +100,15 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa
   if (pSql == NULL) {
     tscError("failed to malloc sqlObj");
     tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY);
-    return;
+    return NULL;
   }
 
   doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen);
+
+  return pSql;
 }
 
+
 static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
   if (tres == NULL) {
     return;

src/client/src/tscSubquery.c

@@ -2802,7 +2802,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
         numOfFailed += 1;
 
         // clean up tableMeta in cache
-        tscFreeQueryInfo(&pSql->cmd);
+        tscFreeQueryInfo(&pSql->cmd, false);
         SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0);
         STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0);
         tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL);

src/client/src/tscUtil.c

@@ -30,7 +30,7 @@
 #include "ttokendef.h"
 
 static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
-static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo);
+static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta);
 
 static void tscStrToLower(char *str, int32_t n) {
   if (str == NULL || n <= 0) { return;}
@@ -367,7 +367,7 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) {
   pRes->data = NULL;  // pRes->data points to the buffer of pRsp, no need to free
 }
 
-void tscFreeQueryInfo(SSqlCmd* pCmd) {
+void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) {
   if (pCmd == NULL || pCmd->numOfClause == 0) {
     return;
   }
@@ -376,7 +376,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) {
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i);
 
     freeQueryInfoImpl(pQueryInfo);
-    clearAllTableMetaInfo(pQueryInfo);
+    clearAllTableMetaInfo(pQueryInfo, removeMeta);
     tfree(pQueryInfo);
   }
 
@@ -404,7 +404,7 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) {
 
   pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta);
   pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-  tscFreeQueryInfo(pCmd);
+  tscFreeQueryInfo(pCmd, removeMeta);
 }
 
 void tscFreeSqlResult(SSqlObj* pSql) {
@@ -1847,10 +1847,17 @@ SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) {
   return pa;
 }
 
-void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) {
+void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) {
   for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
 
+    if (removeMeta) {  
+      char name[TSDB_TABLE_FNAME_LEN] = {0};
+      tNameExtractFullName(&pTableMetaInfo->name, name);
+
+      taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
+    }
+
     tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
     tscClearTableMetaInfo(pTableMetaInfo);
     free(pTableMetaInfo);
@@ -2714,7 +2721,11 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
 uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
 
-  int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
+  int32_t totalCols = 0;
+  if (pTableMeta->tableInfo.numOfColumns >= 0 && pTableMeta->tableInfo.numOfTags >= 0) {
+    totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
+  }
+
   return sizeof(STableMeta) + totalCols * sizeof(SSchema);
 }
 

src/inc/taosdef.h

@@ -286,7 +286,7 @@ do { \
 #define TSDB_MAX_COMP_LEVEL             2
 #define TSDB_DEFAULT_COMP_LEVEL         2
 
-#define TSDB_MIN_WAL_LEVEL              1
+#define TSDB_MIN_WAL_LEVEL              0
 #define TSDB_MAX_WAL_LEVEL              2
 #define TSDB_DEFAULT_WAL_LEVEL          1
 

src/mnode/src/mnodeTable.c

@@ -832,12 +832,13 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) {
         return code;
       } else if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
         ++pMsg->pBatchMasterMsg->received;
+        pMsg->pBatchMasterMsg->code = code;
         mnodeDestroySubMsg(pMsg);
       }
 
       if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received
 	  >= pMsg->pBatchMasterMsg->expected) {
-        dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, TSDB_CODE_SUCCESS);
+        dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->pBatchMasterMsg->code);
       }
 
       return TSDB_CODE_MND_ACTION_IN_PROGRESS;
@@ -1908,7 +1909,8 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
     sdbDeleteRow(&desc);
 
     if (pMsg->pBatchMasterMsg) {
-      ++pMsg->pBatchMasterMsg->successed;
+      ++pMsg->pBatchMasterMsg->received;
+      pMsg->pBatchMasterMsg->code = code;
       if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received
 	  >= pMsg->pBatchMasterMsg->expected) {
 	dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code);
@@ -2690,6 +2692,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
 
       if (pMsg->pBatchMasterMsg) {
 	++pMsg->pBatchMasterMsg->received;
+	pMsg->pBatchMasterMsg->code = code;
 	if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received
 	    >= pMsg->pBatchMasterMsg->expected) {
 	  dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code);
@@ -2728,6 +2731,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
 
       if (pMsg->pBatchMasterMsg) {
 	++pMsg->pBatchMasterMsg->received;
+	pMsg->pBatchMasterMsg->code = rpcMsg->code;
 	if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received
 	    >= pMsg->pBatchMasterMsg->expected) {
 	  dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code);

src/mnode/src/mnodeVgroup.c

@@ -537,6 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
 
     if (pMsg->pBatchMasterMsg) {
       ++pMsg->pBatchMasterMsg->received;
+      pMsg->pBatchMasterMsg->code = pMsg->code;
       if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received
           >= pMsg->pBatchMasterMsg->expected) {
         dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->code);
@@ -1002,6 +1003,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
 
       if (mnodeMsg->pBatchMasterMsg) {
         ++mnodeMsg->pBatchMasterMsg->received;
+        mnodeMsg->pBatchMasterMsg->code = code;
         if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received
             >= mnodeMsg->pBatchMasterMsg->expected) {
           dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code);
@@ -1024,6 +1026,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
 
     if (mnodeMsg->pBatchMasterMsg) {
       ++mnodeMsg->pBatchMasterMsg->received;
+      mnodeMsg->pBatchMasterMsg->code = mnodeMsg->code;
       if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received
           >= mnodeMsg->pBatchMasterMsg->expected) {
         dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code);

src/os/inc/osDef.h

@@ -83,6 +83,20 @@ extern "C" {
     }                            \
   } while (0)
 
+#define DEFAULT_DOUBLE_COMP(x, y)           \
+  do {                                      \
+    if (isnan(x) && isnan(y)) { return 0; } \
+    if (isnan(x)) { return -1; }            \
+    if (isnan(y)) { return 1; }             \
+    if ((x) == (y)) {                       \
+      return 0;                             \
+    } else {                                \
+      return (x) < (y) ? -1 : 1;            \
+    }                                       \
+  } while (0)
+
+#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y)
+
 #define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
 
 // align to 8bytes

src/query/src/qExtbuffer.c

@@ -362,20 +362,10 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i
       return (first < second) ? -1 : 1;
     };
     case TSDB_DATA_TYPE_DOUBLE: {
-      double first  = GET_DOUBLE_VAL(f1);
-      double second = GET_DOUBLE_VAL(f2);
-      if (first == second) {
-        return 0;
-      }
-      return (first < second) ? -1 : 1;
+      DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2));
     };
     case TSDB_DATA_TYPE_FLOAT: {
-      float first  = GET_FLOAT_VAL(f1);
-      float second = GET_FLOAT_VAL(f2);
-      if (first == second) {
-        return 0;
-      }
-      return (first < second) ? -1 : 1;
+      DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2));
     };
     case TSDB_DATA_TYPE_BIGINT: {
       int64_t first = *(int64_t *)f1;

src/query/src/qParserImpl.c

@@ -58,6 +58,15 @@ SSqlInfo qSQLParse(const char *pStr) {
         sqlInfo.valid = false;
         goto abort_parse;
       }
+
+      case TK_HEX:
+      case TK_OCT:
+      case TK_BIN:{
+        snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z);
+        sqlInfo.valid = false;
+        goto abort_parse;
+      }
+
       default:
         Parse(pParser, t0.type, t0, &sqlInfo);
         if (sqlInfo.valid == false) {

src/query/tests/percentileTest.cpp

@@ -48,7 +48,7 @@ tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) {
     uint64_t k = i;
     int32_t ret = tMemBucketPut(pBucket, &k, 1);
     if (ret != 0) {
-      printf("value out of range:%f", k);
+      printf("value out of range:%" PRId64, k);
     }
   }
 
@@ -245,7 +245,7 @@ void unsignedDataTest() {
 }  // namespace
 
 TEST(testCase, percentileTest) {
-  //  qsortTest();
+//  qsortTest();
   intDataTest();
   bigintDataTest();
   doubleDataTest();

src/query/tests/unitTest.cpp

@@ -227,10 +227,10 @@ TEST(testCase, db_table_name) {
   EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_SQL);
 
   char t61[] = "' ABC '";
-  EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_SQL);
 
   char t61_1[] = "'  ABC '";
-  EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_SQL);
 
   char t62[] = " ABC . def ";
   EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_SQL);
@@ -249,13 +249,13 @@ TEST(testCase, db_table_name) {
   EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_SQL);
 
   char t66[] = "' ABC '.'  DEF '";
-  EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_SQL);
 
   char t67[] = "abc . '  DEF  '";
   EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_SQL);
 
   char t68[] = "'  abc '.'   DEF '";
-  EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_SQL);
 
   // do not use key words 
   char t69[] = "table.'DEF'";
@@ -265,7 +265,7 @@ TEST(testCase, db_table_name) {
   EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_SQL);
 
   char t71[] = "'_abXYZ1234  '.' deFF  '";
-  EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_SQL);
 
   char t72[] = "'_abDEF&^%1234'.'  DIef'";
   EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_SQL);