Merge branch 'develop' into master

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "tests/examples/rust"]
 	path = tests/examples/rust
 	url = https://github.com/songtianyi/tdengine-rust-bindings.git
+[submodule "src/connector/hivemq-tdengine-extension"]
+	path = src/connector/hivemq-tdengine-extension
+	url = https://github.com/huskar-t/hivemq-tdengine-extension.git

Jenkinsfile

@@ -79,7 +79,14 @@ pipeline {
             cmake .. > /dev/null
             make > /dev/null
             cd ${WKC}/tests/pytest
-            ./crash_gen.sh -a -p -t 4 -s 2000
+            '''
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./crash_gen.sh -a -p -t 4 -s 2000
+                '''
+            }
+            sh '''
             date
             cd ${WKC}/tests
             ./test-all.sh b2

packaging/deb/makedeb.sh

@@ -58,7 +58,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin        ${pkg_dir}${install_home_pat
 cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/nodejs               ${pkg_dir}${install_home_path}/connector
-cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.*  ${pkg_dir}${install_home_path}/connector
+cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.*  ${pkg_dir}${install_home_path}/connector ||:
 
 cp -r ${compile_dir}/../packaging/deb/DEBIAN        ${pkg_dir}/
 chmod 755 ${pkg_dir}/DEBIAN/*

packaging/release.sh

@@ -156,9 +156,15 @@ build_time=$(date +"%F %R")
 
 # get commint id from git
 gitinfo=$(git rev-parse --verify HEAD)
-enterprise_dir="${top_dir}/../enterprise"
-cd ${enterprise_dir}
-gitinfoOfInternal=$(git rev-parse --verify HEAD)
+
+if [[ "$verMode" == "cluster" ]]; then
+  enterprise_dir="${top_dir}/../enterprise"
+  cd ${enterprise_dir}
+  gitinfoOfInternal=$(git rev-parse --verify HEAD)
+else
+  gitinfoOfInternal=NULL
+fi
+  
 cd ${curr_dir}
 
 # 2. cmake executable file
@@ -193,23 +199,35 @@ cd ${curr_dir}
 # 3. Call the corresponding script for packaging
 if [ "$osType" != "Darwin" ]; then
   if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
-    echo "====do deb package for the ubuntu system===="
-    output_dir="${top_dir}/debs"
-    if [ -d ${output_dir} ]; then
-      ${csudo} rm -rf ${output_dir}
+    ret='0'    
+    command -v dpkg >/dev/null 2>&1 || { ret='1'; }
+    if [ "$ret" -eq 0 ]; then  
+      echo "====do deb package for the ubuntu system===="
+      output_dir="${top_dir}/debs"
+      if [ -d ${output_dir} ]; then
+        ${csudo} rm -rf ${output_dir}
+      fi
+      ${csudo} mkdir -p ${output_dir}
+      cd ${script_dir}/deb
+      ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+    else
+      echo "==========dpkg command not exist, so not release deb package!!!"
     fi
-    ${csudo} mkdir -p ${output_dir}
-    cd ${script_dir}/deb
-    ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
-
-    echo "====do rpm package for the centos system===="
-    output_dir="${top_dir}/rpms"
-    if [ -d ${output_dir} ]; then
-      ${csudo} rm -rf ${output_dir}
+
+    ret='0'    
+    command -v rpmbuild >/dev/null 2>&1 || { ret='1'; }
+    if [ "$ret" -eq 0 ]; then  
+      echo "====do rpm package for the centos system===="
+      output_dir="${top_dir}/rpms"
+      if [ -d ${output_dir} ]; then
+        ${csudo} rm -rf ${output_dir}
+      fi
+      ${csudo} mkdir -p ${output_dir}
+      cd ${script_dir}/rpm
+      ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+    else
+      echo "==========rpmbuild command not exist, so not release rpm package!!!"
     fi
-    ${csudo} mkdir -p ${output_dir}
-    cd ${script_dir}/rpm
-    ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
   fi
 	
   echo "====do tar.gz package for all systems===="

packaging/rpm/tdengine.spec

@@ -65,7 +65,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn
 cp -r %{_compiledir}/../src/connector/python        %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/go            %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/nodejs        %{buildroot}%{homepath}/connector
-cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.*  %{buildroot}%{homepath}/connector
+cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.*  %{buildroot}%{homepath}/connector ||:
 cp -r %{_compiledir}/../tests/examples/*            %{buildroot}%{homepath}/examples
 
 #Scripts executed before installation

packaging/tools/make_install.sh

@@ -278,11 +278,11 @@ function install_service_on_sysvinit() {
 
     # Install taosd service
     if ((${os_type}==1)); then
-    ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d
-    ${csudo} cp    ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+    ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d
+    ${csudo} cp    ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
     elif ((${os_type}==2)); then
-    ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d
-    ${csudo} cp    ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+    ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d
+    ${csudo} cp    ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
     fi
 
     #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"

packaging/tools/makeclient.sh

@@ -110,7 +110,7 @@ mkdir -p ${install_dir}/connector
 
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then
-    cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+    cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
   fi
   cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
   cp -r ${connector_dir}/python        ${install_dir}/connector/

packaging/tools/makeclient_power.sh

@@ -135,7 +135,7 @@ mkdir -p ${install_dir}/connector
 
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then
-    cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+    cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
   fi
   cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
   cp -r ${connector_dir}/python  ${install_dir}/connector/

packaging/tools/makepkg.sh

@@ -124,7 +124,7 @@ cp ${lib_files} ${install_dir}/driver
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-  cp ${build_dir}/lib/*.jar            ${install_dir}/connector
+  cp ${build_dir}/lib/*.jar            ${install_dir}/connector ||:
   cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
   cp -r ${connector_dir}/python        ${install_dir}/connector/
   cp -r ${connector_dir}/go            ${install_dir}/connector

packaging/tools/makepkg_power.sh

@@ -156,7 +156,7 @@ cp ${lib_files} ${install_dir}/driver
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-  cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+  cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
   cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
   cp -r ${connector_dir}/python  ${install_dir}/connector/
   cp -r ${connector_dir}/go      ${install_dir}/connector

src/client/inc/tscUtil.h

@@ -82,6 +82,7 @@ typedef struct SJoinSupporter {
   char*           pIdTagList;      // result of first stage tags
   int32_t         totalLen;
   int32_t         num;
+  SArray*         pVgroupTables;
 } SJoinSupporter;
 
 typedef struct SVgroupTableInfo {
@@ -215,7 +216,7 @@ SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex);
 void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache);
 
 STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
-    SVgroupsInfo* vgroupList, SArray* pTagCols);
+    SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
 
 STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo);
 int32_t tscAddSubqueryInfo(SSqlCmd *pCmd);
@@ -224,6 +225,8 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo);
 
 void tscClearSubqueryInfo(SSqlCmd* pCmd);
 void tscFreeVgroupTableInfo(SArray* pVgroupTables);
+SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables);
+void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
 
 int  tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
 int  tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);

src/client/inc/tsclient.h

@@ -30,6 +30,7 @@ extern "C" {
 #include "tsqlfunction.h"
 #include "tutil.h"
 #include "tcache.h"
+#include "tref.h"
 
 #include "qExecutor.h"
 #include "qSqlparser.h"
@@ -446,7 +447,7 @@ void tscFreeSqlObj(SSqlObj *pSql);
 void tscFreeRegisteredSqlObj(void *pSql);
 void tscFreeTableMetaHelper(void *pTableMeta);
 
-void tscCloseTscObj(STscObj *pObj);
+void tscCloseTscObj(void *pObj);
 
 // todo move to taos? or create a new file: taos_internal.h
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
@@ -516,6 +517,7 @@ extern void *    tscQhandle;
 extern int       tscKeepConn[];
 extern int       tsInsertHeadSize;
 extern int       tscNumOfThreads;
+extern int       tscRefId;
 
 extern SRpcCorEpSet tscMgmtEpSet;
 

src/client/src/tscFunctionImpl.c

@@ -2461,12 +2461,22 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
   // the first stage, only acquire the min/max value
   if (pInfo->stage == 0) {
     if (pCtx->preAggVals.isSet) {
-      if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
-        SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
+      double tmin = 0.0, tmax = 0.0;
+      if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
+        tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); 
+        tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); 
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+        tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); 
+        tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); 
+      } else {
+        assert(true);
+      }
+      if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) {
+        SET_DOUBLE_VAL(&pInfo->minval, tmin);
       }
 
-      if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
-        SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) {
+        SET_DOUBLE_VAL(&pInfo->maxval, tmax);
       }
 
       pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
@@ -4025,11 +4035,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) {
 
   // primary ts must be existed, so no need to check its existance
   if (pCtx->order == TSDB_ORDER_ASC) {
-    tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE);
+    tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE);
   } else {
     for (int32_t i = pCtx->size - 1; i >= 0; --i) {
       char *d = GET_INPUT_CHAR_INDEX(pCtx, i);
-      tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE);
+      tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE);
     }
   }
 
@@ -4048,7 +4058,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) {
 
   STSBuf *pTSbuf = pInfo->pTSBuf;
 
-  tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE);
+  tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE);
   SET_VAL(pCtx, pCtx->size, 1);
 
   pResInfo->hasResult = DATA_SET_FLAG;

src/client/src/tscLocalMerge.c

@@ -698,7 +698,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
     pg *= 2;
   }
 
-  size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
+  size_t numOfSubs = pSql->subState.numOfSub;
+  assert(numOfSubs <= pTableMetaInfo->vgroupList->numOfVgroups);
   for (int32_t i = 0; i < numOfSubs; ++i) {
     (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel);
     (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;

src/client/src/tscSQLParser.c

@@ -877,22 +877,13 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
 
   int32_t nLen = 0;
   for (int32_t i = 0; i < pFieldList->nField; ++i) {
-    if (pFieldList->p[i].bytes == 0) {
+    TAOS_FIELD* pField = &pFieldList->p[i];
+
+    if (pField->bytes == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
       return false;
     }
-    nLen += pFieldList->p[i].bytes;
-  }
-
-  // max row length must be less than TSDB_MAX_BYTES_PER_ROW
-  if (nLen > TSDB_MAX_BYTES_PER_ROW) {
-    invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
-    return false;
-  }
 
-  // field name must be unique
-  for (int32_t i = 0; i < pFieldList->nField; ++i) {
-    TAOS_FIELD* pField = &pFieldList->p[i];
     if (pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
       return false;
@@ -909,10 +900,19 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
       return false;
     }
 
+    // field name must be unique
     if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       return false;
     }
+
+    nLen += pField->bytes;
+  }
+
+  // max row length must be less than TSDB_MAX_BYTES_PER_ROW
+  if (nLen > TSDB_MAX_BYTES_PER_ROW) {
+    invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    return false;
   }
 
   return true;

src/client/src/tscSchemaUtil.c

@@ -177,7 +177,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
     pVgroupInfo->epAddr[i].port = pEpMsg->port;
   }
 
-  tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
+  tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo);
 
   pTableMeta->sversion = pTableMetaMsg->sversion;
   pTableMeta->tversion = pTableMetaMsg->tversion;