diff --git a/Jenkinsfile2 b/Jenkinsfile2 index fc00c5e2dce5..1b2f28908c63 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -7,7 +7,8 @@ file_zh_changed = '' file_en_changed = '' file_no_doc_changed = '1' file_only_tdgpt_change_except = '1' -tdgpt_file = "forecastoperator.c\\|anomalywindowoperator.c\\|tanalytics.h\\|tanalytics.c\\|tdgpt_cases.task\\|analytics" +tdgpt_file = "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" + def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -69,7 +70,7 @@ def check_docs(){ returnStdout: true ) - file_no_doc_changed = sh ( + def file_no_doc_changed = sh ( script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" || : @@ -80,7 +81,7 @@ def check_docs(){ file_only_tdgpt_change_except = sh ( script: ''' cd ${WKC} - git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v ${tdgpt_file} || : + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" ||: ''', returnStdout: true ).trim() @@ -572,7 +573,7 @@ pipeline { cd ${WKC}/tests/parallel_test ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' - if ( file_no_doc_changed =~ /orecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ) { + if ( file_no_doc_changed =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ) { sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 diff --git a/source/util/src/tanalytics.c b/source/util/src/tanalytics.c index e68edd4b7699..5c2814138685 100644 --- a/source/util/src/tanalytics.c +++ b/source/util/src/tanalytics.c @@ -7,7 +7,7 @@ * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. + * FITNESS FOR A PARTICULAR PURPOSE. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . diff --git a/tests/army/frame/server/dnode.py b/tests/army/frame/server/dnode.py index fdb48e480dcf..5534bd93fa61 100644 --- a/tests/army/frame/server/dnode.py +++ b/tests/army/frame/server/dnode.py @@ -23,6 +23,7 @@ import copy from fabric2 import Connection from shutil import which +from pathlib import Path # self from frame.log import * @@ -221,26 +222,42 @@ def deploy(self, *updatecfgDict): tdLog.debug( "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - + def getPath(self, tool="taosd"): + if platform.system().lower() == 'windows': + tool += ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] - else: + elif ("tests" in selfPath): projPath = selfPath[:selfPath.find("tests")] + else: + tdLog.exit("can't find community or tests in path %s" % selfPath) + paths = [] + exclude_dirs = ["packaging", ".git"] + + binPath = os.path.join(projPath, "debug", "build", "bin") + if tool in binPath: + paths.append(binPath) + return paths[0] + for root, dirs, files in os.walk(projPath): - if ((tool) in files or ("%s.exe"%tool) in files): + if tool in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): + if all(excl not in rootRealPath for excl in exclude_dirs): paths.append(os.path.join(root, tool)) break if (len(paths) == 0): - return "" + tdLog.exit(f"{tool} not found in {projPath}") + else: + tdLog.info(f"{tool} found in {paths[0]}") + return paths[0] + def starttaosd(self): binPath = self.getPath() diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0377ff1641f1..50cb91216a4e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1098,6 +1098,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f eco-system/manager/schema_change.py -N 3 -M 3 +,,y,system-test,./pytest.sh python3 test.py -f 99-TDcase/TD-32391.py #tsim test ,,y,script,./test.sh -f tsim/query/timeline.sim diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index c12f324fd7bb..882d637c8ff3 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -2131,6 +2131,41 @@ def update_json_file_replica(self, json_file_path, new_replica_value, output_fil except Exception as e: # Handle any other exceptions that may occur print(f"An error occurred: {e}") + + def get_path(self, tool="taosd"): + if platform.system().lower() == 'windows': + tool += ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + elif ("tests" in selfPath): + projPath = selfPath[:selfPath.find("tests")] + else: + tdLog.exit("can't find community or tests in path %s" % selfPath) + + + paths = [] + exclude_dirs = ["packaging", ".git"] + + binPath = os.path.join(projPath, "debug", "build", "bin") + if tool in binPath: + paths.append(binPath) + return paths[0] + + for root, dirs, files in os.walk(projPath): + if tool in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if all(excl not in rootRealPath for excl in exclude_dirs): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit(f"{tool} not found in {projPath}") + else: + tdLog.info(f"{tool} found in {paths[0]}") + + return paths[0] + def is_json(msg): if isinstance(msg, str): @@ -2142,24 +2177,6 @@ def is_json(msg): else: return False -def get_path(tool="taosd"): - selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - paths = [] - for root, dirs, files in os.walk(projPath): - if ((tool) in files or ("%s.exe"%tool) in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - paths.append(os.path.join(root, tool)) - break - if (len(paths) == 0): - return "" - return paths[0] - def dict2toml(in_dict: dict, file:str): if not isinstance(in_dict, dict): return "" diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2c2e11cfccad..fc13747e5a49 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -289,24 +289,40 @@ def deploy(self, *updatecfgDict): (self.index, self.cfgPath)) def getPath(self, tool="taosd"): + if platform.system().lower() == 'windows': + tool += ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] - else: + elif ("tests" in selfPath): projPath = selfPath[:selfPath.find("tests")] + else: + tdLog.exit("can't find community or tests in path %s" % selfPath) + paths = [] + exclude_dirs = ["packaging", ".git"] + + binPath = os.path.join(projPath, "debug", "build", "bin") + if tool in binPath: + paths.append(binPath) + return paths[0] + for root, dirs, files in os.walk(projPath): - if ((tool) in files or ("%s.exe"%tool) in files): + if tool in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): + if all(excl not in rootRealPath for excl in exclude_dirs): paths.append(os.path.join(root, tool)) break if (len(paths) == 0): - return "" + tdLog.exit(f"{tool} not found in {projPath}") + else: + tdLog.info(f"{tool} found in {paths[0]}") + return paths[0] + def starttaosd(self): binPath = self.getPath() diff --git a/tests/pytest/util/taosadapter.py b/tests/pytest/util/taosadapter.py index 1f1c38672e13..e053c6cb672e 100644 --- a/tests/pytest/util/taosadapter.py +++ b/tests/pytest/util/taosadapter.py @@ -82,7 +82,7 @@ def __init__(self): def init(self, path, remoteIP=""): self.path = path self.remoteIP = remoteIP - binPath = get_path() + "/../../../" + binPath = tdCom.get_path() + "/../../../" binPath = os.path.realpath(binPath) if path == "": @@ -111,7 +111,7 @@ def update_cfg(self, update_dict :dict): self.taosadapter_cfg_dict[key] = value def check_adapter(self): - if get_path(tool="taosadapter"): + if tdCom.get_path(tool="taosadapter"): return False else: return True @@ -154,7 +154,7 @@ def deploy(self, *update_cfg_dict): tdLog.debug(f"taosadapter is deployed and configured by {self.cfg_path}") def start(self): - bin_path = get_path(tool="taosadapter") + bin_path = tdCom.get_path(tool="taosadapter") if (bin_path == ""): tdLog.exit("taosadapter not found!") @@ -197,7 +197,7 @@ def start_taosadapter(self): """ use this method, must deploy taosadapter """ - bin_path = get_path(tool="taosadapter") + bin_path = tdCom.get_path(tool="taosadapter") if (bin_path == ""): tdLog.exit("taosadapter not found!") diff --git a/tests/system-test/99-TDcase/TD-32391.py b/tests/system-test/99-TDcase/TD-32391.py new file mode 100644 index 000000000000..60c0241c0c23 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-32391.py @@ -0,0 +1,52 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +class TDTestCase: + """ + test for TD-32391 + """ + clientCfgDict = {'debugFlag': 131} + updatecfgDict = { + "debugFlag" : "131", + "queryBufferSize" : 10240, + 'clientCfg' : clientCfgDict + } + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def test_td_32391(self): + tdLog.printNoPrefix("======== test TD-32391") + taosBen_bin = tdCom.get_path(tool="taosBenchmark") + if not taosBen_bin: + tdLog.exit("taosBenchmark not found!") + else: + tdLog.info(f"taosBenchmark found: {taosBen_bin}") + os.system(f"{taosBen_bin} -f ./99-TDcase/com_totest_stream.json") + tdSql.execute("create stream if not exists insert_energy_consmption fill_history 1 trigger window_close watermark 10s ignore expired 1 into test.energy_consumption_record tags(device_no varchar(64)) subtable(tbname) as select _wstart as ts_start ,_wend as ts_end,_wduration as ts_duration, (first(val) + last(val))/2*(_wduration/1000) as val, first(order_no) as order_no, first(production_no) as production_no, first(modal_no) as modal_no from test.meters partition by tbname,device_no count_window(2,1);") + tdSql.execute("create stream if not exists insert_oee_status fill_history 1 trigger window_close into test.oee_status tags(device_no varchar(64)) subtable(device_no) as select max(case point_no when 'STP0' then 1 when 'STP1' then 3 when 'STP2' then 2 when 'STP4' then 4 else null end) as status_index, first(order_no) as order_no, first(production_no) as production_no, first(modal_no) as modal_no from test.meters partition by device_no interval(2s);") + os.system(f"{taosBen_bin} -f ./99-TDcase/com_totest_stream_continue.json") + tdSql.query("select count(*) from test.meters") + + + + def run(self): + self.test_td_32391() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/99-TDcase/com_totest_stream.json b/tests/system-test/99-TDcase/com_totest_stream.json new file mode 100644 index 000000000000..06c2dd86039d --- /dev/null +++ b/tests/system-test/99-TDcase/com_totest_stream.json @@ -0,0 +1,73 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 20000, + "thread_count": 8, + "create_table_thread_count": 10, + "result_file": "./insert_res_mix.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "continue_if_fail": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "vgroups": 8, + "replica": 1, + "precision": "ms", + "stt_trigger": 1, + "minRows": 100, + "WAL_RETENTION_PERIOD": 10, + "maxRows": 4096 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "auto_create_table":"no", + "childtable_count": 1000, + "insert_rows": 1728, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "insert_interval": 0, + "timestamp_step": 900000, + "start_timestamp":"2022-09-01 10:00:00", + "disorder_ratio": 0, + "update_ratio": 0, + "delete_ratio": 0, + "continue_if_fail": "yes", + "disorder_fill_interval": 0, + "update_fill_interval": 0, + "generate_row_rule": 0, + "columns": [ + { "type": "binary", "name": "val", "len": 64}, + { "type": "binary", "name": "order_no", "len": 64}, + { "type": "binary", "name": "production_no", "len": 64}, + { "type": "binary", "name": "modal_no", "len": 64} + + ], + "tags": [ + { "type": "binary", "name": "device_no", "len": 64 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] }, + { "type": "int", "name": "channel_id", "max": 100, "min": 0}, + { "type": "binary", "name": "point_no", "len": 64 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"]}, + { "type": "int", "name": "datatype", "max": 100, "min": 0}, + { "type": "int", "name": "business_type", "max": 100, "min": 0}, + { "type": "binary", "name": "unit", "len": 16 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"]} + ] + } + ] + } + ] +} diff --git a/tests/system-test/99-TDcase/com_totest_stream_continue.json b/tests/system-test/99-TDcase/com_totest_stream_continue.json new file mode 100644 index 000000000000..a45fa7c721bb --- /dev/null +++ b/tests/system-test/99-TDcase/com_totest_stream_continue.json @@ -0,0 +1,73 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 20000, + "thread_count": 8, + "create_table_thread_count": 10, + "result_file": "./insert_res_mix.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "continue_if_fail": "yes", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "no", + "vgroups": 8, + "replica": 1, + "precision": "ms", + "stt_trigger": 1, + "minRows": 100, + "WAL_RETENTION_PERIOD": 10, + "maxRows": 4096 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "yes", + "auto_create_table":"no", + "childtable_count": 1000, + "insert_rows": 1728, + "childtable_prefix": "d", + "insert_mode": "stmt2", + "insert_interval": 0, + "timestamp_step": 900000, + "start_timestamp":"2022-09-01 10:00:00", + "disorder_ratio": 0, + "update_ratio": 0, + "delete_ratio": 0, + "continue_if_fail": "yes", + "disorder_fill_interval": 0, + "update_fill_interval": 0, + "generate_row_rule": 0, + "columns": [ + { "type": "binary", "name": "val", "len": 64}, + { "type": "binary", "name": "order_no", "len": 64}, + { "type": "binary", "name": "production_no", "len": 64}, + { "type": "binary", "name": "modal_no", "len": 64} + + ], + "tags": [ + { "type": "binary", "name": "device_no", "len": 64 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] }, + { "type": "int", "name": "channel_id", "max": 100, "min": 0}, + { "type": "binary", "name": "point_no", "len": 64 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"]}, + { "type": "int", "name": "datatype", "max": 100, "min": 0}, + { "type": "int", "name": "business_type", "max": 100, "min": 0}, + { "type": "binary", "name": "unit", "len": 16 ,"values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"]} + ] + } + ] + } + ] +} \ No newline at end of file