Skip to content

Commit

Permalink
[TD-14540]<impr>: taosdump code cleanup for2.4 (#11316)
Browse files Browse the repository at this point in the history 
* [TD-14540]<impr>: taosdump code cleanup

for 2.4

* update taos-tools

for 2.4

* refine 5-taos-tools/taosdump/taosdumpTestInspect.py

* [TD-14540]<impr>: refine tools/taosdumpTest2.py

for 2.4

* pick up tools/taosdumpTest2.py from develop branch
  • Loading branch information
@sangshuduo
sangshuduo authored Apr 8, 2022
1 parent 9d4f88c commit 21b4528
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 49 deletions.
78 changes: 53 additions & 25 deletions tests/pytest/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,16 @@
tdLog.printNoPrefix('-c Test Cluster Flag')
tdLog.printNoPrefix('-g valgrind Test Flag')
tdLog.printNoPrefix('-r taosd restart test')
tdLog.printNoPrefix('-w taos on windows')
tdLog.printNoPrefix('-w taos on windows')
sys.exit(0)

if key in ['-r', '--restart']:
if key in ['-r', '--restart']:
restart = True

if key in ['-f', '--file']:
fileName = value

fileName = os.path.normpath(value)
print(fileName)

if key in ['-d', '--docker']:
fileName = os.path.normpath(value)
docker = True
Expand All @@ -90,7 +91,7 @@
valgrind = 1

if key in ['-s', '--stop']:
stop = 1
stop = 1

if key in ['-w', '--windows']:
windows = 1
Expand Down Expand Up @@ -124,13 +125,13 @@
time.sleep(2)

tdLog.info('stop All dnodes')

if masterIp == "":
host = '127.0.0.1'
else:
host = masterIp
host = masterIp

tdLog.info("Procedures for tdengine deployed in %s" % (host))
tdLog.info("Procedures for tdengine deployed in %s" % (host))
if docker:
tdCases.logSql(logSql)
tdLog.info("Procedures for testing self-deployment")
Expand Down Expand Up @@ -176,13 +177,40 @@
tdLog.info("Procedures for testing self-deployment")
td_clinet = TDSimClient("C:\\TDengine")
td_clinet.deploy()
remote_conn = Connection("root@%s"%host)
with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
remote_conn.run("python3 ./test.py")
if masterIp == "" or masterIp == "localhost":
tdDnodes.init(deployPath)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.stopAll()
is_test_framework = 0
key_word = 'tdCases.addWindows'
try:
if key_word in open(fileName).read():
is_test_framework = 1
except:
pass
if is_test_framework:
moduleName = fileName.replace(".py", "").replace(os.sep, ".")
uModule = importlib.import_module(moduleName)
try:
ucase = uModule.TDTestCase()
tdDnodes.deploy(1,ucase.updatecfgDict)
except :
tdDnodes.deploy(1,{})
else:
pass
tdDnodes.deploy(1,{})
tdDnodes.startWin(1)
else:
remote_conn = Connection("root@%s"%host)
with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
remote_conn.run("python3 ./test.py")
conn = taos.connect(
host="%s"%(host),
host="%s" % (host),
config=td_clinet.cfgDir)
tdCases.runOneWindows(conn, fileName)
tdCases.logSql(logSql)

else:
tdDnodes.init(deployPath)
tdDnodes.setTestCluster(testCluster)
Expand All @@ -193,23 +221,21 @@
try:
if key_word in open(fileName).read():
is_test_framework = 1
except:
except BaseException:
pass
if is_test_framework:
moduleName = fileName.replace(".py", "").replace("/", ".")
moduleName = fileName.replace(".py", "").replace(os.sep, ".")
uModule = importlib.import_module(moduleName)
try:
ucase = uModule.TDTestCase()
tdDnodes.deploy(1,ucase.updatecfgDict)
except :
tdDnodes.deploy(1,{})
tdDnodes.deploy(1, ucase.updatecfgDict)
except BaseException:
tdDnodes.deploy(1, {})
else:
pass
tdDnodes.deploy(1,{})
tdDnodes.deploy(1, {})
tdDnodes.start(1)



tdCases.logSql(logSql)

if testCluster:
Expand All @@ -226,18 +252,20 @@
if fileName == "all":
tdCases.runAllLinux(conn)
else:
tdCases.runOneWindows(conn, fileName)
tdCases.runOneLinux(conn, fileName)
if restart:
if fileName == "all":
tdLog.info("not need to query ")
else:
else:
sp = fileName.rsplit(".", 1)
if len(sp) == 2 and sp[1] == "py":
tdDnodes.stopAll()
tdDnodes.start(1)
time.sleep(1)
conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
tdLog.info("Procedures for tdengine deployed in %s" % (host))
time.sleep(1)
conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
tdLog.info(
"Procedures for tdengine deployed in %s" %
(host))
tdLog.info("query test after taosd restart")
tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
else:
Expand Down
50 changes: 26 additions & 24 deletions tests/pytest/tools/taosdumpTest2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

# -*- coding: utf-8 -*-

from logging.config import dictConfig
import sys
import os
from util.log import *
Expand All @@ -22,30 +23,33 @@


class TDTestCase:

def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)

self.ts = 1601481600000
self.numberOfTables = 1
self.numberOfRecords = 15000
self.tmpdir = "tmp"

def getBuildPath(self):
def getPath(self, tool="taosdump"):
selfPath = os.path.dirname(os.path.realpath(__file__))

if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]

paths = []
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
if ((tool) in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
paths.append(os.path.join(root, tool))
break
return buildPath
if (len(paths) == 0):
return ""
return paths[0]

def generateString(self, length):
chars = string.ascii_uppercase + string.ascii_lowercase
Expand All @@ -55,6 +59,11 @@ def generateString(self, length):
return v

def run(self):
if not os.path.exists("./taosdumptest/tmp"):
os.makedirs("./taosdumptest/tmp")
else:
print("directory exists")

tdSql.prepare()

tdSql.execute("create table st(ts timestamp, c1 timestamp, c2 int, c3 bigint, c4 float, c5 double, c6 binary(8), c7 smallint, c8 tinyint, c9 bool, c10 nchar(8)) tags(t1 int)")
Expand All @@ -70,29 +79,23 @@ def run(self):
break
tdSql.execute(sql)

buildPath = self.getBuildPath()
if (buildPath == ""):
binPath = self.getPath()
if (binPath == ""):
tdLog.exit("taosdump not found!")
else:
tdLog.info("taosdump found in %s" % buildPath)
binPath = buildPath + "/build/bin/"

if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
else:
print("directory exists")
os.system("rm -rf %s" % self.tmpdir)
os.makedirs(self.tmpdir)
tdLog.info("taosdump found in %s" % binPath)

os.system("rm ./taosdumptest/tmp/*.sql")
os.system("rm ./taosdumptest/tmp/*.avro*")
os.system(
"%staosdump --databases db -o %s -y" %
(binPath, self.tmpdir))
"%s --databases db -o ./taosdumptest/tmp " %
binPath)

tdSql.execute("drop database db")
tdSql.query("show databases")
tdSql.checkRows(0)

os.system("%staosdump -i %s -y" % (binPath, self.tmpdir))
os.system("%s -i ./taosdumptest/tmp -y" % binPath)

tdSql.query("show databases")
tdSql.checkRows(1)
Expand All @@ -105,7 +108,6 @@ def run(self):

tdSql.query("select count(*) from t1")
tdSql.checkData(0, 0, self.numberOfRecords)
os.system("rm -rf %s" % self.tmpdir)

# test case for TS-1225
tdSql.execute("create database test")
Expand All @@ -118,15 +120,15 @@ def run(self):
self.generateString(16374),
self.generateString(16374)))

os.system("rm /tmp/*.sql")
os.system("rm /tmp/*.avro*")
os.system("%staosdump -D test -o /tmp -y" % binPath)
os.system("rm ./taosdumptest/tmp/*.sql")
os.system("rm ./taosdumptest/tmp/*.avro*")
os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath)

tdSql.execute("drop database test")
tdSql.query("show databases")
tdSql.checkRows(1)

os.system("%staosdump -i /tmp -y" % binPath)
os.system("%s -i ./taosdumptest/tmp -y" % binPath)

tdSql.execute("use test")
tdSql.error("show vnodes '' ")
Expand Down

0 comments on commit 21b4528

Please sign in to comment.