fix: taosbenchmark rest mode in json (taosdata#513)

* fix: taosbenchmark rest mode in json * fix: use default port if not inputed
facetosea · Dec 12, 2022 · 121c8a3 · 121c8a3
1 parent 295cdcd
commit 121c8a3
Show file tree

Hide file tree

Showing 15 changed files with 126 additions and 31 deletions.
diff --git a/.github/workflows/3.0-coveralls.yml b/.github/workflows/3.0-coveralls.yml
@@ -70,7 +70,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-macos-release.yml b/.github/workflows/3.0-macos-release.yml
@@ -64,7 +64,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml
@@ -133,7 +133,7 @@ jobs:
             apt install -y cmake build-essential git libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev > /dev/null
 
             echo "clone TDengine 3.0 on ${{ steps.setup.outputs.uname }}"
-            git clone --branch fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1
+            git clone --branch fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1
 
             echo "build TDengine 3.0 on ${{ steps.setup.outputs.uname }}"
             cd TDengine || exit 1

diff --git a/.github/workflows/3.0-taosBenchmark-debug.yml b/.github/workflows/3.0-taosBenchmark-debug.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-taosBenchmark-release.yml b/.github/workflows/3.0-taosBenchmark-release.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-taosdump-debug.yml b/.github/workflows/3.0-taosdump-debug.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-taosdump-release-ws.yml b/.github/workflows/3.0-taosdump-release-ws.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-taosdump-release.yml b/.github/workflows/3.0-taosdump-release.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml
@@ -62,7 +62,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TS-2238-taosbenchmark-rest-taosc'
+          ref: 'fix/sangshuduo/TS-2253-taosbenchmark-rest-mode-in-json'
 
       - name: create debug directory
         if: |

diff --git a/example/insert-rest.json b/example/insert-rest.json
@@ -0,0 +1,86 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6041,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "thread_count": 4,
+    "create_table_thread_count": 2,
+    "insert_mode": "rest",
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 100,
+    "num_of_records_per_req": 100,
+    "prepared_rand": 10000,
+    "chinese": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "test",
+                "drop": "yes",
+                "replica": 1,
+                "precision": "ms",
+                "keep": 3650,
+                "minRows": 100,
+                "maxRows": 4096,
+                "comp": 2
+            },
+            "super_tables": [
+                {
+                    "name": "meters",
+                    "child_table_exists": "no",
+                    "childtable_count": 1,
+                    "childtable_prefix": "d",
+                    "escape_character": "yes",
+                    "auto_create_table": "no",
+                    "batch_create_tbl_num": 5,
+                    "data_source": "rand",
+                    "insert_mode": "rest",
+                    "non_stop_mode": "no",
+                    "line_protocol": "line",
+                    "insert_rows": 10,
+                    "childtable_limit": 10,
+                    "childtable_offset": 100,
+                    "interlace_rows": 0,
+                    "insert_interval": 0,
+                    "partial_col_num": 0,
+                    "disorder_ratio": 0,
+                    "disorder_range": 1000,
+                    "timestamp_step": 10,
+                    "start_timestamp": "2022-10-01 00:00:00.000",
+                    "tags_file": "",
+                    "columns": [
+                        {
+                            "type": "FLOAT",
+                            "name": "current",
+                            "count": 1,
+                            "max": 12,
+                            "min": 8
+                        },
+                        { "type": "INT", "name": "voltage", "max": 225, "min": 215 },
+                        { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 }
+                    ],
+                    "tags": [
+                        {
+                            "type": "TINYINT",
+                            "name": "groupid",
+                            "max": 10,
+                            "min": 1
+                        },
+                        {
+                            "name": "location",
+                            "type": "BINARY",
+                            "len": 16,
+                            "values": ["San Francisco", "Los Angles", "San Diego",
+                                "San Jose", "Palo Alto", "Campbell", "Mountain View",
+                                "Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
diff --git a/inc/bench.h b/inc/bench.h
@@ -141,6 +141,7 @@
 
 #define DEFAULT_CHILDTABLES    10000
 #define DEFAULT_PORT           6030
+#define DEFAULT_REST_PORT      6041
 #define DEFAULT_DATABASE       "test"
 #define DEFAULT_TB_PREFIX      "d"
 #define DEFAULT_OUTPUT         "./output.txt"
@@ -576,6 +577,7 @@ typedef struct SArguments_S {
     uint16_t            port;
     bool                host_auto;
     bool                port_auto;
+    bool                port_inputed;
     bool                cfg_inputed;
     uint16_t            telnet_tcp_port;
     char *              user;

diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c
@@ -314,16 +314,20 @@ static int32_t bench_parse_single_opt(int32_t key, char* arg) {
             if (!toolsIsStringNumber(arg)) {
                 errorPrintReqArg2("taosBenchmark", "P");
             }
-
             g_arguments->port = atoi(arg);
             if (g_arguments->port <= 0) {
                 errorPrint(
                            "Invalid -P: %s, will auto set to default(6030)\n",
                            arg);
-                g_arguments->port = DEFAULT_PORT;
+                if (REST_IFACE == g_arguments->iface) {
+                    g_arguments->port = DEFAULT_REST_PORT;
+                } else {
+                    g_arguments->port = DEFAULT_PORT;
+                }
             } else {
                 g_arguments->port_auto = false;
             }
+            g_arguments->port_inputed = true;
             break;
 
         case 'I':
@@ -334,6 +338,9 @@ static int32_t bench_parse_single_opt(int32_t key, char* arg) {
             } else if (0 == strcasecmp(arg, "rest")) {
                 stbInfo->iface = REST_IFACE;
                 g_arguments->nthreads_auto = false;
+                if (false == g_arguments->port_inputed) {
+                    g_arguments->port = DEFAULT_REST_PORT;
+                }
             } else if (0 == strcasecmp(arg, "sml")) {
                 stbInfo->iface = SML_IFACE;
             } else {
@@ -852,6 +859,7 @@ void init_argument() {
     g_arguments->host = NULL;
     g_arguments->host_auto = true;
     g_arguments->port = DEFAULT_PORT;
+    g_arguments->port_inputed = false;
     g_arguments->port_auto = true;
     g_arguments->telnet_tcp_port = TELNET_TCP_PORT;
     g_arguments->user = TSDB_DEFAULT_USER;

diff --git a/src/benchInsert.c b/src/benchInsert.c
@@ -670,6 +670,7 @@ static void *createTable(void *sarg) {
         len = 0;
 
         int ret = 0;
+        debugPrint("creating table: %s\n", pThreadInfo->buffer);
         if (REST_IFACE == stbInfo->iface) {
             ret = queryDbExecRest(pThreadInfo->buffer,
                                   database->dbName,
@@ -2544,6 +2545,13 @@ int insertTestProcess() {
     encode_base_64();
 
     for (int i = 0; i < g_arguments->databases->size; ++i) {
+        if (REST_IFACE == g_arguments->iface) {
+            if (0 != convertServAddr(g_arguments->iface,
+                                     false,
+                                     1)) {
+                return -1;
+            }
+        }
         SDataBase * database = benchArrayGet(g_arguments->databases, i);
 
         if (database->drop && !(g_arguments->supplementInsert)) {

diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c
@@ -880,6 +880,14 @@ static int getMetaFromInsertJsonFile(tools_cJSON *json) {
         g_arguments->insert_interval = top_insertInterval->valueint;
     }
 
+    tools_cJSON *insert_mode = tools_cJSON_GetObjectItem(json, "insert_mode");
+    if (insert_mode && insert_mode->type == tools_cJSON_String
+            && insert_mode->valuestring != NULL) {
+        if (0 == strcasecmp(insert_mode->valuestring, "rest")) {
+            g_arguments->iface = REST_IFACE;
+        }
+    }
+
     tools_cJSON *answerPrompt =
         tools_cJSON_GetObjectItem(json, "confirm_parameter_prompt");  // yes, no,
     if (answerPrompt && answerPrompt->type == tools_cJSON_String

diff --git a/src/benchUtil.c b/src/benchUtil.c
@@ -925,26 +925,9 @@ int convertServAddr(int iface, bool tcp, int protocol) {
             }
         } else {
             if (convertHostToServAddr(g_arguments->host,
-                        g_arguments->port + TSDB_PORT_HTTP,
-                        &(g_arguments->serv_addr))) {
-                errorPrint("%s\n", "convert host to server address");
-                return -1;
-            }
-        }
-    }
-    if (iface == REST_IFACE || iface == SML_REST_IFACE) {
-        if (tcp
-                && iface == SML_REST_IFACE
-                && protocol == TSDB_SML_TELNET_PROTOCOL) {
-            if (convertHostToServAddr(g_arguments->host,
-                        g_arguments->telnet_tcp_port,
-                        &(g_arguments->serv_addr))) {
-                errorPrint("%s\n", "convert host to server address");
-                return -1;
-            }
-        } else {
-            if (convertHostToServAddr(g_arguments->host,
-                        g_arguments->port + TSDB_PORT_HTTP,
+                        (g_arguments->port_inputed)?
+                                      g_arguments->port:
+                                      DEFAULT_REST_PORT,
                         &(g_arguments->serv_addr))) {
                 errorPrint("%s\n", "convert host to server address");
                 return -1;