Add section on how to setup testing workspace

DEVELOP.md

@@ -31,6 +31,24 @@ python -m tdvt.tdvt
 git clone [email protected]:crate/cratedb-tableau-connector.git
 ```
 
-# Setting up the database and connector:
+# Setting up the testing workspace:
 
-Now that the environment is set up, we need to also set up the tableau test files and data.
+Now that the environment is set up, we need to also set up the tableau test files and data.
+
+Tdvt workflow is documented here: https://tableau.github.io/connector-plugin-sdk/docs/tdvt, it is pretty lengthy, you
+can follow the official tdvt docs or continue reading and do step by step.
+
+Note: There is also a [50 minute video by the tableau team ](https://www.youtube.com/watch?v=rAgnnByJIJA) .
+## Set up CrateDB.
+
+The quickest way to setup CrateDB is with docker https://cratedb.com/docs/guide/install/container/index.html
+```shell
+docker run --publish=4200:4200 --publish=5432:5432 --env CRATE_HEAP_SIZE=1g --pull=always crate
+```
+
+## Set up ``tdvt`` execution profile.
+
+1. Go to `./tests/config/tdvt/tdvt_override.ini` and put the path of your installation's `tabquerytool.exe`, for example
+for my in windows it is: `C:\Program Files\Tableau\Tableau 2024.3\bin\tabquerytool.exe`
+2. [Download](https://github.com/tableau/connector-plugin-sdk/tree/master/tests/datasets/TestV1) and Load the table `calcs` and `staples` to your CrateDB instance. You can use the script in `./data/setup_data.py`,
+it will create the tables and load the data automatically.

data/calcs.sql

@@ -0,0 +1,30 @@
+CREATE TABLE "Calcs"
+(
+    "key" VARCHAR(255),
+    "num0" DOUBLE PRECISION,
+    "num1" DOUBLE PRECISION,
+    "num2" DOUBLE PRECISION,
+    "num3" DOUBLE PRECISION,
+    "num4" DOUBLE PRECISION,
+    "str0" VARCHAR(255),
+    "str1" VARCHAR(255),
+    "str2" VARCHAR(255),
+    "str3" VARCHAR(255),
+    "int0" INTEGER,
+    "int1" INTEGER,
+    "int2" INTEGER,
+    "int3" INTEGER,
+    "bool0" BOOLEAN,
+    "bool1" BOOLEAN,
+    "bool2" BOOLEAN,
+    "bool3" BOOLEAN,
+    "date0" TIMESTAMP,
+    "date1" TIMESTAMP,
+    "date2" TIMESTAMP,
+    "date3" TIMESTAMP,
+    "time0" TIMESTAMP,
+    "time1" TIMESTAMP,
+    "datetime0" TIMESTAMP,
+    "datetime1" VARCHAR(255),
+    "zzz" VARCHAR(255)
+);

data/setup_data.py

@@ -0,0 +1,31 @@
+import requests
+import pathlib
+import logging
+import polars
+
+logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
+
+CRATEDB_URI_HTTP = 'http://192.168.88.251:4200' + '/_sql'
+CRATEDB_URI = 'crate://192.168.88.251:4200'
+CONNECTOR_SDK_URL = pathlib.Path(__file__).parent.parent / 'connector-plugin-sdk/tests/datasets/TestV1/'
+DATA_URL = pathlib.Path(__file__).parent.parent
+
+FILES_URL = [
+    {'name': 'Calcs',
+     'url': 'https://raw.githubusercontent.com/tableau/connector-plugin-sdk/refs/heads/master/tests/datasets/TestV1/Calcs_headers.csv', },
+    {'name': 'Staples',
+     'url': 'https://raw.githubusercontent.com/tableau/connector-plugin-sdk/refs/heads/master/tests/datasets/TestV1/Staples_utf8_headers.csv'}
+]
+
+for file in DATA_URL.iterdir():
+    if file.suffix == '.sql':
+        statement = file.read_text()
+        response = requests.post(CRATEDB_URI_HTTP, json={'stmt': statement})
+        logging.info(f'Tried creating {file.name}, response from CrateDB: {response.text}')
+
+# Download data and load it.
+for file in FILES_URL:
+    result = requests.get(file.get('url'))
+    df = polars.read_csv(result.content)
+    df.write_database(file.get('name'), connection=CRATEDB_URI, if_table_exists='append')
+

data/staples.sql

@@ -0,0 +1,53 @@
+CREATE TABLE "Staples"
+(
+    "Item Count" INTEGER NOT NULL,
+    "Ship Priority" VARCHAR(14) NOT NULL,
+    "Order Priority" VARCHAR(15) NOT NULL,
+    "Order Status" VARCHAR(13) NOT NULL,
+    "Order Quantity" DOUBLE PRECISION NOT NULL,
+    "Sales Total" DOUBLE PRECISION NOT NULL,
+    "Discount" DOUBLE PRECISION NOT NULL,
+    "Tax Rate" DOUBLE PRECISION NOT NULL,
+    "Ship Mode" VARCHAR(25) NOT NULL,
+    "Fill Time" DOUBLE PRECISION NOT NULL,
+    "Gross Profit" DOUBLE PRECISION NOT NULL,
+    "Price" NUMERIC(18,4) NOT NULL,
+    "Ship Handle Cost" NUMERIC(18,4) NOT NULL,
+    "Employee Name" VARCHAR(50) NOT NULL,
+    "Employee Dept" VARCHAR(4) NOT NULL,
+    "Manager Name" VARCHAR(255) NOT NULL,
+    "Employee Yrs Exp" DOUBLE PRECISION NOT NULL,
+    "Employee Salary" NUMERIC(18,4) NOT NULL,
+    "Customer Name" VARCHAR(50) NOT NULL,
+    "Customer State" VARCHAR(50) NOT NULL,
+    "Call Center Region" VARCHAR(25) NOT NULL,
+    "Customer Balance" DOUBLE PRECISION NOT NULL,
+    "Customer Segment" VARCHAR(25) NOT NULL,
+    "Prod Type1" VARCHAR(50) NOT NULL,
+    "Prod Type2" VARCHAR(50) NOT NULL,
+    "Prod Type3" VARCHAR(50) NOT NULL,
+    "Prod Type4" VARCHAR(50) NOT NULL,
+    "Product Name" VARCHAR(100) NOT NULL,
+    "Product Container" VARCHAR(25) NOT NULL,
+    "Ship Promo" VARCHAR(25) NOT NULL,
+    "Supplier Name" VARCHAR(25) NOT NULL,
+    "Supplier Balance" DOUBLE PRECISION NOT NULL,
+    "Supplier Region" VARCHAR(25) NOT NULL,
+    "Supplier State" VARCHAR(50) NOT NULL,
+    "Order ID" VARCHAR(10) NOT NULL,
+    "Order Year" INTEGER NOT NULL,
+    "Order Month" INTEGER NOT NULL,
+    "Order Day" INTEGER NOT NULL,
+    "Order Date" TIMESTAMP NOT NULL,
+    "Order Quarter" VARCHAR(2) NOT NULL,
+    "Product Base Margin" DOUBLE PRECISION NOT NULL,
+    "Product ID" VARCHAR(5) NOT NULL,
+    "Receive Time" DOUBLE PRECISION NOT NULL,
+    "Received Date" TIMESTAMP NOT NULL,
+    "Ship Date" TIMESTAMP NOT NULL,
+    "Ship Charge" NUMERIC(18,4) NOT NULL,
+    "Total Cycle Time" DOUBLE PRECISION NOT NULL,
+    "Product In Stock" VARCHAR(3) NOT NULL,
+    "PID" INTEGER NOT NULL,
+    "Market Segment" VARCHAR(25) NOT NULL
+);