Skip to content

Commit

Permalink
Merge pull request #283 from youngsofun/test
Browse files Browse the repository at this point in the history 
ci: test cluster behind nginx
  • Loading branch information
@youngsofun
youngsofun authored Oct 21, 2024
2 parents 558c070 + 61fa49d commit 1e1bb15
Show file tree
Hide file tree
Showing 16 changed files with 174 additions and 161 deletions.
7 changes: 7 additions & 0 deletions .github/actions/setup_databend_cluster/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ runs:
aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
- name: Start Nginx
shell: bash
run: |
docker run -d --network host --name nginx-lb \
-v ${{ github.workspace }}/scripts/ci/nginx.conf:/etc/nginx/nginx.conf:ro \
nginx
- name: Download binary and extract into target directory
shell: bash
run: |
Expand Down
9 changes: 8 additions & 1 deletion .github/workflows/test_cluster.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,14 @@ jobs:
with:
version: '1.2.629-nightly'
target: 'x86_64-unknown-linux-gnu'
- name: Run Maven clean deploy with release profile

- name: Test with conn to node 1
run: mvn test -DexcludedGroups=FLAKY
env:
MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

- name: Test with conn to nginx
run: mvn test -DexcludedGroups=FLAKY
env:
MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
DATABEND_TEST_CONN_PORT: 8010
2 changes: 2 additions & 0 deletions databend-client/src/main/java/com/databend/client/ClientSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public class ClientSettings {
public static final int DEFAULT_RETRY_ATTEMPTS = 5;
public static final String X_Databend_Query_ID = "X-DATABEND-QUERY-ID";
public static final String X_DATABEND_ROUTE_HINT = "X-DATABEND-ROUTE-HINT";
public static final String X_DATABEND_STAGE_NAME = "X-DATABEND-STAGE-NAME";
public static final String X_DATABEND_RELATIVE_PATH = "X-DATABEND-RELATIVE-PATH";
public static final String DatabendWarehouseHeader = "X-DATABEND-WAREHOUSE";
public static final String DatabendTenantHeader = "X-DATABEND-TENANT";
private final String host;
Expand Down
4 changes: 3 additions & 1 deletion databend-client/src/test/java/com/databend/client/TestClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@

@Test(timeOut = 10000)
public class TestClientIT {
static String port = System.getenv("DATABEND_TEST_CONN_PORT") != null ? System.getenv("DATABEND_TEST_CONN_PORT").trim() : "8000";

// please setup a local databend cluster before running this test, and create databend
private static final String DATABEND_HOST = "http://databend:[email protected]:8000";
private static final String DATABEND_HOST = "http://databend:[email protected]:" + port;
private static final String DATABASE = "default";

@Test(groups = {"it"})
Expand Down
6 changes: 4 additions & 2 deletions databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import static com.databend.client.ClientSettings.X_DATABEND_RELATIVE_PATH;
import static com.databend.client.ClientSettings.X_DATABEND_STAGE_NAME;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
Expand Down Expand Up @@ -53,8 +55,8 @@ private void uploadFromStream(InputStream inputStream, String stageName, String
.addFormDataPart("upload", name, new InputStreamRequestBody(null, inputStream, fileSize))
.build();
Headers headers = new Headers.Builder()
.add("stage_name", stageName)
.add("relative_path", relativePath)
.add(X_DATABEND_STAGE_NAME, stageName)
.add(X_DATABEND_RELATIVE_PATH, relativePath)
.build();

HttpUrl url = HttpUrl.get(this.uri);
Expand Down
55 changes: 19 additions & 36 deletions databend-jdbc/src/test/java/com/databend/jdbc/TestBasicDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
Expand All @@ -25,27 +24,11 @@

@Test(timeOut = 10000)
public class TestBasicDriver {
private Connection createConnection()
throws SQLException {
String url = "jdbc:databend://localhost:8000";
return DriverManager.getConnection(url, "databend", "databend");
}

private Connection createConnection(String database) throws SQLException {
String url = "jdbc:databend://localhost:8000/" + database;
return DriverManager.getConnection(url, "databend", "databend");
}

private Connection createConnection(String database, Properties p) throws SQLException {
String url = "jdbc:databend://localhost:8000/" + database;
return DriverManager.getConnection(url, p);
}

@BeforeTest
public void setUp()
throws SQLException {
// create table
Connection c = createConnection();
Connection c = Utils.createConnection();
c.createStatement().execute("drop database if exists test_basic_driver");
c.createStatement().execute("drop database if exists test_basic_driver_2");
c.createStatement().execute("create database test_basic_driver");
Expand All @@ -61,7 +44,7 @@ public void setUp()
@Test(groups = {"IT"})
public void testBasic()
throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
PaginationOptions p = connection.unwrap(DatabendConnection.class).getPaginationOptions();
Assert.assertEquals(p.getWaitTimeSecs(), PaginationOptions.getDefaultWaitTimeSec());
Assert.assertEquals(p.getMaxRowsInBuffer(), PaginationOptions.getDefaultMaxRowsInBuffer());
Expand All @@ -83,15 +66,15 @@ public void testBasic()
@Test(groups = {"IT"})
public void testExecuteInvalidSql() {
assertThrows(SQLException.class, () -> {
try (Connection connection = createConnection();
try (Connection connection = Utils.createConnection();
Statement statement = connection.createStatement()) {
statement.execute("create tabl xxx (a int, b varchar)");
}
});
}

public void testSchema() {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
PaginationOptions p = connection.unwrap(DatabendConnection.class).getPaginationOptions();
Assert.assertEquals(p.getWaitTimeSecs(), PaginationOptions.getDefaultWaitTimeSec());
Assert.assertEquals(p.getMaxRowsInBuffer(), PaginationOptions.getDefaultMaxRowsInBuffer());
Expand Down Expand Up @@ -122,13 +105,13 @@ public void testCreateUserFunction() throws SQLException {
" return i+k;\n" +
"}\n" +
"$$;";
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.execute(s);
} catch (SQLException throwables) {
throwables.printStackTrace();
}
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.execute("select add_plus(1,2)");
ResultSet r = statement.getResultSet();
Expand All @@ -141,7 +124,7 @@ public void testCreateUserFunction() throws SQLException {

@Test
public void TestMergeinto() throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.execute("CREATE TABLE IF NOT EXISTS test_basic_driver.target_table (\n" +
" ID INT,\n" +
Expand Down Expand Up @@ -184,7 +167,7 @@ public void TestMergeinto() throws SQLException {

@Test
public void testWriteDouble() throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.execute("CREATE TABLE IF NOT EXISTS test_basic_driver.table_double (\n" +
" ID INT,\n" +
Expand Down Expand Up @@ -216,7 +199,7 @@ public void testWriteDouble() throws SQLException {

@Test
public void testDefaultSelectNullValue() throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.executeQuery("SELECT a,b from test_basic_driver.table_with_null");
ResultSet r = statement.getResultSet();
Expand All @@ -231,7 +214,7 @@ public void testDefaultSelectNullValue() throws SQLException {
@Test(groups = {"IT"})
public void testQueryUpdateCount()
throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
DatabendStatement statement = (DatabendStatement) connection.createStatement();
statement.execute("SELECT version()");
ResultSet r = statement.getResultSet();
Expand All @@ -247,11 +230,11 @@ public void testBasicWithProperties() throws SQLException {
p.setProperty("wait_time_secs", "10");
p.setProperty("max_rows_in_buffer", "100");
p.setProperty("max_rows_per_page", "100");
p.setProperty("user", "databend");
p.setProperty("password", "databend");
p.setProperty("user", Utils.getUsername());
p.setProperty("password", Utils.getPassword());

//INFO databend_query::servers::http::v1::http_query_handlers: receive http query: HttpQueryRequest { session_id: None, session: Some(HttpSessionConf { database: Some("test_basic_driver"), keep_server_session_secs: None, settings: None }), sql: "SELECT 1", pagination: PaginationConf { wait_time_secs: 10, max_rows_in_buffer: 100, max_rows_per_page: 100 }, string_fields: true, stage_attachment: None }
try (Connection connection = createConnection("test_basic_driver", p)) {
try (Connection connection = Utils.createConnection("test_basic_driver", p)) {
PaginationOptions options = connection.unwrap(DatabendConnection.class).getPaginationOptions();
Assert.assertEquals(options.getWaitTimeSecs(), 10);
Assert.assertEquals(options.getMaxRowsInBuffer(), 100);
Expand All @@ -267,7 +250,7 @@ public void testBasicWithProperties() throws SQLException {
@Test
public void testPrepareStatementQuery() throws SQLException {
String sql = "SELECT number from numbers(100) where number = ? or number = ?";
Connection conn = createConnection("test_basic_driver");
Connection conn = Utils.createConnection("test_basic_driver");
try (PreparedStatement statement = conn.prepareStatement(sql)) {
statement.setInt(1, 1);
statement.setInt(2, 2);
Expand All @@ -281,7 +264,7 @@ public void testPrepareStatementQuery() throws SQLException {
@Test(groups = {"IT"})
public void testBasicWithDatabase()
throws SQLException {
try (Connection connection = createConnection("test_basic_driver")) {
try (Connection connection = Utils.createConnection("test_basic_driver")) {
Statement statement = connection.createStatement();
statement.execute("SELECT i from table1");
ResultSet r = statement.getResultSet();
Expand All @@ -305,7 +288,7 @@ public void testBasicWithDatabase()
@Test(groups = {"IT"})
public void testUpdateSession()
throws SQLException {
try (Connection connection = createConnection("test_basic_driver")) {
try (Connection connection = Utils.createConnection("test_basic_driver")) {
connection.createStatement().execute("set max_threads=1");
connection.createStatement().execute("use test_basic_driver_2");
DatabendSession session = connection.unwrap(DatabendConnection.class).getSession();
Expand All @@ -316,7 +299,7 @@ public void testUpdateSession()

@Test(groups = {"IT"})
public void testResultException() {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
Statement statement = connection.createStatement();
ResultSet r = statement.executeQuery("SELECT 1e189he 198h");

Expand All @@ -329,7 +312,7 @@ public void testResultException() {
@Test(groups = {"IT"})
public void testSelectWithPreparement()
throws SQLException {
try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
connection.createStatement().execute("create or replace table test_basic_driver.table_time(t timestamp, d date, ts timestamp)");
connection.createStatement().execute("insert into test_basic_driver.table_time values('2021-01-01 00:00:00', '2021-01-01', '2021-01-01 00:00:00')");
PreparedStatement statement = connection.prepareStatement("SELECT * from test_basic_driver.table_time where t < ? and d < ? and ts < ?");
Expand All @@ -346,7 +329,7 @@ public void testSelectWithPreparement()
public void testSelectGeometry() throws SQLException, ParseException {
// skip due to failed cluster tests

try (Connection connection = createConnection()) {
try (Connection connection = Utils.createConnection()) {
connection.createStatement().execute("set enable_geo_create_table=1");
connection.createStatement().execute("CREATE or replace table cities ( id INT, name VARCHAR NOT NULL, location GEOMETRY);");
connection.createStatement().execute("INSERT INTO cities (id, name, location) VALUES (1, 'New York', 'POINT (-73.935242 40.73061))');");
Expand Down
Loading

0 comments on commit 1e1bb15

Please sign in to comment.