Skip to content

Commit

Permalink
feat: add setting to disable clickhouse handler (databendlabs#13965)
Browse files Browse the repository at this point in the history 
* feat: support settings enable_clickhouse_handler

* fix test case

* fix test case

* return 405

* delete dirty

* fix conflict
  • Loading branch information
@guojidan
guojidan authored Dec 14, 2023
1 parent ae52de2 commit 457baeb
Show file tree
Hide file tree
Showing 14 changed files with 104 additions and 73 deletions.
22 changes: 21 additions & 1 deletion src/query/service/src/servers/http/clickhouse_handler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use common_sql::plans::Plan;
use common_sql::Planner;
use futures::StreamExt;
use http::HeaderMap;
use http::StatusCode;
use log::debug;
use log::info;
use log::warn;
Expand Down Expand Up @@ -231,7 +232,6 @@ pub async fn clickhouse_handler_get(
headers: &HeaderMap,
) -> PoemResult<WithContentType<Body>> {
let root = Span::root(full_name!(), SpanContext::random());

async {
let session = ctx.upgrade_session(SessionType::ClickHouseHttpHandler)?;
if let Some(db) = &params.database {
Expand All @@ -247,6 +247,16 @@ pub async fn clickhouse_handler_get(
.set_batch_settings(&params.settings)
.map_err(BadRequest)?;

if !settings
.get_enable_clickhouse_handler()
.map_err(InternalServerError)?
{
return Err(poem::Error::from_string(
"default settings: enable_clickhouse_handler is 0".to_string(),
StatusCode::METHOD_NOT_ALLOWED,
));
}

let default_format = get_default_format(&params, headers).map_err(BadRequest)?;
let sql = params.query();
let mut planner = Planner::new(context.clone());
Expand Down Expand Up @@ -301,6 +311,16 @@ pub async fn clickhouse_handler_post(
.set_batch_settings(&params.settings)
.map_err(BadRequest)?;

if !settings
.get_enable_clickhouse_handler()
.map_err(InternalServerError)?
{
return Err(poem::Error::from_string(
"default settings: enable_clickhouse_handler is 0".to_string(),
StatusCode::METHOD_NOT_ALLOWED,
));
}

let default_format = get_default_format(&params, headers).map_err(BadRequest)?;
let mut sql = params.query();
if !sql.is_empty() {
Expand Down
2 changes: 1 addition & 1 deletion src/query/service/tests/it/servers/http/clickhouse_handler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ impl QueryBuilder {
}
let uri = uri.finish();

let uri = "/?".to_string() + &uri;
let uri = "/?enable_clickhouse_handler=1&".to_string() + &uri;
let uri = uri.parse::<Uri>().unwrap();
let (method, body) = match self.body {
None => (Method::GET, Body::empty()),
Expand Down
1 change: 1 addition & 0 deletions src/query/service/tests/it/storages/testdata/settings_table.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ DB.Table: 'system'.'settings', Table: settings-table_id:1, ver:0, Engine: System
| 'enable_aggregating_index_scan' | '1' | '1' | 'SESSION' | 'Enable scanning aggregating index data while querying.' | 'UInt64' |
| 'enable_bushy_join' | '0' | '0' | 'SESSION' | 'Enables generating a bushy join plan with the optimizer.' | 'UInt64' |
| 'enable_cbo' | '1' | '1' | 'SESSION' | 'Enables cost-based optimization.' | 'UInt64' |
| 'enable_clickhouse_handler' | '0' | '0' | 'SESSION' | 'Enables clickhouse handler.' | 'UInt64' |
| 'enable_distributed_compact' | '0' | '0' | 'SESSION' | 'Enable distributed execution of table compaction.' | 'UInt64' |
| 'enable_distributed_copy_into' | '1' | '1' | 'SESSION' | 'Enable distributed execution of copy into.' | 'UInt64' |
| 'enable_distributed_merge_into' | '0' | '0' | 'SESSION' | 'Enable distributed merge into.' | 'UInt64' |
Expand Down
7 changes: 7 additions & 0 deletions src/query/settings/src/settings_default.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ impl DefaultSettings {
let global_conf = GlobalConfig::try_get_instance();

let default_settings = HashMap::from([
("enable_clickhouse_handler", DefaultSettingValue {
value: UserSettingValue::UInt64(0),
desc: "Enables clickhouse handler.",
possible_values: None,
mode: SettingMode::Both,
range: None,
}),
("max_block_size", DefaultSettingValue {
value: UserSettingValue::UInt64(65536),
desc: "Sets the maximum byte size of a single data block that can be read.",
Expand Down
3 changes: 3 additions & 0 deletions src/query/settings/src/settings_getter_setter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@ impl Settings {
)))
}

pub fn get_enable_clickhouse_handler(&self) -> Result<bool> {
Ok(self.try_get_u64("enable_clickhouse_handler")? != 0)
}
// Get max_block_size.
pub fn get_max_block_size(&self) -> Result<u64> {
self.try_get_u64("max_block_size")
Expand Down
32 changes: 16 additions & 16 deletions tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,30 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../../../shell_env.sh

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select 1'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}?enable_clickhouse_handler=1" -d 'select 1'

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'select 1'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select 1'

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}//" -d 'select 1'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}//?enable_clickhouse_handler=1" -d 'select 1'

curl -s -u root: "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/ping"

curl -s -u root: "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/replicas_status"

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'select version()'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select version()'

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'drop database if exists db2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'create database db2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d "show databases like 'db%' format TabSeparatedWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'create table t2(a int not null)'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d "show TABLES LIKE 't%' format TabSeparatedWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'show create table t2 format TabSeparatedWithNamesAndTypes'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'desc t2 format TabSeparatedWithNamesAndTypes'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'insert into table t2 values(1)'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?" -d 'select * from db2.t2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'drop database if exists db2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'drop database if exists db2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'create database db2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "show databases like 'db%' format TabSeparatedWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2&enable_clickhouse_handler=1" -d 'create table t2(a int not null)'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2&enable_clickhouse_handler=1" -d "show TABLES LIKE 't%' format TabSeparatedWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2&enable_clickhouse_handler=1" -d 'show create table t2 format TabSeparatedWithNamesAndTypes'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2&enable_clickhouse_handler=1" -d 'desc t2 format TabSeparatedWithNamesAndTypes'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2&enable_clickhouse_handler=1" -d 'insert into table t2 values(1)'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select * from db2.t2'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'drop database if exists db2'

echo "--- default_format"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?default_format=TabSeparatedWithNamesAndTypes" -d 'select number as a from numbers(1)'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?default_format=TabSeparatedWithNamesAndTypes&enable_clickhouse_handler=1" -d 'select number as a from numbers(1)'

curl -s -u root: -H "X-CLICKHOUSE-FORMAT: TabSeparatedWithNamesAndTypes" -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d 'select number as a from numbers(1)'
curl -s -u root: -H "X-CLICKHOUSE-FORMAT: TabSeparatedWithNamesAndTypes" -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select number as a from numbers(1)'
26 changes: 13 additions & 13 deletions tests/suites/0_stateless/14_clickhouse_http_handler/14_0002_select_formats.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,40 +5,40 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)


echo "----csv"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select number, number::varchar, number + 1 from numbers(3) FORMAT CSV'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select number, number::varchar, number + 1 from numbers(3) FORMAT CSV'

echo "----tsv"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select number, number::varchar, number + 1 from numbers(3) FORMAT TSV'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select number, number::varchar, number + 1 from numbers(3) FORMAT TSV'

echo "----tsv header escape"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select regexp_like('fo\nfo', '^fo$') FORMAT TabSeparatedWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select regexp_like('fo\nfo', '^fo$') FORMAT TabSeparatedWithNamesAndTypes"

echo "----NDJSON"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select number, number::varchar from numbers(2) FORMAT NDJSON'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select number, number::varchar from numbers(2) FORMAT NDJSON'

echo "----JSONEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONEachRow"

echo "----JSONStringsEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONStringsEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONStringsEachRow"

echo "----JSONCompactEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRow"

echo "----JSONCompactEachRowWithNames"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRowWithNames"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRowWithNames"

echo "----JSONCompactEachRowWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRowWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactEachRowWithNamesAndTypes"

echo "----JSONCompactStringsEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRow"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRow"

echo "----JSONCompactStringsEachRowWithNames"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRowWithNames"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRowWithNames"

echo "----JSONCompactStringsEachRowWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRowWithNamesAndTypes"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSONCompactStringsEachRowWithNamesAndTypes"

echo "----JSON"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "select number, number::varchar from numbers(2) FORMAT JSON"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "select number, number::varchar from numbers(2) FORMAT JSON"
12 changes: 6 additions & 6 deletions tests/suites/0_stateless/14_clickhouse_http_handler/14_0003_http_auth.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../../../shell_env.sh

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'drop user if exists user1'
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'drop user if exists user1'

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "create user user1 identified by 'abc123'"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "create user user1 identified by 'abc123'"

curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d "grant select on *.* to 'user1'"
curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d "grant select on *.* to 'user1'"

curl -s -u user1:abc123 -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select 1 FORMAT CSV'
curl -s -u user1:abc123 -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select 1 FORMAT CSV'

curl -s -H 'X-ClickHouse-User: user1' -H 'X-ClickHouse-Key: abc123' -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}" -d 'select 1 FORMAT CSV'
curl -s -H 'X-ClickHouse-User: user1' -H 'X-ClickHouse-Key: abc123' -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" -d 'select 1 FORMAT CSV'

curl -s -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}?user=user1&password=abc123" -d 'select 1 FORMAT CSV'
curl -s -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?user=user1&password=abc123&enable_clickhouse_handler=1" -d 'select 1 FORMAT CSV'
Loading

0 comments on commit 457baeb

Please sign in to comment.