Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implemented query for schema related Information schema & Fixed the bug that the deviceEntry get from cache does not reserve query memory #14610

Merged
merged 43 commits into from
Jan 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
cbba50f
partial
Caideyipi Dec 31, 2024
274e051
Update InformationSchemaContentSupplierFactory.java
Caideyipi Dec 31, 2024
f09a278
Update InformationSchemaContentSupplierFactory.java
Caideyipi Dec 31, 2024
b74cd7c
partial
Caideyipi Dec 31, 2024
b3df323
partial
Caideyipi Dec 31, 2024
68fcc05
Update InformationSchemaContentSupplierFactory.java
Caideyipi Dec 31, 2024
e926fb6
Update DataNodeLocationSupplierFactory.java
Caideyipi Dec 31, 2024
13f2087
partial
Caideyipi Dec 31, 2024
aa3da5a
tempo
Caideyipi Dec 31, 2024
3919e5c
Update DescTable4InformationSchemaResp.java
Caideyipi Dec 31, 2024
f9d95db
partial
Caideyipi Dec 31, 2024
61a74fe
Merge branch 'master' of https://github.com/apache/iotdb into informa…
Caideyipi Dec 31, 2024
6ace165
partial
Caideyipi Dec 31, 2024
f26e42f
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
e250d32
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
db6d72b
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
0c61779
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
cadb359
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
375a716
partial
Caideyipi Jan 2, 2025
8421d3d
Update InformationSchemaUtils.java
Caideyipi Jan 2, 2025
5769ee6
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
1126689
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
ea5ffe4
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
d2b57db
Update InformationSchemaContentSupplierFactory.java
Caideyipi Jan 2, 2025
d113fd2
Update IoTDBDatabaseIT.java
Caideyipi Jan 2, 2025
a5c15f5
Update IoTDBDatabaseIT.java
Caideyipi Jan 2, 2025
f9d5447
Merge branch 'master' of https://github.com/apache/iotdb into informa…
Caideyipi Jan 2, 2025
866ef41
Update IoTDBDatabaseIT.java
Caideyipi Jan 2, 2025
94ad5ca
partial
Caideyipi Jan 2, 2025
0ec0ab7
Update IoTDBDatabaseIT.java
Caideyipi Jan 2, 2025
6c7e633
Update IoTDBDatabaseIT.java
Caideyipi Jan 2, 2025
dd79457
end
Caideyipi Jan 2, 2025
c8058c3
Update StatementAnalyzer.java
Caideyipi Jan 2, 2025
1e4e3f6
Merge branch 'master' of https://github.com/apache/iotdb into informa…
Caideyipi Jan 2, 2025
7483620
Update TableDeviceSchemaFetcher.java
Caideyipi Jan 2, 2025
186bba7
Merge remote-tracking branch 'upstream/master' into information_schema
Caideyipi Jan 7, 2025
e95dcc3
Refactor
Caideyipi Jan 8, 2025
2d96de7
Update TableDeviceSchemaFetcher.java
Caideyipi Jan 8, 2025
41532e6
Update TableDeviceSchemaFetcher.java
Caideyipi Jan 8, 2025
3276a6a
Fix
Caideyipi Jan 8, 2025
310716c
Deletion
Caideyipi Jan 9, 2025
2a2d636
Merge branch 'master' of https://github.com/apache/iotdb into informa…
Caideyipi Jan 9, 2025
986539e
spotless
Caideyipi Jan 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,47 @@ public void testInformationSchema() throws SQLException {
"datanode_id,INT32,ATTRIBUTE,",
"elapsed_time,FLOAT,ATTRIBUTE,",
"statement,STRING,ATTRIBUTE,")));

// Test table query
statement.execute("create database test");
statement.execute("create table test.test (a tag, b attribute, c int32)");

TestUtils.assertResultSetEqual(
statement.executeQuery("select * from databases"),
"database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,",
new HashSet<>(
Arrays.asList(
"information_schema,INF,null,null,null,null,null,",
"test,INF,1,1,604800000,0,0,")));
TestUtils.assertResultSetEqual(
statement.executeQuery("show devices from tables where status = 'USING'"),
"database,table_name,ttl(ms),status,",
new HashSet<>(
Arrays.asList(
"information_schema,databases,INF,USING,",
"information_schema,tables,INF,USING,",
"information_schema,columns,INF,USING,",
"information_schema,queries,INF,USING,",
"test,test,INF,USING,")));
TestUtils.assertResultSetEqual(
statement.executeQuery("count devices from tables where status = 'USING'"),
"count(devices),",
Collections.singleton("5,"));
TestUtils.assertResultSetEqual(
statement.executeQuery(
"select * from columns where table_name = 'queries' or database = 'test'"),
"database,table_name,column_name,datatype,category,status,",
new HashSet<>(
Arrays.asList(
"information_schema,queries,query_id,STRING,TAG,USING,",
"information_schema,queries,start_time,TIMESTAMP,ATTRIBUTE,USING,",
"information_schema,queries,datanode_id,INT32,ATTRIBUTE,USING,",
"information_schema,queries,elapsed_time,FLOAT,ATTRIBUTE,USING,",
"information_schema,queries,statement,STRING,ATTRIBUTE,USING,",
"test,test,time,TIMESTAMP,TIME,USING,",
"test,test,a,STRING,TAG,USING,",
"test,test,b,STRING,ATTRIBUTE,USING,",
"test,test,c,INT32,FIELD,USING,")));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ public enum ConfigPhysicalPlanType {
PreDeleteColumn((short) 860),
CommitDeleteColumn((short) 861),
DescTable((short) 862),
ShowTable4InformationSchema((short) 863),
DescTable4InformationSchema((short) 864),

/** Deprecated types for sync, restored them for upgrade. */
@Deprecated
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.read.table;

import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan;

public class DescTable4InformationSchemaPlan extends ConfigPhysicalReadPlan {
public DescTable4InformationSchemaPlan() {
super(ConfigPhysicalPlanType.DescTable4InformationSchema);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.read.table;

import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan;

public class ShowTable4InformationSchemaPlan extends ConfigPhysicalReadPlan {

public ShowTable4InformationSchemaPlan() {
super(ConfigPhysicalPlanType.ShowTable4InformationSchema);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.response.table;

import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TTableColumnInfo;
import org.apache.iotdb.consensus.common.DataSet;

import java.util.Map;

public class DescTable4InformationSchemaResp implements DataSet {
private final TSStatus status;
private final Map<String, Map<String, TTableColumnInfo>> tableColumnInfoMap;

public DescTable4InformationSchemaResp(
final TSStatus status, final Map<String, Map<String, TTableColumnInfo>> tableColumnInfoMap) {
this.status = status;
this.tableColumnInfoMap = tableColumnInfoMap;
}

public TDescTable4InformationSchemaResp convertToTDescTable4InformationSchemaResp() {
return new TDescTable4InformationSchemaResp(status).setTableColumnInfoMap(tableColumnInfoMap);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.response.table;

import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TTableInfo;
import org.apache.iotdb.consensus.common.DataSet;

import java.util.List;
import java.util.Map;

public class ShowTable4InformationSchemaResp implements DataSet {
private final TSStatus status;
private final Map<String, List<TTableInfo>> databaseTableInfoMap;

public ShowTable4InformationSchemaResp(
final TSStatus status, final Map<String, List<TTableInfo>> databaseTableInfoMap) {
this.status = status;
this.databaseTableInfoMap = databaseTableInfoMap;
}

public TShowTable4InformationSchemaResp convertToTShowTable4InformationSchemaResp() {
return new TShowTable4InformationSchemaResp(status)
.setDatabaseTableInfoMap(databaseTableInfoMap);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq;
Expand Down Expand Up @@ -218,6 +219,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq;
Expand Down Expand Up @@ -2650,6 +2652,14 @@ public TShowTableResp showTables(final String database, final boolean isDetails)
: new TShowTableResp(status);
}

@Override
public TShowTable4InformationSchemaResp showTables4InformationSchema() {
final TSStatus status = confirmLeader();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? clusterSchemaManager.showTables4InformationSchema()
: new TShowTable4InformationSchemaResp(status);
}

@Override
public TDescTableResp describeTable(
final String database, final String tableName, final boolean isDetails) {
Expand All @@ -2659,6 +2669,14 @@ public TDescTableResp describeTable(
: new TDescTableResp(status);
}

@Override
public TDescTable4InformationSchemaResp describeTable4InformationSchema() {
final TSStatus status = confirmLeader();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? clusterSchemaManager.describeTables4InformationSchema()
: new TDescTable4InformationSchemaResp(status);
}

@Override
public TFetchTableResp fetchTables(final Map<String, Set<String>> fetchTableMap) {
final TSStatus status = confirmLeader();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq;
Expand Down Expand Up @@ -141,6 +142,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp;
Expand Down Expand Up @@ -843,8 +845,12 @@ TDataPartitionTableResp getOrCreateDataPartition(

TShowTableResp showTables(final String database, final boolean isDetails);

TShowTable4InformationSchemaResp showTables4InformationSchema();

TDescTableResp describeTable(
final String database, final String tableName, final boolean isDetails);

TDescTable4InformationSchemaResp describeTable4InformationSchema();

TFetchTableResp fetchTables(final Map<String, Set<String>> fetchTableMap);
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.read.table.DescTable4InformationSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.read.table.DescTablePlan;
import org.apache.iotdb.confignode.consensus.request.read.table.FetchTablePlan;
import org.apache.iotdb.confignode.consensus.request.read.table.ShowTable4InformationSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan;
import org.apache.iotdb.confignode.consensus.request.read.template.GetAllSchemaTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.read.template.GetAllTemplateSetInfoPlan;
Expand All @@ -67,8 +69,10 @@
import org.apache.iotdb.confignode.consensus.response.database.CountDatabaseResp;
import org.apache.iotdb.confignode.consensus.response.database.DatabaseSchemaResp;
import org.apache.iotdb.confignode.consensus.response.partition.PathInfoResp;
import org.apache.iotdb.confignode.consensus.response.table.DescTable4InformationSchemaResp;
import org.apache.iotdb.confignode.consensus.response.table.DescTableResp;
import org.apache.iotdb.confignode.consensus.response.table.FetchTableResp;
import org.apache.iotdb.confignode.consensus.response.table.ShowTable4InformationSchemaResp;
import org.apache.iotdb.confignode.consensus.response.table.ShowTableResp;
import org.apache.iotdb.confignode.consensus.response.template.AllTemplateSetInfoResp;
import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp;
Expand All @@ -82,12 +86,14 @@
import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetTemplateResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.schemaengine.template.Template;
Expand Down Expand Up @@ -1092,6 +1098,19 @@ public TShowTableResp showTables(final String database, final boolean isDetails)
}
}

public TShowTable4InformationSchemaResp showTables4InformationSchema() {
try {
return ((ShowTable4InformationSchemaResp)
configManager.getConsensusManager().read(new ShowTable4InformationSchemaPlan()))
.convertToTShowTable4InformationSchemaResp();
} catch (final ConsensusException e) {
LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e);
final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
res.setMessage(e.getMessage());
return new TShowTable4InformationSchemaResp(res);
}
}

public TDescTableResp describeTable(
final String database, final String tableName, final boolean isDetails) {
try {
Expand All @@ -1108,6 +1127,19 @@ public TDescTableResp describeTable(
}
}

public TDescTable4InformationSchemaResp describeTables4InformationSchema() {
try {
return ((DescTable4InformationSchemaResp)
configManager.getConsensusManager().read(new DescTable4InformationSchemaPlan()))
.convertToTDescTable4InformationSchemaResp();
} catch (final ConsensusException e) {
LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e);
final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
res.setMessage(e.getMessage());
return new TDescTable4InformationSchemaResp(res);
}
}

public TFetchTableResp fetchTables(final Map<String, Set<String>> fetchTableMap) {
try {
return ((FetchTableResp)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -322,10 +322,14 @@ public DataSet executeQueryPlan(final ConfigPhysicalReadPlan req)
return clusterSchemaInfo.getTemplateSetInfo((GetTemplateSetInfoPlan) req);
case ShowTable:
return clusterSchemaInfo.showTables((ShowTablePlan) req);
case ShowTable4InformationSchema:
return clusterSchemaInfo.showTables4InformationSchema();
case FetchTable:
return clusterSchemaInfo.fetchTables((FetchTablePlan) req);
case DescTable:
return clusterSchemaInfo.descTable((DescTablePlan) req);
case DescTable4InformationSchema:
return clusterSchemaInfo.descTable4InformationSchema();
case GetTriggerTable:
return triggerInfo.getTriggerTable((GetTriggerTablePlan) req);
case GetTriggerLocation:
Expand Down
Loading
Loading