From de1ce5d4c2c72827ba8d873301a5ee743bab6a44 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 24 May 2024 21:35:41 +0800 Subject: [PATCH 01/73] SQLValueProcessor --- .../mysql/type/MysqlValueProcessorEnum.java | 71 ++++++++++++++++++ .../mysql/value/MysqlValueProcessor.java | 44 +++++++++++ .../main/java/ai/chat2db/spi/MetaData.java | 2 + .../ai/chat2db/spi/SQLValueProcessor.java | 10 +++ .../chat2db/spi/jdbc/DefaultMetaService.java | 13 +++- .../spi/jdbc/DefaultSQLValueProcessor.java | 74 +++++++++++++++++++ .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 33 ++++++++- 7 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java new file mode 100644 index 000000000..a9a69e6be --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java @@ -0,0 +1,71 @@ +package ai.chat2db.plugin.mysql.type; + +import ai.chat2db.spi.SQLValueProcessor; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.WKBReader; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.sql.ResultSet; +import java.sql.SQLException; + +public enum MysqlValueProcessorEnum implements SQLValueProcessor { + GEOMETRY{ + @Override + public String getSqlValueString(ResultSet rs, int index) throws SQLException { + try { + InputStream inputStream = rs.getBinaryStream(index); + Geometry dbGeometry = null; + if (inputStream != null) { + + //convert the stream to a byte[] array + //so it can be passed to the WKBReader + byte[] buffer = new byte[255]; + + int bytesRead = 0; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((bytesRead = inputStream.read(buffer)) != -1) { + baos.write(buffer, 0, bytesRead); + } + + byte[] geometryAsBytes = baos.toByteArray(); + + if (geometryAsBytes.length < 5) { + throw new Exception("Invalid geometry inputStream - less than five bytes"); + } + + //first four bytes of the geometry are the SRID, + //followed by the actual WKB. Determine the SRID + //here + byte[] sridBytes = new byte[4]; + System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); + boolean bigEndian = (geometryAsBytes[4] == 0x00); + + int srid = 0; + if (bigEndian) { + for (int i = 0; i < sridBytes.length; i++) { + srid = (srid << 8) + (sridBytes[i] & 0xff); + } + } else { + for (int i = 0; i < sridBytes.length; i++) { + srid += (sridBytes[i] & 0xff) << (8 * i); + } + } + + //use the JTS WKBReader for WKB parsing + WKBReader wkbReader = new WKBReader(); + + //copy the byte array, removing the first four + //SRID bytes + byte[] wkb = new byte[geometryAsBytes.length - 4]; + System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); + dbGeometry = wkbReader.read(wkb); + dbGeometry.setSRID(srid); + } + return dbGeometry.toString(); + } catch (Exception e) { + return rs.getString(index); + } + } + }; +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java new file mode 100644 index 000000000..5aadec812 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -0,0 +1,44 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; +import ai.chat2db.plugin.mysql.type.MysqlValueProcessorEnum; +import ai.chat2db.spi.jdbc.DefaultSQLValueProcessor; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * @author: zgq + * @date: 2024年05月24日 21:02 + */ +public class MysqlValueProcessor extends DefaultSQLValueProcessor { + /** + * @param rs + * @param index + * @return + * @throws SQLException + */ + @Override + public String getSqlValueString(ResultSet rs, int index) throws SQLException { + Object obj = rs.getObject(index); + if (obj == null) { + return null; + } + String columnTypeName = rs.getMetaData().getColumnTypeName(index); + if (MysqlColumnTypeEnum.GEOMETRY.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.POINT.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.LINESTRING.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.POLYGON.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.MULTIPOINT.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.MULTILINESTRING.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.MULTIPOLYGON.name().equalsIgnoreCase(columnTypeName) + || MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name().equalsIgnoreCase(columnTypeName) + ) { + return MysqlValueProcessorEnum.GEOMETRY.getSqlValueString(rs, index); + } else { + super.getSqlValueString(rs, index); + } + return null; + } + +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java index e6dc2bde6..79c347722 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java @@ -244,6 +244,8 @@ List indexes(Connection connection, @NotEmpty String databaseName, S */ ValueHandler getValueHandler(); + SQLValueProcessor getSQLValueProcessor(); + /** * Get command executor. diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java new file mode 100644 index 000000000..a485836e0 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java @@ -0,0 +1,10 @@ +package ai.chat2db.spi; + + +import java.sql.ResultSet; +import java.sql.SQLException; + +public interface SQLValueProcessor { + + String getSqlValueString(ResultSet rs, int index) throws SQLException; +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java index a84d43dff..752d04b5f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java @@ -1,10 +1,7 @@ package ai.chat2db.spi.jdbc; import ai.chat2db.server.tools.base.wrapper.result.PageResult; -import ai.chat2db.spi.CommandExecutor; -import ai.chat2db.spi.MetaData; -import ai.chat2db.spi.SqlBuilder; -import ai.chat2db.spi.ValueHandler; +import ai.chat2db.spi.*; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; import com.google.common.collect.Lists; @@ -167,6 +164,14 @@ public ValueHandler getValueHandler() { return new DefaultValueHandler(); } + /** + * @return + */ + @Override + public SQLValueProcessor getSQLValueProcessor() { + return new DefaultSQLValueProcessor(); + } + @Override public CommandExecutor getCommandExecutor() { return SQLExecutor.getInstance(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java new file mode 100644 index 000000000..d9f694d1b --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java @@ -0,0 +1,74 @@ +package ai.chat2db.spi.jdbc; + +import ai.chat2db.spi.SQLValueProcessor; +import com.google.common.io.BaseEncoding; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年05月24日 14:30 + */ +public class DefaultSQLValueProcessor implements SQLValueProcessor { + /** + * @param rs + * @param index + * @return + */ + @Override + public String getSqlValueString(ResultSet rs, int index) throws SQLException { + Object object = rs.getObject(index); + if (Objects.isNull(object)) { + return "NULL"; + } + if (object instanceof BigDecimal bigDecimal) { + return bigDecimal.toPlainString(); + } else if (object instanceof Float f) { + return BigDecimal.valueOf(f).toPlainString(); + } else if (object instanceof Double d) { + return BigDecimal.valueOf(d).toPlainString(); + } else if (object instanceof Number n) { + return n.toString(); + } else if (object instanceof Boolean) { + return (Boolean) object ? "1" : "0"; + } else if (object instanceof byte[]) { + return converterByteArray2Str((byte[]) object); + } else if (object instanceof Blob B) { + return converterByteArray2Str(B.getBytes(1, Math.toIntExact(B.length()))); + } else if (object instanceof Clob c) { + return converterClob2Str(c); + } + return "'" + escapeString(object) + "'"; + } + + private String escapeString(Object object) { + String s = (String) object; + return s.replace("\\", "\\\\").replace("'", "''"); + } + + private String converterClob2Str(Clob c) { + StringBuilder stringBuilder = new StringBuilder(); + try (Reader reader = c.getCharacterStream()) { + BufferedReader bufferedReader = new BufferedReader(reader); + String line; + while ((line = bufferedReader.readLine()) != null) { + stringBuilder.append(line); + } + return escapeString(stringBuilder.toString()); + } catch (SQLException | IOException e) { + throw new RuntimeException(e); + } + } + + private String converterByteArray2Str(byte[] bytes) { + return "0x" + BaseEncoding.base16().encode(bytes); + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index d63d1603c..0273682e6 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -17,7 +17,9 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; public class DefaultSqlBuilder implements SqlBuilder { @@ -122,13 +124,13 @@ public String getTableDmlSql(Table table, String type) { if (table == null || CollectionUtils.isEmpty(table.getColumnList()) || StringUtils.isBlank(type)) { return ""; } - if(DmlType.INSERT.name().equalsIgnoreCase(type)) { + if (DmlType.INSERT.name().equalsIgnoreCase(type)) { return getInsertSql(table.getName(), table.getColumnList()); - } else if(DmlType.UPDATE.name().equalsIgnoreCase(type)) { + } else if (DmlType.UPDATE.name().equalsIgnoreCase(type)) { return getUpdateSql(table.getName(), table.getColumnList()); - } else if(DmlType.DELETE.name().equalsIgnoreCase(type)) { + } else if (DmlType.DELETE.name().equalsIgnoreCase(type)) { return getDeleteSql(table.getName(), table.getColumnList()); - }else if(DmlType.SELECT.name().equalsIgnoreCase(type)) { + } else if (DmlType.SELECT.name().equalsIgnoreCase(type)) { return getSelectSql(table.getName(), table.getColumnList()); } return ""; @@ -187,6 +189,29 @@ private String getInsertSql(String name, List columnList) { return script.toString(); } + private String getInsertSql(String schemaName, String tableName, List columnList, List valueList) { + StringBuilder script = new StringBuilder(); + script.append("INSERT INTO ").append(schemaName).append(".").append(tableName) + .append(" (") + .append(String.join(",", columnList)) + .append(") VALUES (") + .append(String.join(",", valueList)) + .append(");"); + return script.toString(); + } + + private String getMultiInsertSql(String schemaName, String tableName, List columnList, List> valueList) { + StringBuilder script = new StringBuilder(); + script.append("INSERT INTO ").append(schemaName).append(".").append(tableName) + .append(" (") + .append(String.join(",", columnList)) + .append(") VALUES ") + .append(valueList.stream() + .map(values -> "(" + String.join(",", values) + ")") + .collect(Collectors.joining(",\n"))) + .append(");"); + return script.toString(); + } private List getPrimaryColumns(List
headerList) { if (CollectionUtils.isEmpty(headerList)) { return Lists.newArrayList(); From d8970c6d0ae97f6b5a41632c991f8fcec5c09e88 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 24 May 2024 21:52:11 +0800 Subject: [PATCH 02/73] fix blank string --- .../java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java index d9f694d1b..9f699e344 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.SQLValueProcessor; import com.google.common.io.BaseEncoding; +import org.apache.commons.lang3.StringUtils; import java.io.BufferedReader; import java.io.IOException; @@ -51,6 +52,9 @@ public String getSqlValueString(ResultSet rs, int index) throws SQLException { private String escapeString(Object object) { String s = (String) object; + if (StringUtils.isBlank(s)) { + return ""; + } return s.replace("\\", "\\\\").replace("'", "''"); } From 37db8873831c7c17ee05ae6ab295ef7d01517b9b Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 24 May 2024 21:53:54 +0800 Subject: [PATCH 03/73] fix NULL --- .../java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index 5aadec812..dbd9ddd79 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -22,7 +22,7 @@ public class MysqlValueProcessor extends DefaultSQLValueProcessor { public String getSqlValueString(ResultSet rs, int index) throws SQLException { Object obj = rs.getObject(index); if (obj == null) { - return null; + return "NULL"; } String columnTypeName = rs.getMetaData().getColumnTypeName(index); if (MysqlColumnTypeEnum.GEOMETRY.name().equalsIgnoreCase(columnTypeName) From 053ed0d30a7b87359b4b2242aa24486d92a26934 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 24 May 2024 22:00:45 +0800 Subject: [PATCH 04/73] fix escapeString --- .../chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java | 7 ++++--- .../java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java index a9a69e6be..1246777be 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java @@ -1,6 +1,7 @@ package ai.chat2db.plugin.mysql.type; import ai.chat2db.spi.SQLValueProcessor; +import ai.chat2db.spi.jdbc.DefaultSQLValueProcessor; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.WKBReader; @@ -10,7 +11,7 @@ import java.sql.SQLException; public enum MysqlValueProcessorEnum implements SQLValueProcessor { - GEOMETRY{ + GEOMETRY { @Override public String getSqlValueString(ResultSet rs, int index) throws SQLException { try { @@ -62,9 +63,9 @@ public String getSqlValueString(ResultSet rs, int index) throws SQLException { dbGeometry = wkbReader.read(wkb); dbGeometry.setSRID(srid); } - return dbGeometry.toString(); + return DefaultSQLValueProcessor.escapeString(dbGeometry.toString()); } catch (Exception e) { - return rs.getString(index); + return DefaultSQLValueProcessor.escapeString(rs.getString(index)); } } }; diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java index 9f699e344..e93e8a64a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java @@ -50,7 +50,7 @@ public String getSqlValueString(ResultSet rs, int index) throws SQLException { return "'" + escapeString(object) + "'"; } - private String escapeString(Object object) { + public static String escapeString(Object object) { String s = (String) object; if (StringUtils.isBlank(s)) { return ""; @@ -58,7 +58,7 @@ private String escapeString(Object object) { return s.replace("\\", "\\\\").replace("'", "''"); } - private String converterClob2Str(Clob c) { + public static String converterClob2Str(Clob c) { StringBuilder stringBuilder = new StringBuilder(); try (Reader reader = c.getCharacterStream()) { BufferedReader bufferedReader = new BufferedReader(reader); @@ -72,7 +72,7 @@ private String converterClob2Str(Clob c) { } } - private String converterByteArray2Str(byte[] bytes) { + public static String converterByteArray2Str(byte[] bytes) { return "0x" + BaseEncoding.base16().encode(bytes); } } From 3a7e19d912bab7c02c1c3cf298ef0a3e9093c762 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 1 Jun 2024 20:18:58 +0800 Subject: [PATCH 05/73] basic MysqlValueProcessor --- .../chat2db/plugin/mysql/MysqlMetaData.java | 8 +- .../mysql/type/MysqlValueProcessorEnum.java | 72 -------------- .../plugin/mysql/value/MysqlBitProcessor.java | 63 ++++++++++++ .../mysql/value/MysqlDateTimeProcessor.java | 8 ++ .../mysql/value/MysqlDecimalProcessor.java | 29 ++++++ .../mysql/value/MysqlGeometryProcessor.java | 96 +++++++++++++++++++ .../mysql/value/MysqlTimestampProcessor.java | 40 ++++++++ .../mysql/value/MysqlValueProcessor.java | 83 ++++++++++------ .../mysql/value/MysqlYearProcessor.java | 62 ++++++++++++ .../value/template/MysqlDmlValueTemplate.java | 12 +++ .../template/OracleDmlValueTemplate.java | 10 ++ .../tools/common/util/EasyStringUtils.java | 80 ++++++++++++++-- .../main/java/ai/chat2db/spi/MetaData.java | 3 +- .../ai/chat2db/spi/SQLValueProcessor.java | 10 -- .../java/ai/chat2db/spi/ValueProcessor.java | 40 ++++++++ .../chat2db/spi/jdbc/BaseValueProcessor.java | 60 ++++++++++++ .../chat2db/spi/jdbc/DefaultMetaService.java | 8 +- .../spi/jdbc/DefaultSQLValueProcessor.java | 78 --------------- .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 73 ++++++++++---- .../spi/jdbc/DefaultValueProcessor.java | 59 ++++++++++++ .../java/ai/chat2db/spi/model/DataType.java | 32 +++++++ .../ai/chat2db/spi/model/JDBCDataValue.java | 73 ++++++++++++++ .../ai/chat2db/spi/model/SQLDataValue.java | 21 ++++ .../java/ai/chat2db/spi/sql/SQLExecutor.java | 6 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 93 ++++++++++++++++-- 25 files changed, 885 insertions(+), 234 deletions(-) delete mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java delete mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java delete mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/DataType.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java index 50e7322f4..b241197db 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java @@ -2,7 +2,9 @@ import ai.chat2db.plugin.mysql.builder.MysqlSqlBuilder; import ai.chat2db.plugin.mysql.type.*; +import ai.chat2db.plugin.mysql.value.MysqlValueProcessor; import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.ValueHandler; import ai.chat2db.spi.jdbc.DefaultMetaService; @@ -10,7 +12,6 @@ import ai.chat2db.spi.sql.SQLExecutor; import jakarta.validation.constraints.NotEmpty; import org.apache.commons.lang3.StringUtils; -import org.checkerframework.checker.units.qual.A; import java.sql.Connection; import java.sql.ResultSet; @@ -352,6 +353,11 @@ public ValueHandler getValueHandler() { return new MysqlValueHandler(); } + @Override + public ValueProcessor getValueProcessor() { + return new MysqlValueProcessor(); + } + @Override public List getSystemDatabases() { return systemDatabases; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java deleted file mode 100644 index 1246777be..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlValueProcessorEnum.java +++ /dev/null @@ -1,72 +0,0 @@ -package ai.chat2db.plugin.mysql.type; - -import ai.chat2db.spi.SQLValueProcessor; -import ai.chat2db.spi.jdbc.DefaultSQLValueProcessor; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.io.WKBReader; - -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.sql.ResultSet; -import java.sql.SQLException; - -public enum MysqlValueProcessorEnum implements SQLValueProcessor { - GEOMETRY { - @Override - public String getSqlValueString(ResultSet rs, int index) throws SQLException { - try { - InputStream inputStream = rs.getBinaryStream(index); - Geometry dbGeometry = null; - if (inputStream != null) { - - //convert the stream to a byte[] array - //so it can be passed to the WKBReader - byte[] buffer = new byte[255]; - - int bytesRead = 0; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - while ((bytesRead = inputStream.read(buffer)) != -1) { - baos.write(buffer, 0, bytesRead); - } - - byte[] geometryAsBytes = baos.toByteArray(); - - if (geometryAsBytes.length < 5) { - throw new Exception("Invalid geometry inputStream - less than five bytes"); - } - - //first four bytes of the geometry are the SRID, - //followed by the actual WKB. Determine the SRID - //here - byte[] sridBytes = new byte[4]; - System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); - boolean bigEndian = (geometryAsBytes[4] == 0x00); - - int srid = 0; - if (bigEndian) { - for (int i = 0; i < sridBytes.length; i++) { - srid = (srid << 8) + (sridBytes[i] & 0xff); - } - } else { - for (int i = 0; i < sridBytes.length; i++) { - srid += (sridBytes[i] & 0xff) << (8 * i); - } - } - - //use the JTS WKBReader for WKB parsing - WKBReader wkbReader = new WKBReader(); - - //copy the byte array, removing the first four - //SRID bytes - byte[] wkb = new byte[geometryAsBytes.length - 4]; - System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); - dbGeometry = wkbReader.read(wkb); - dbGeometry.setSRID(srid); - } - return DefaultSQLValueProcessor.escapeString(dbGeometry.toString()); - } catch (Exception e) { - return DefaultSQLValueProcessor.escapeString(rs.getString(index)); - } - } - }; -} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java new file mode 100644 index 000000000..dfe2f1741 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java @@ -0,0 +1,63 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年06月01日 13:08 + */ +public class MysqlBitProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return getString(dataValue.getValue()); + } + + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + int precision = dataValue.getPrecision(); + byte[] bytes = dataValue.getBytes(); + if (precision == 1) { + //bit(1) [1 -> true] [0 -> false] + if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { + return String.valueOf(dataValue.getBoolean()); + } + // tinyint(1) + return String.valueOf(dataValue.getInt()); + } + //bit(m) m: 1~64 + return EasyStringUtils.getBitString(bytes, precision); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getString((String) convertJDBCValueByType(dataValue)); + } + + public String getString(String value) { + + if (Objects.equals("true", value.toLowerCase())) { + return "1"; + } + if (Objects.equals("false", value.toLowerCase())) { + return "0"; + } + if (StringUtils.isBlank(value)) { + return "NULL"; + } + return wrap(value); + } + + private String wrap(String value) { + return String.format(MysqlDmlValueTemplate.BIT_TEMPLATE, value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java new file mode 100644 index 000000000..fe65e07ec --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java @@ -0,0 +1,8 @@ +package ai.chat2db.plugin.mysql.value; + +/** + * @author: zgq + * @date: 2024年06月01日 19:20 + */ +public class MysqlDateTimeProcessor extends MysqlTimestampProcessor{ +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java new file mode 100644 index 000000000..2e00006c7 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java @@ -0,0 +1,29 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月01日 18:01 + */ +public class MysqlDecimalProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + return new String(dataValue.getBytes()); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return (String) convertJDBCValueByType(dataValue); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java new file mode 100644 index 000000000..b0e5e0d6e --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java @@ -0,0 +1,96 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.WKBReader; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; + +/** + * @author: zgq + * @date: 2024年06月01日 12:42 + */ +public class MysqlGeometryProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue()); + } + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + try { + InputStream inputStream = dataValue.getBinaryStream(); + Geometry dbGeometry = null; + if (inputStream != null) { + + //convert the stream to a byte[] array + //so it can be passed to the WKBReader + byte[] buffer = new byte[255]; + + int bytesRead = 0; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((bytesRead = inputStream.read(buffer)) != -1) { + baos.write(buffer, 0, bytesRead); + } + + byte[] geometryAsBytes = baos.toByteArray(); + + if (geometryAsBytes.length < 5) { + throw new Exception("Invalid geometry inputStream - less than five bytes"); + } + + //first four bytes of the geometry are the SRID, + //followed by the actual WKB. Determine the SRID + //here + byte[] sridBytes = new byte[4]; + System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); + boolean bigEndian = (geometryAsBytes[4] == 0x00); + + int srid = 0; + if (bigEndian) { + for (int i = 0; i < sridBytes.length; i++) { + srid = (srid << 8) + (sridBytes[i] & 0xff); + } + } else { + for (int i = 0; i < sridBytes.length; i++) { + srid += (sridBytes[i] & 0xff) << (8 * i); + } + } + + //use the JTS WKBReader for WKB parsing + WKBReader wkbReader = new WKBReader(); + + //copy the byte array, removing the first four + //SRID bytes + byte[] wkb = new byte[geometryAsBytes.length - 4]; + System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); + dbGeometry = wkbReader.read(wkb); + dbGeometry.setSRID(srid); + } + return dbGeometry != null ? dbGeometry.toString() : null; + } catch (Exception e) { + return super.getJdbcValue(dataValue); + } + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + Object jdbcValue = getJdbcValue(dataValue); + if (jdbcValue == null) { + return super.getJdbcValueString(dataValue); + } + return wrap((String) jdbcValue); + } + + private String wrap(String value) { + return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE,value); + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java new file mode 100644 index 000000000..832c27352 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java @@ -0,0 +1,40 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月01日 18:26 + */ +public class MysqlTimestampProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return super.convertSQLValueByType(dataValue); + } + + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + return isValidTimestamp(dataValue) ? new String(dataValue.getBytes()) : "0000-00-00 00:00:00"; + } + + protected boolean isValidTimestamp(JDBCDataValue data) { + byte[] buffer = data.getBytes(); + String stringValue = new String(buffer); + return stringValue.length() <= 0 + || stringValue.charAt(0) != '0' + || !"0000-00-00".equals(stringValue) + && !"0000-00-00 00:00:00".equals(stringValue) + && !"00000000000000".equals(stringValue) + && !"0".equals(stringValue); + } + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return String.format(MysqlDmlValueTemplate.COMMON_TEMPLATE, convertJDBCValueByType(dataValue)); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index dbd9ddd79..ed8bfd5fd 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -1,44 +1,69 @@ package ai.chat2db.plugin.mysql.value; import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; -import ai.chat2db.plugin.mysql.type.MysqlValueProcessorEnum; -import ai.chat2db.spi.jdbc.DefaultSQLValueProcessor; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.util.Map; +import java.util.Set; /** * @author: zgq * @date: 2024年05月24日 21:02 + *
+ * TODO: + * attribute: [zerofill] example tinyint[5] zerofill 34->00034 */ -public class MysqlValueProcessor extends DefaultSQLValueProcessor { - /** - * @param rs - * @param index - * @return - * @throws SQLException - */ +public class MysqlValueProcessor extends DefaultValueProcessor { + + private static final Set GEOMETRY_TYPE = Set.of(MysqlColumnTypeEnum.GEOMETRY.name() + , MysqlColumnTypeEnum.POINT.name() + , MysqlColumnTypeEnum.LINESTRING.name() + , MysqlColumnTypeEnum.POLYGON.name() + , MysqlColumnTypeEnum.MULTIPOINT.name() + , MysqlColumnTypeEnum.MULTILINESTRING.name() + , MysqlColumnTypeEnum.MULTIPOLYGON.name() + , MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name()); + + + private static final Map PROCESSOR_MAP = Map.of( + MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor(), + MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor(), + MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor(), + MysqlColumnTypeEnum.TIMESTAMP.name(), new MysqlTimestampProcessor(), + MysqlColumnTypeEnum.DATETIME.name(), new MysqlDateTimeProcessor() + ); + public static final Set FUNCTION_SET = Set.of("now()"); + @Override - public String getSqlValueString(ResultSet rs, int index) throws SQLException { - Object obj = rs.getObject(index); - if (obj == null) { - return "NULL"; + public String convertSQLValueByType(SQLDataValue dataValue) { + if (FUNCTION_SET.contains(dataValue.getValue())) { + return dataValue.getValue(); + } + String dataType = dataValue.getDateTypeName(); + if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { + return new MysqlGeometryProcessor().convertSQLValueByType(dataValue); } - String columnTypeName = rs.getMetaData().getColumnTypeName(index); - if (MysqlColumnTypeEnum.GEOMETRY.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.POINT.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.LINESTRING.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.POLYGON.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTIPOINT.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTILINESTRING.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTIPOLYGON.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name().equalsIgnoreCase(columnTypeName) - ) { - return MysqlValueProcessorEnum.GEOMETRY.getSqlValueString(rs, index); - } else { - super.getSqlValueString(rs, index); + return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertSQLValueByType(dataValue); + } + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + + String dataType = dataValue.getType(); + if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { + return new MysqlGeometryProcessor().convertJDBCValueByType(dataValue); } - return null; + return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueByType(dataValue); } + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + String dataType = dataValue.getType(); + if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { + return new MysqlGeometryProcessor().convertJDBCValueStrByType(dataValue); + } + return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueStrByType(dataValue); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java new file mode 100644 index 000000000..d686397f7 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java @@ -0,0 +1,62 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Date; +import java.util.Calendar; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月01日 12:57 + */ +public class MysqlYearProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + Date date = dataValue.getDate(); + if (!isValidYear(dataValue)) { + return "0000"; + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int year = calendar.get(Calendar.YEAR); + String yStr; + String yZerosPadding = "0000"; + if (year < 1000) { + yStr = "" + year; + yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; + } else { + yStr = "" + year; + } + return yStr; + } + + private boolean isValidYear(JDBCDataValue data) { + byte[] buffer = data.getBytes(); + String stringValue = new String(buffer); + return stringValue.length() <= 0 + || stringValue.charAt(0) != '0' + || !"0000-00-00".equals(stringValue) + && !"0000-00-00 00:00:00".equals(stringValue) + && !"00000000000000".equals(stringValue) + && !"0".equals(stringValue) + && !"00000000".equals(stringValue) + && !"0000".equals(stringValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return (String) getJdbcValue(dataValue); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java new file mode 100644 index 000000000..f3547f62b --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -0,0 +1,12 @@ +package ai.chat2db.plugin.mysql.value.template; + +/** + * @author: zgq + * @date: 2024年06月01日 13:31 + */ +public class MysqlDmlValueTemplate { + + public static final String COMMON_TEMPLATE = "'%s'"; + public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; + public static final String BIT_TEMPLATE = "b'%s'"; +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java new file mode 100644 index 000000000..52da8412d --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java @@ -0,0 +1,10 @@ +package ai.chat2db.plugin.oracle.template; + +/** + * @author: zgq + * @date: 2024年06月01日 13:35 + */ +public class OracleDmlValueTemplate { + + public static final String DATE_TEMPLATE = "TO_DATE('%s', 'SYYYY-MM-DD HH24:MI:SS')"; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 5f2987bb9..2f117c31f 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -1,13 +1,14 @@ package ai.chat2db.server.tools.common.util; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import jakarta.validation.constraints.NotNull; import org.apache.commons.lang3.RegExUtils; import org.apache.commons.lang3.StringUtils; +import java.util.*; +import java.util.stream.Collectors; + /** * String utility class * @@ -87,7 +88,7 @@ public static String padUserId(String userId) { /** * Build the name of the display * - * @param name name + * @param name name * @param nickName flower name * @return display name name (flower name) */ @@ -108,7 +109,7 @@ public static String buildShowName(String name, String nickName) { * Splice multiple strings together * * @param delimiter delimiter cannot be empty - * @param elements string can be empty and empty strings will be ignored + * @param elements string can be empty and empty strings will be ignored * @return */ public static String join(CharSequence delimiter, CharSequence... elements) { @@ -116,7 +117,7 @@ public static String join(CharSequence delimiter, CharSequence... elements) { return null; } List charSequenceList = Arrays.stream(elements).filter( - org.apache.commons.lang3.StringUtils::isNotBlank).collect(Collectors.toList()); + org.apache.commons.lang3.StringUtils::isNotBlank).collect(Collectors.toList()); if (charSequenceList.isEmpty()) { return null; } @@ -126,7 +127,7 @@ public static String join(CharSequence delimiter, CharSequence... elements) { /** * Limit the length of a string string. If it exceeds the length, it will be replaced with... * - * @param str string + * @param str string * @param length limit length * @return */ @@ -141,4 +142,65 @@ public static String limitString(String str, int length) { return limitString; } + /** + * 对字符串中的特定字符进行转义处理。 + *

+ * 根据提供的转义映射表，该方法遍历输入字符串中的每个字符，如果字符存在于映射表中， + * 则在该字符前插入映射表中对应的转义字符，否则保持字符不变。这常用于创建符合特定格式要求的字符串， + * 比如SQL查询中的字符串转义，或为文本添加特殊标记等。 + * + * @param str 目标字符串，需要进行转义处理的原始字符串。 + * @param escapeMap 映射关系表，键为需要被转义的字符，值为该字符前应添加的转义字符。 + * 例如，如果希望转义单引号(')，可以传入Map中键为单引号，值也为单引号的映射。 + * @return 转义后的字符串，含有特定字符前添加了转义符的副本。 + */ + public static String escapeString(@NotNull String str, Map escapeMap) { + if (str == null) { + return null; + } + if (StringUtils.isBlank(str)) { + return str; + } + + + StringBuilder escapedString = new StringBuilder(str.length() * 2); + + for (char c : str.toCharArray()) { + if (escapeMap.containsKey(c)) { + escapedString.append(escapeMap.get(c)).append(c); + } else { + escapedString.append(c); + } + } + return escapedString.toString(); + } + + public static String escapeString(String str) { + HashMap escapeMap = Maps.newHashMapWithExpectedSize(2); + // (char)39 -> ' + escapeMap.put((char) 39, (char) 39); + // (char)92 -> \ + escapeMap.put((char) 92, (char) 92); + return escapeString(str, escapeMap); + } + + public static String getBitString(byte[] bytes, final int precision) { + if (bytes == null || bytes.length == 0) { + return ""; + } + + StringBuilder builder = new StringBuilder(precision); + for (byte b : bytes) { + builder.append(Integer.toBinaryString(b & 0xFF)); + } + + // 获取完整的二进制字符串 + String bitString = builder.toString(); + + // 填充前导零以匹配所需的总长度 + bitString = Strings.padStart(bitString, precision, '0'); + + return bitString; + } + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java index 79c347722..eae653013 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java @@ -5,7 +5,6 @@ import ai.chat2db.server.tools.base.wrapper.result.PageResult; import ai.chat2db.spi.model.*; -import cn.hutool.db.Page; import jakarta.validation.constraints.NotEmpty; /** @@ -244,7 +243,7 @@ List indexes(Connection connection, @NotEmpty String databaseName, S */ ValueHandler getValueHandler(); - SQLValueProcessor getSQLValueProcessor(); + ValueProcessor getValueProcessor(); /** diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java deleted file mode 100644 index a485836e0..000000000 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SQLValueProcessor.java +++ /dev/null @@ -1,10 +0,0 @@ -package ai.chat2db.spi; - - -import java.sql.ResultSet; -import java.sql.SQLException; - -public interface SQLValueProcessor { - - String getSqlValueString(ResultSet rs, int index) throws SQLException; -} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java new file mode 100644 index 000000000..05bfaa06e --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java @@ -0,0 +1,40 @@ +package ai.chat2db.spi; + + +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +public interface ValueProcessor { + + /** + * Converts a given value into a format suitable for use in an SQL statement + *
+ * Example: + *
+ * Input oracle DATE : '2024-05-29 11:35:20.0' + *
+ * Output for Oracle DATE: TO_DATE('2024-05-29 14:25:00', 'SYYYY-MM-DD HH24:MI:SS') + */ + String getSqlValueString(SQLDataValue dataValue); + + + /** + * 将JDBC数据值对象转换为适合前端展示的字符串格式。 + *

+ * 它旨在处理包括但不限于数字、日期、字符串以及特殊的空数据，确保这些数据 + * 在传递到前端用户界面时是格式化良好且可理解的。 + * + * @param dataValue ResultSetMetaData, ResultSet, columnIndex的组合对象，用于获取数据值。 + * @return 一个格式化后的字符串，适配于前端展示。例如，日期可能会转换为"YYYY-MM-DD"格式，以方便用户直观理解。 + */ + Object getJdbcValue(JDBCDataValue dataValue); + + /** + * 将从JDBC ResultSet中获取的数据值转换并构造为适合DML语句的格式。 + * + * @param dataValue JDBC数据源中检索出的数据值对象，用于准备DML操作的值。 + * + * @return 一个格式化后的字符串，可以直接用于DML语句中，确保数据的正确插入或更新。 + */ + String getJdbcValueString(JDBCDataValue dataValue); +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java new file mode 100644 index 000000000..4c7a671ee --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -0,0 +1,60 @@ +package ai.chat2db.spi.jdbc; + +import ai.chat2db.spi.ValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年05月30日 15:33 + */ +public abstract class BaseValueProcessor implements ValueProcessor { + + @Override + public String getSqlValueString(SQLDataValue dataValue) { + if (Objects.isNull(dataValue.getValue())) { + return "NULL"; + } + return convertSQLValueByType(dataValue); + + } + + + @Override + public Object getJdbcValue(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(dataValue.getObject())) { + return null; + } + if (value instanceof String stringValue) { + if (StringUtils.isBlank(stringValue)) { + return StringUtils.wrap(stringValue, "\""); + } + } + return convertJDBCValueByType(dataValue); + } + + + @Override + public String getJdbcValueString(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + return "NULL"; + } + if (value instanceof String stringValue) { + if (StringUtils.isBlank(stringValue)) { + return StringUtils.wrap(stringValue, "'"); + } + } + return convertJDBCValueStrByType(dataValue); + } + + public abstract String convertSQLValueByType(SQLDataValue dataValue); + + public abstract Object convertJDBCValueByType(JDBCDataValue dataValue); + + public abstract String convertJDBCValueStrByType(JDBCDataValue dataValue); +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java index 752d04b5f..1c1c836c6 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java @@ -164,12 +164,10 @@ public ValueHandler getValueHandler() { return new DefaultValueHandler(); } - /** - * @return - */ + @Override - public SQLValueProcessor getSQLValueProcessor() { - return new DefaultSQLValueProcessor(); + public ValueProcessor getValueProcessor() { + return new DefaultValueProcessor(); } @Override diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java deleted file mode 100644 index e93e8a64a..000000000 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSQLValueProcessor.java +++ /dev/null @@ -1,78 +0,0 @@ -package ai.chat2db.spi.jdbc; - -import ai.chat2db.spi.SQLValueProcessor; -import com.google.common.io.BaseEncoding; -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.math.BigDecimal; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Objects; - -/** - * @author: zgq - * @date: 2024年05月24日 14:30 - */ -public class DefaultSQLValueProcessor implements SQLValueProcessor { - /** - * @param rs - * @param index - * @return - */ - @Override - public String getSqlValueString(ResultSet rs, int index) throws SQLException { - Object object = rs.getObject(index); - if (Objects.isNull(object)) { - return "NULL"; - } - if (object instanceof BigDecimal bigDecimal) { - return bigDecimal.toPlainString(); - } else if (object instanceof Float f) { - return BigDecimal.valueOf(f).toPlainString(); - } else if (object instanceof Double d) { - return BigDecimal.valueOf(d).toPlainString(); - } else if (object instanceof Number n) { - return n.toString(); - } else if (object instanceof Boolean) { - return (Boolean) object ? "1" : "0"; - } else if (object instanceof byte[]) { - return converterByteArray2Str((byte[]) object); - } else if (object instanceof Blob B) { - return converterByteArray2Str(B.getBytes(1, Math.toIntExact(B.length()))); - } else if (object instanceof Clob c) { - return converterClob2Str(c); - } - return "'" + escapeString(object) + "'"; - } - - public static String escapeString(Object object) { - String s = (String) object; - if (StringUtils.isBlank(s)) { - return ""; - } - return s.replace("\\", "\\\\").replace("'", "''"); - } - - public static String converterClob2Str(Clob c) { - StringBuilder stringBuilder = new StringBuilder(); - try (Reader reader = c.getCharacterStream()) { - BufferedReader bufferedReader = new BufferedReader(reader); - String line; - while ((line = bufferedReader.readLine()) != null) { - stringBuilder.append(line); - } - return escapeString(stringBuilder.toString()); - } catch (SQLException | IOException e) { - throw new RuntimeException(e); - } - } - - public static String converterByteArray2Str(byte[] bytes) { - return "0x" + BaseEncoding.base16().encode(bytes); - } -} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 0273682e6..a2b2e6294 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -17,7 +17,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -189,29 +188,65 @@ private String getInsertSql(String name, List columnList) { return script.toString(); } - private String getInsertSql(String schemaName, String tableName, List columnList, List valueList) { + /** + * Generates the base part of the INSERT SQL statement. + * Optionally includes column names if provided. + * + * @param schemaName Name of the database schema. + * @param tableName Name of the table to insert into. + * @param columnList Optional list of column names. + * @return The base part of the INSERT SQL statement. + */ + private String generateBaseInsertSql(String schemaName, String tableName, List columnList) { StringBuilder script = new StringBuilder(); - script.append("INSERT INTO ").append(schemaName).append(".").append(tableName) - .append(" (") - .append(String.join(",", columnList)) - .append(") VALUES (") - .append(String.join(",", valueList)) - .append(");"); + + if (StringUtils.isNotBlank(schemaName)) { + script.append(schemaName).append('.'); + } + + script.append(tableName); + + if (CollectionUtils.isNotEmpty(columnList)) { + script.append(" (") + .append(String.join(",", columnList)) + .append(") "); + } + + script.append("VALUES "); return script.toString(); } - private String getMultiInsertSql(String schemaName, String tableName, List columnList, List> valueList) { - StringBuilder script = new StringBuilder(); - script.append("INSERT INTO ").append(schemaName).append(".").append(tableName) - .append(" (") - .append(String.join(",", columnList)) - .append(") VALUES ") - .append(valueList.stream() - .map(values -> "(" + String.join(",", values) + ")") - .collect(Collectors.joining(",\n"))) - .append(");"); - return script.toString(); + /** + * Generates a single INSERT SQL statement for one record. + * + * @param schemaName Name of the database schema. + * @param tableName Name of the table to insert into. + * @param columnList Optional list of column names. + * @param valueList List of values to be inserted. + * @return The complete INSERT SQL statement for a single record. + */ + public String generateSingleInsertSql(String schemaName, String tableName, List columnList, List valueList) { + String baseSql = generateBaseInsertSql(schemaName, tableName, columnList); + return baseSql + "(" + String.join(",", valueList) + ");"; } + + /** + * Generates a multi-row INSERT SQL statement. + * + * @param schemaName Name of the database schema. + * @param tableName Name of the table to insert into. + * @param columnList Optional list of column names. + * @param valueLists List of lists, each inner list represents values for a row. + * @return The complete multi-row INSERT SQL statement. + */ + public String generateMultiInsertSql(String schemaName, String tableName, List columnList, List> valueLists) { + String baseSql = generateBaseInsertSql(schemaName, tableName, columnList); + String valuesPart = valueLists.stream() + .map(values -> "(" + String.join(",", values) + ")") + .collect(Collectors.joining(",\n")); + return baseSql + valuesPart + ";"; + } + private List getPrimaryColumns(List

headerList) { if (CollectionUtils.isEmpty(headerList)) { return Lists.newArrayList(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java new file mode 100644 index 000000000..569e5d543 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java @@ -0,0 +1,59 @@ +package ai.chat2db.spi.jdbc; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.lang3.time.DateUtils; + +import java.text.ParseException; + +/** + * @author: zgq + * @date: 2024年05月24日 14:30 + */ +public class DefaultValueProcessor extends BaseValueProcessor { + + public static final String[] DATE_FORMATS = { + "yyyy-MM-dd", + "yyyy-MM-dd HH:mm:ss", + "HH:mm:ss" + }; + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + String value = dataValue.getValue(); + return getString(value); + } + + + @Override + public Object convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getString(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + String value = dataValue.getString(); + return getString(value); + + } + + private boolean isNumber(String value) { + return NumberUtils.isCreatable(value); + } + + private String getString(String value) { + if (isNumber(value)) { + return value; + } + try { + DateUtils.parseDate(value, DATE_FORMATS); + return StringUtils.wrap(value, "'"); + } catch (ParseException e) { + return StringUtils.wrap(EasyStringUtils.escapeString(value), "'"); + } + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/DataType.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/DataType.java new file mode 100644 index 000000000..e61b948b2 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/DataType.java @@ -0,0 +1,32 @@ +package ai.chat2db.spi.model; + +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年05月31日 16:47 + */ +@Data +public class DataType { + + /** + * 数据类型的名称，如 "VARCHAR", "INTEGER", "DECIMAL", "DATE" 等。 + * 这个名称反映了数据库中字段的确切数据类型，是根据`ResultSetMetaData.getColumnTypeName()`获取的， + * 对理解和转换字段值至关重要，尤其是在处理数据库特定类型（如Oracle的NUMBER，MySQL的DATETIME）时。 + */ + private String dataTypeName; + + /** + * 精度（Precision），通常用于数值类型和字符串类型，表示该类型能够存储的最大字符数量或数字的总位数。 + * 对于数值类型，如`DECIMAL(5,2)`，精度5指的是整数部分加上小数部分的总位数。 + * 在从`ResultSetMetaData.getPrecision()`获取时，它帮助确定如何格式化数值，以确保数据的完整性和准确性。 + */ + private Integer precision; + + /** + * 小数位数（Scale），仅对数值类型有意义，表示小数点右侧的位数。 + * 例如，在`DECIMAL(5,2)`中，比例2表示小数点后保留两位数。 + * 通过`ResultSetMetaData.getScale()`获得，对于构造精确的数值字符串（特别是在财务和科学计算中）非常重要。 + */ + private Integer scale; +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java new file mode 100644 index 000000000..844b522f3 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -0,0 +1,73 @@ +package ai.chat2db.spi.model; + +import ai.chat2db.spi.util.ResultSetUtils; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.InputStream; +import java.sql.*; + +/** + * @author: zgq + * @date: 2024年05月30日 20:48 + */ +@Data +@AllArgsConstructor +public class JDBCDataValue { + private ResultSet resultSet; + private ResultSetMetaData metaData; + private int columnIndex; + + public Object getObject() { + try { + return resultSet.getObject(columnIndex); + } catch (Exception e) { + try { + return resultSet.getString(columnIndex); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + } + + public String getString() { + return ResultSetUtils.getString(resultSet, columnIndex); + } + + public String getType() { + return ResultSetUtils.getColumnDataTypeName(metaData, columnIndex); + } + + public InputStream getBinaryStream() { + return ResultSetUtils.getBinaryStream(resultSet, columnIndex); + } + + public int getPrecision() { + return ResultSetUtils.getColumnPrecision(metaData, columnIndex); + } + + public byte[] getBytes() { + return ResultSetUtils.getBytes(resultSet, columnIndex); + } + + public boolean getBoolean() { + return ResultSetUtils.getBoolean(resultSet, columnIndex); + } + + public int getScale() { + return ResultSetUtils.getColumnScale(metaData, columnIndex); + } + + public int getInt() { + + return ResultSetUtils.getInt(resultSet,columnIndex); + } + + public Date getDate() { + return ResultSetUtils.getDate(resultSet,columnIndex); + } + + public Timestamp getTimestamp() { + return ResultSetUtils.getTimestamp(resultSet,columnIndex); + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java new file mode 100644 index 000000000..eb434a277 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java @@ -0,0 +1,21 @@ +package ai.chat2db.spi.model; + +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年05月30日 15:01 + */ +@Data +public class SQLDataValue { + private String value; + private DataType dataType; + + public String getDateTypeName(){ + return dataType.getDataTypeName(); + } + + public int getPrecision(){ + return dataType.getPrecision(); + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 05f248ff1..6d867cc01 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -18,6 +18,7 @@ import ai.chat2db.spi.CommandExecutor; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.ValueHandler; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DataTypeEnum; import ai.chat2db.spi.enums.SqlTypeEnum; import ai.chat2db.spi.jdbc.DefaultValueHandler; @@ -30,7 +31,6 @@ import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; -import com.alibaba.druid.sql.parser.ParserException; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @@ -274,7 +274,9 @@ public ExecuteResult execute(final String sql, Connection connection, boolean li if (chat2dbAutoRowIdIndex == i) { continue; } - row.add(valueHandler.getString(rs, i, limitRowSize)); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + row.add((String) valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i))); +// row.add(valueHandler.getString(rs, i, limitRowSize)); } } else { for (int i = 1; i <= col; i++) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index c308d3555..76710bcb2 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -1,19 +1,18 @@ package ai.chat2db.spi.util; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.google.common.collect.Lists; +import java.io.InputStream; +import java.sql.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * @author jipengfei @@ -80,4 +79,84 @@ public static String getColumnName(ResultSetMetaData resultSetMetaData, int colu } return resultSetMetaData.getColumnName(column); } + + public static String getColumnDataTypeName(ResultSetMetaData resultSetMetaData, int columnIndex) { + try { + return resultSetMetaData.getColumnTypeName(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static int getColumnPrecision(ResultSetMetaData resultSetMetaData, int columnIndex){ + try { + return resultSetMetaData.getPrecision(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static int getColumnScale(ResultSetMetaData resultSetMetaData, int columnIndex){ + try { + return resultSetMetaData.getScale(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static String getString(ResultSet rs, int columnIndex){ + try { + return rs.getString(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static InputStream getBinaryStream(ResultSet rs, int columnIndex) { + try { + return rs.getBinaryStream(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static byte[] getBytes(ResultSet rs, int columnIndex) { + try { + return rs.getBytes(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static boolean getBoolean(ResultSet rs, int columnIndex) { + try { + return rs.getBoolean(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static int getInt(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getInt(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static Date getDate(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getDate(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static Timestamp getTimestamp(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getTimestamp(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file From 1f533933f92b947b1c2ab686cdb20ed73897c0c0 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 3 Jun 2024 22:28:22 +0800 Subject: [PATCH 06/73] Optimize MysqlValueProcessor --- .../mysql/value/MysqlBinaryProcessor.java | 35 ++++++++ .../plugin/mysql/value/MysqlBitProcessor.java | 4 +- .../mysql/value/MysqlDateTimeProcessor.java | 8 -- .../mysql/value/MysqlDecimalProcessor.java | 4 +- .../mysql/value/MysqlGeometryProcessor.java | 12 +-- .../mysql/value/MysqlTimestampProcessor.java | 2 +- .../mysql/value/MysqlValueProcessor.java | 36 ++++++-- .../mysql/value/MysqlVarBinaryProcessor.java | 82 +++++++++++++++++++ .../mysql/value/MysqlYearProcessor.java | 4 +- .../value/template/MysqlDmlValueTemplate.java | 6 ++ .../chat2db-server-tools-common/pom.xml | 5 ++ .../tools/common/util/EasyStringUtils.java | 26 ++++++ .../java/ai/chat2db/spi/ValueProcessor.java | 2 +- .../chat2db/spi/jdbc/BaseValueProcessor.java | 13 +-- .../spi/jdbc/DefaultValueProcessor.java | 24 +----- 15 files changed, 207 insertions(+), 56 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java delete mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java new file mode 100644 index 000000000..a378e0d51 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java @@ -0,0 +1,35 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import com.google.common.io.BaseEncoding; + +/** + * @author: zgq + * @date: 2024年06月03日 19:43 + */ +public class MysqlBinaryProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(BaseEncoding.base16().encode(dataValue.getValue().getBytes())); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return wrap(BaseEncoding.base16().encode(dataValue.getBytes())); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return convertJDBCValueByType(dataValue); + } + + private String wrap(String value) { + return String.format(MysqlDmlValueTemplate.BINARY_TEMPLATE, value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java index dfe2f1741..0389d3f4b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java @@ -22,7 +22,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { int precision = dataValue.getPrecision(); byte[] bytes = dataValue.getBytes(); if (precision == 1) { @@ -40,7 +40,7 @@ public Object convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getString((String) convertJDBCValueByType(dataValue)); + return getString(convertJDBCValueByType(dataValue)); } public String getString(String value) { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java deleted file mode 100644 index fe65e07ec..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDateTimeProcessor.java +++ /dev/null @@ -1,8 +0,0 @@ -package ai.chat2db.plugin.mysql.value; - -/** - * @author: zgq - * @date: 2024年06月01日 19:20 - */ -public class MysqlDateTimeProcessor extends MysqlTimestampProcessor{ -} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java index 2e00006c7..436a444e5 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java @@ -17,13 +17,13 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { return new String(dataValue.getBytes()); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return (String) convertJDBCValueByType(dataValue); + return convertJDBCValueByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java index b0e5e0d6e..3021ea93a 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java @@ -14,7 +14,7 @@ * @author: zgq * @date: 2024年06月01日 12:42 */ -public class MysqlGeometryProcessor extends DefaultValueProcessor { +public class MysqlGeometryProcessor extends DefaultValueProcessor { @Override @@ -23,7 +23,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { } @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { try { InputStream inputStream = dataValue.getBinaryStream(); Geometry dbGeometry = null; @@ -82,15 +82,11 @@ public Object convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - Object jdbcValue = getJdbcValue(dataValue); - if (jdbcValue == null) { - return super.getJdbcValueString(dataValue); - } - return wrap((String) jdbcValue); + return wrap(convertJDBCValueByType(dataValue)); } private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE,value); + return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE, value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java index 832c27352..9af8510e2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java @@ -18,7 +18,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { return isValidTimestamp(dataValue) ? new String(dataValue.getBytes()) : "0000-00-00 00:00:00"; } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index ed8bfd5fd..473af4231 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -26,35 +26,53 @@ public class MysqlValueProcessor extends DefaultValueProcessor { , MysqlColumnTypeEnum.MULTIPOLYGON.name() , MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name()); + public static final Set BINARY_TYPE = Set.of(MysqlColumnTypeEnum.VARBINARY.name() + , MysqlColumnTypeEnum.BLOB.name() + , MysqlColumnTypeEnum.LONGBLOB.name() + , MysqlColumnTypeEnum.TINYBLOB.name() + , MysqlColumnTypeEnum.MEDIUMBLOB.name()); + + public static final Set DATE_TYPE = Set.of(MysqlColumnTypeEnum.TIMESTAMP.name(), + MysqlColumnTypeEnum.DATETIME.name()); private static final Map PROCESSOR_MAP = Map.of( MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor(), MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor(), MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor(), - MysqlColumnTypeEnum.TIMESTAMP.name(), new MysqlTimestampProcessor(), - MysqlColumnTypeEnum.DATETIME.name(), new MysqlDateTimeProcessor() + MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor() ); - public static final Set FUNCTION_SET = Set.of("now()"); + public static final Set FUNCTION_SET = Set.of("now()", "default"); @Override public String convertSQLValueByType(SQLDataValue dataValue) { - if (FUNCTION_SET.contains(dataValue.getValue())) { + if (FUNCTION_SET.contains(dataValue.getValue().toLowerCase())) { return dataValue.getValue(); } String dataType = dataValue.getDateTypeName(); if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { return new MysqlGeometryProcessor().convertSQLValueByType(dataValue); } + if (BINARY_TYPE.contains(dataType)) { + return new MysqlVarBinaryProcessor().convertSQLValueByType(dataValue); + } + if (DATE_TYPE.contains(dataType)) { + return new MysqlTimestampProcessor().convertSQLValueByType(dataValue); + } return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertSQLValueByType(dataValue); } @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { - + public String convertJDBCValueByType(JDBCDataValue dataValue) { String dataType = dataValue.getType(); if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { return new MysqlGeometryProcessor().convertJDBCValueByType(dataValue); } + if (BINARY_TYPE.contains(dataType)) { + return new MysqlVarBinaryProcessor().convertJDBCValueByType(dataValue); + } + if (DATE_TYPE.contains(dataType)) { + return new MysqlTimestampProcessor().convertJDBCValueByType(dataValue); + } return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueByType(dataValue); } @@ -64,6 +82,12 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { return new MysqlGeometryProcessor().convertJDBCValueStrByType(dataValue); } + if (BINARY_TYPE.contains(dataType)) { + return new MysqlVarBinaryProcessor().convertJDBCValueStrByType(dataValue); + } + if (DATE_TYPE.contains(dataType)) { + return new MysqlTimestampProcessor().convertJDBCValueStrByType(dataValue); + } return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java new file mode 100644 index 000000000..187631b8b --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java @@ -0,0 +1,82 @@ +package ai.chat2db.plugin.mysql.value; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.apache.tika.Tika; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * @author: zgq + * @date: 2024年06月03日 20:48 + */ +public class MysqlVarBinaryProcessor extends MysqlBinaryProcessor { + private final static int KB = 1024; + private final static int MB = KB * 1024; + private final static int GB = MB * 1024; + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + // TODO: insert file + return super.convertSQLValueByType(dataValue); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + //TODO: Identify more file types + InputStream binaryStream = dataValue.getBinaryStream(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = binaryStream.read(buffer)) != -1) { + baos.write(buffer, 0, bytesRead); + } + baos.flush(); + + InputStream copiedStream = new ByteArrayInputStream(baos.toByteArray()); + + Tika tika = new Tika(); + String fileType = tika.detect(copiedStream); + + if ("image/jpeg".equals(fileType)) { + BufferedImage bufferedImage = ImageIO.read(copiedStream); + copiedStream.reset(); + long size = baos.size(); + String unit = "B"; + if (size > GB) { + size /= GB; + unit = "GB"; + } else if (size > MB) { + size /= MB; + unit = "MB"; + } else if (size > KB) { + size /= KB; + unit = "KB"; + } + return String.format(MysqlDmlValueTemplate.IMAGE_TEMPLATE, dataValue.getType(), + bufferedImage.getWidth(), bufferedImage.getHeight(), size, unit); + } else if ("text/plain".equals(fileType)) { + return baos.toString(StandardCharsets.UTF_8); + } + return super.convertJDBCValueByType(dataValue); + } catch (IOException e) { + return super.convertJDBCValueByType(dataValue); + } + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return super.convertJDBCValueStrByType(dataValue); + } +} + diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java index d686397f7..d47419b7c 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java @@ -22,7 +22,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { Date date = dataValue.getDate(); if (!isValidYear(dataValue)) { return "0000"; @@ -57,6 +57,6 @@ private boolean isValidYear(JDBCDataValue data) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return (String) getJdbcValue(dataValue); + return getJdbcValue(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java index f3547f62b..2caaab8eb 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -9,4 +9,10 @@ public class MysqlDmlValueTemplate { public static final String COMMON_TEMPLATE = "'%s'"; public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; public static final String BIT_TEMPLATE = "b'%s'"; + public static final String BINARY_TEMPLATE = "0x%s"; + + /** + * example: [VARBINARY] 525x542 JPEG image 34.67 KB + */ + public static final String IMAGE_TEMPLATE = "[%s] %dx%d JPEG image %d %s"; } diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml index 2958b662d..704b53b7c 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml @@ -18,6 +18,11 @@ ai.chat2db chat2db-server-tools-base + + org.apache.tika + tika-core + 2.7.0 + org.apache.commons commons-lang3 diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 2f117c31f..8033c61e2 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -184,6 +184,32 @@ public static String escapeString(String str) { return escapeString(str, escapeMap); } + /** + * @param value str="'abc\" + * @return "'''abc\\'" + */ + public static String escapeAndQuoteString(String value) { + return quoteString(escapeString(value)); + } + + /** + * @param value "abcd" + * @param quoteChar '%' + * @return "%abcd%" + */ + public static String quoteString(String value, char quoteChar) { + return StringUtils.wrap(value, quoteChar); + } + + /** + * @param value "abcd" + * @return "'abcd'" + */ + public static String quoteString(String value) { + // (char)39 -> ' + return quoteString(value, (char) 39); + } + public static String getBitString(byte[] bytes, final int precision) { if (bytes == null || bytes.length == 0) { return ""; diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java index 05bfaa06e..953a2ccc1 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java @@ -27,7 +27,7 @@ public interface ValueProcessor { * @param dataValue ResultSetMetaData, ResultSet, columnIndex的组合对象，用于获取数据值。 * @return 一个格式化后的字符串，适配于前端展示。例如，日期可能会转换为"YYYY-MM-DD"格式，以方便用户直观理解。 */ - Object getJdbcValue(JDBCDataValue dataValue); + String getJdbcValue(JDBCDataValue dataValue); /** * 将从JDBC ResultSet中获取的数据值转换并构造为适合DML语句的格式。 diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index 4c7a671ee..a0ca67500 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -1,5 +1,6 @@ package ai.chat2db.spi.jdbc; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -24,14 +25,14 @@ public String getSqlValueString(SQLDataValue dataValue) { @Override - public Object getJdbcValue(JDBCDataValue dataValue) { + public String getJdbcValue(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(dataValue.getObject())) { return null; } - if (value instanceof String stringValue) { - if (StringUtils.isBlank(stringValue)) { - return StringUtils.wrap(stringValue, "\""); + if (value instanceof String emptySry) { + if (StringUtils.isBlank(emptySry)) { + return emptySry; } } return convertJDBCValueByType(dataValue); @@ -46,7 +47,7 @@ public String getJdbcValueString(JDBCDataValue dataValue) { } if (value instanceof String stringValue) { if (StringUtils.isBlank(stringValue)) { - return StringUtils.wrap(stringValue, "'"); + return EasyStringUtils.quoteString(stringValue); } } return convertJDBCValueStrByType(dataValue); @@ -54,7 +55,7 @@ public String getJdbcValueString(JDBCDataValue dataValue) { public abstract String convertSQLValueByType(SQLDataValue dataValue); - public abstract Object convertJDBCValueByType(JDBCDataValue dataValue); + public abstract String convertJDBCValueByType(JDBCDataValue dataValue); public abstract String convertJDBCValueStrByType(JDBCDataValue dataValue); } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java index 569e5d543..8e63debe6 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java @@ -3,11 +3,7 @@ import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.commons.lang3.time.DateUtils; - -import java.text.ParseException; /** * @author: zgq @@ -15,29 +11,22 @@ */ public class DefaultValueProcessor extends BaseValueProcessor { - public static final String[] DATE_FORMATS = { - "yyyy-MM-dd", - "yyyy-MM-dd HH:mm:ss", - "HH:mm:ss" - }; @Override public String convertSQLValueByType(SQLDataValue dataValue) { - String value = dataValue.getValue(); - return getString(value); + return getString(dataValue.getValue()); } @Override - public Object convertJDBCValueByType(JDBCDataValue dataValue) { + public String convertJDBCValueByType(JDBCDataValue dataValue) { return dataValue.getString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - String value = dataValue.getString(); - return getString(value); + return getString(dataValue.getString()); } @@ -49,11 +38,6 @@ private String getString(String value) { if (isNumber(value)) { return value; } - try { - DateUtils.parseDate(value, DATE_FORMATS); - return StringUtils.wrap(value, "'"); - } catch (ParseException e) { - return StringUtils.wrap(EasyStringUtils.escapeString(value), "'"); - } + return EasyStringUtils.escapeAndQuoteString(value); } } From e32fce497982df1ea13f0adfd984db046b15022a Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 3 Jun 2024 23:22:49 +0800 Subject: [PATCH 07/73] Optimize code --- .../mysql/value/MysqlValueProcessor.java | 64 ++----------------- .../factory/MysqlValueProcessorFactory.java | 52 +++++++++++++++ .../value/{ => sub}/MysqlBinaryProcessor.java | 2 +- .../value/{ => sub}/MysqlBitProcessor.java | 2 +- .../{ => sub}/MysqlDecimalProcessor.java | 2 +- .../{ => sub}/MysqlGeometryProcessor.java | 2 +- .../{ => sub}/MysqlTimestampProcessor.java | 2 +- .../{ => sub}/MysqlVarBinaryProcessor.java | 2 +- .../value/{ => sub}/MysqlYearProcessor.java | 2 +- .../oracle/value/OracleValueProcessor.java | 30 +++++++++ .../factory/OracleValueProcessorFactory.java | 9 +++ .../template/OracleDmlValueTemplate.java | 2 +- 12 files changed, 103 insertions(+), 68 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlBinaryProcessor.java (95%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlBitProcessor.java (97%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlDecimalProcessor.java (93%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlGeometryProcessor.java (98%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlTimestampProcessor.java (96%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlVarBinaryProcessor.java (98%) rename chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/{ => sub}/MysqlYearProcessor.java (97%) create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java rename chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/{ => value}/template/OracleDmlValueTemplate.java (79%) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index 473af4231..5e53f0e8e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -1,11 +1,10 @@ package ai.chat2db.plugin.mysql.value; -import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; +import ai.chat2db.plugin.mysql.value.factory.MysqlValueProcessorFactory; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import java.util.Map; import java.util.Set; /** @@ -16,31 +15,6 @@ * attribute: [zerofill] example tinyint[5] zerofill 34->00034 */ public class MysqlValueProcessor extends DefaultValueProcessor { - - private static final Set GEOMETRY_TYPE = Set.of(MysqlColumnTypeEnum.GEOMETRY.name() - , MysqlColumnTypeEnum.POINT.name() - , MysqlColumnTypeEnum.LINESTRING.name() - , MysqlColumnTypeEnum.POLYGON.name() - , MysqlColumnTypeEnum.MULTIPOINT.name() - , MysqlColumnTypeEnum.MULTILINESTRING.name() - , MysqlColumnTypeEnum.MULTIPOLYGON.name() - , MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name()); - - public static final Set BINARY_TYPE = Set.of(MysqlColumnTypeEnum.VARBINARY.name() - , MysqlColumnTypeEnum.BLOB.name() - , MysqlColumnTypeEnum.LONGBLOB.name() - , MysqlColumnTypeEnum.TINYBLOB.name() - , MysqlColumnTypeEnum.MEDIUMBLOB.name()); - - public static final Set DATE_TYPE = Set.of(MysqlColumnTypeEnum.TIMESTAMP.name(), - MysqlColumnTypeEnum.DATETIME.name()); - - private static final Map PROCESSOR_MAP = Map.of( - MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor(), - MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor(), - MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor(), - MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor() - ); public static final Set FUNCTION_SET = Set.of("now()", "default"); @Override @@ -48,46 +22,16 @@ public String convertSQLValueByType(SQLDataValue dataValue) { if (FUNCTION_SET.contains(dataValue.getValue().toLowerCase())) { return dataValue.getValue(); } - String dataType = dataValue.getDateTypeName(); - if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { - return new MysqlGeometryProcessor().convertSQLValueByType(dataValue); - } - if (BINARY_TYPE.contains(dataType)) { - return new MysqlVarBinaryProcessor().convertSQLValueByType(dataValue); - } - if (DATE_TYPE.contains(dataType)) { - return new MysqlTimestampProcessor().convertSQLValueByType(dataValue); - } - return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertSQLValueByType(dataValue); + return MysqlValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()).convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - String dataType = dataValue.getType(); - if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { - return new MysqlGeometryProcessor().convertJDBCValueByType(dataValue); - } - if (BINARY_TYPE.contains(dataType)) { - return new MysqlVarBinaryProcessor().convertJDBCValueByType(dataValue); - } - if (DATE_TYPE.contains(dataType)) { - return new MysqlTimestampProcessor().convertJDBCValueByType(dataValue); - } - return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueByType(dataValue); + return MysqlValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - String dataType = dataValue.getType(); - if (GEOMETRY_TYPE.contains(dataType.toUpperCase())) { - return new MysqlGeometryProcessor().convertJDBCValueStrByType(dataValue); - } - if (BINARY_TYPE.contains(dataType)) { - return new MysqlVarBinaryProcessor().convertJDBCValueStrByType(dataValue); - } - if (DATE_TYPE.contains(dataType)) { - return new MysqlTimestampProcessor().convertJDBCValueStrByType(dataValue); - } - return PROCESSOR_MAP.getOrDefault(dataType, new DefaultValueProcessor()).convertJDBCValueStrByType(dataValue); + return MysqlValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java new file mode 100644 index 000000000..43f29e543 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -0,0 +1,52 @@ +package ai.chat2db.plugin.mysql.value.factory; + +import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; +import ai.chat2db.plugin.mysql.value.sub.*; +import ai.chat2db.plugin.mysql.value.template.sub.*; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; + +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年06月03日 23:16 + */ +public class MysqlValueProcessorFactory { + + private static final Map PROCESSOR_MAP; + + static { + MysqlGeometryProcessor mysqlGeometryProcessor = new MysqlGeometryProcessor(); + MysqlVarBinaryProcessor mysqlVarBinaryProcessor = new MysqlVarBinaryProcessor(); + MysqlTimestampProcessor mysqlTimestampProcessor = new MysqlTimestampProcessor(); + PROCESSOR_MAP = Map.ofEntries( + // geometry + Map.entry(MysqlColumnTypeEnum.GEOMETRY.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.POINT.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.LINESTRING.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.POLYGON.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTIPOINT.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTILINESTRING.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTIPOLYGON.name(), mysqlGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name(), mysqlGeometryProcessor), + // binary + Map.entry(MysqlColumnTypeEnum.BLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.LONGBLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.TINYBLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.MEDIUMBLOB.name(), mysqlVarBinaryProcessor), + // timestamp + Map.entry(MysqlColumnTypeEnum.TIMESTAMP.name(), mysqlTimestampProcessor), + Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mysqlTimestampProcessor), + //others + Map.entry(MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor()), + Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor()), + Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), + Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) + ); + } + + public static DefaultValueProcessor getValueProcessor(String type) { + DefaultValueProcessor processor = PROCESSOR_MAP.get(type); + return processor==null?new DefaultValueProcessor():processor; + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java similarity index 95% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index a378e0d51..411d7d4ff 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java similarity index 97% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java index 0389d3f4b..2cba0e86a 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.server.tools.common.util.EasyStringUtils; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java similarity index 93% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java index 436a444e5..9b29f23e8 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java similarity index 98% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java index 3021ea93a..76f05f98b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlGeometryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java similarity index 96% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 9af8510e2..93f0897fe 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java similarity index 98% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index 187631b8b..1624daeca 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.model.JDBCDataValue; diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java similarity index 97% rename from chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java rename to chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java index d47419b7c..db9f78311 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlYearProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.mysql.value; +package ai.chat2db.plugin.mysql.value.sub; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java new file mode 100644 index 000000000..25bf85401 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -0,0 +1,30 @@ +package ai.chat2db.plugin.oracle.value; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月03日 22:30 + */ +public class OracleValueProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return super.convertSQLValueByType(dataValue); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return super.convertJDBCValueByType(dataValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return super.convertJDBCValueStrByType(dataValue); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java new file mode 100644 index 000000000..4ed86f2b1 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -0,0 +1,9 @@ +package ai.chat2db.plugin.oracle.value.factory; + +/** + * @author: zgq + * @date: 2024年06月03日 23:21 + */ +public class OracleValueProcessorFactory { + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java similarity index 79% rename from chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java rename to chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index 52da8412d..eb9a70db7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -1,4 +1,4 @@ -package ai.chat2db.plugin.oracle.template; +package ai.chat2db.plugin.oracle.value.template; /** * @author: zgq From de9c187a689eca2f13df2859ef5bbd229f93e757 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 3 Jun 2024 23:40:12 +0800 Subject: [PATCH 08/73] Optimize code --- .../plugin/mysql/value/factory/MysqlValueProcessorFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index 43f29e543..b53d35afe 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -2,7 +2,6 @@ import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; import ai.chat2db.plugin.mysql.value.sub.*; -import ai.chat2db.plugin.mysql.value.template.sub.*; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import java.util.Map; @@ -47,6 +46,6 @@ public class MysqlValueProcessorFactory { public static DefaultValueProcessor getValueProcessor(String type) { DefaultValueProcessor processor = PROCESSOR_MAP.get(type); - return processor==null?new DefaultValueProcessor():processor; + return processor == null ? new DefaultValueProcessor() : processor; } } From f0b90cab0daeb5f3f064155635886c9d940b9053 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 5 Jun 2024 20:17:33 +0800 Subject: [PATCH 09/73] OracleValueProcessor --- .../factory/MysqlValueProcessorFactory.java | 6 + .../mysql/value/sub/MysqlBinaryProcessor.java | 8 +- .../value/sub/MysqlDecimalProcessor.java | 4 +- .../mysql/value/sub/MysqlTextProcessor.java | 49 ++++++ .../value/sub/MysqlVarBinaryProcessor.java | 57 +------ .../value/template/MysqlDmlValueTemplate.java | 6 - .../chat2db/plugin/oracle/OracleMetaData.java | 10 ++ .../oracle/value/OracleValueProcessor.java | 8 +- .../factory/OracleValueProcessorFactory.java | 40 ++++- .../oracle/value/sub/OracleBlobProcessor.java | 38 +++++ .../oracle/value/sub/OracleClobProcessor.java | 34 ++++ .../oracle/value/sub/OracleDateProcessor.java | 56 +++++++ .../value/sub/OracleIntervalDSProcessor.java | 35 ++++ .../value/sub/OracleIntervalYMProcessor.java | 36 +++++ .../value/sub/OracleNumberProcessor.java | 31 ++++ .../value/sub/OracleTimeStampProcessor.java | 52 ++++++ .../value/sub/OracleTimeStampTZProcessor.java | 41 +++++ .../template/OracleDmlValueTemplate.java | 10 +- .../ai/chat2db/spi/model/JDBCDataValue.java | 153 +++++++++++++++++- .../ai/chat2db/spi/model/SQLDataValue.java | 13 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 25 +++ 21 files changed, 638 insertions(+), 74 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index b53d35afe..8301e87ad 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -18,7 +18,13 @@ public class MysqlValueProcessorFactory { MysqlGeometryProcessor mysqlGeometryProcessor = new MysqlGeometryProcessor(); MysqlVarBinaryProcessor mysqlVarBinaryProcessor = new MysqlVarBinaryProcessor(); MysqlTimestampProcessor mysqlTimestampProcessor = new MysqlTimestampProcessor(); + MysqlTextProcessor mysqlTextProcessor = new MysqlTextProcessor(); PROCESSOR_MAP = Map.ofEntries( + //text + Map.entry(MysqlColumnTypeEnum.TEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.TINYTEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.MEDIUMTEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.LONGTEXT.name(), mysqlTextProcessor), // geometry Map.entry(MysqlColumnTypeEnum.GEOMETRY.name(), mysqlGeometryProcessor), Map.entry(MysqlColumnTypeEnum.POINT.name(), mysqlGeometryProcessor), diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index 411d7d4ff..84ebeae4e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -14,13 +14,13 @@ public class MysqlBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(BaseEncoding.base16().encode(dataValue.getValue().getBytes())); + return dataValue.getBlobHexString(); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return wrap(BaseEncoding.base16().encode(dataValue.getBytes())); + return dataValue.getBlobHexString(); } @@ -28,8 +28,4 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { public String convertJDBCValueStrByType(JDBCDataValue dataValue) { return convertJDBCValueByType(dataValue); } - - private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.BINARY_TEMPLATE, value); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java index 9b29f23e8..781fc8639 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java @@ -12,13 +12,13 @@ public class MysqlDecimalProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); + return super.convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return new String(dataValue.getBytes()); + return dataValue.getBigDecimalString(); } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java new file mode 100644 index 000000000..0d4589047 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java @@ -0,0 +1,49 @@ +package ai.chat2db.plugin.mysql.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import lombok.extern.slf4j.Slf4j; + +import java.util.function.Function; + +/** + * @author: zgq + * @date: 2024年06月05日 0:11 + */ +@Slf4j +public class MysqlTextProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return getClobString(dataValue, true, super::convertJDBCValueByType); + } + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(getClobString(dataValue, false, super::convertJDBCValueStrByType)); + } + + private String getClobString(JDBCDataValue dataValue, boolean limitSize, Function function) { + try { + return dataValue.getClobString(limitSize); + } catch (Exception e) { + log.warn("convertJDBCValue error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); + return function.apply(dataValue); + } + } + + private String wrap(String value) { + return EasyStringUtils.escapeAndQuoteString(value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index 1624daeca..fc4e853fa 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,26 +1,16 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import org.apache.tika.Tika; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import ai.chat2db.spi.sql.Chat2DBContext; +import lombok.extern.slf4j.Slf4j; /** * @author: zgq * @date: 2024年06月03日 20:48 */ +@Slf4j public class MysqlVarBinaryProcessor extends MysqlBinaryProcessor { - private final static int KB = 1024; - private final static int MB = KB * 1024; - private final static int GB = MB * 1024; @Override public String convertSQLValueByType(SQLDataValue dataValue) { @@ -31,44 +21,11 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - //TODO: Identify more file types - InputStream binaryStream = dataValue.getBinaryStream(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = binaryStream.read(buffer)) != -1) { - baos.write(buffer, 0, bytesRead); - } - baos.flush(); - - InputStream copiedStream = new ByteArrayInputStream(baos.toByteArray()); - - Tika tika = new Tika(); - String fileType = tika.detect(copiedStream); - - if ("image/jpeg".equals(fileType)) { - BufferedImage bufferedImage = ImageIO.read(copiedStream); - copiedStream.reset(); - long size = baos.size(); - String unit = "B"; - if (size > GB) { - size /= GB; - unit = "GB"; - } else if (size > MB) { - size /= MB; - unit = "MB"; - } else if (size > KB) { - size /= KB; - unit = "KB"; - } - return String.format(MysqlDmlValueTemplate.IMAGE_TEMPLATE, dataValue.getType(), - bufferedImage.getWidth(), bufferedImage.getHeight(), size, unit); - } else if ("text/plain".equals(fileType)) { - return baos.toString(StandardCharsets.UTF_8); - } - return super.convertJDBCValueByType(dataValue); - } catch (IOException e) { + return dataValue.getBlobString(true); + } catch (Exception e) { + log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); return super.convertJDBCValueByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java index 2caaab8eb..f3547f62b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -9,10 +9,4 @@ public class MysqlDmlValueTemplate { public static final String COMMON_TEMPLATE = "'%s'"; public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; public static final String BIT_TEMPLATE = "b'%s'"; - public static final String BINARY_TEMPLATE = "0x%s"; - - /** - * example: [VARBINARY] 525x542 JPEG image 34.67 KB - */ - public static final String IMAGE_TEMPLATE = "[%s] %dx%d JPEG image %d %s"; } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 5da77632a..eac7b6f6a 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -11,8 +11,10 @@ import ai.chat2db.plugin.oracle.type.OracleColumnTypeEnum; import ai.chat2db.plugin.oracle.type.OracleDefaultValueEnum; import ai.chat2db.plugin.oracle.type.OracleIndexTypeEnum; +import ai.chat2db.plugin.oracle.value.OracleValueProcessor; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.jdbc.DefaultMetaService; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; @@ -340,4 +342,12 @@ public String getMetaDataName(String... names) { public List getSystemSchemas() { return systemSchemas; } + + /** + * @return + */ + @Override + public ValueProcessor getValueProcessor() { + return new OracleValueProcessor(); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 25bf85401..9f1c2e277 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -1,5 +1,6 @@ package ai.chat2db.plugin.oracle.value; +import ai.chat2db.plugin.oracle.value.factory.OracleValueProcessorFactory; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -13,18 +14,19 @@ public class OracleValueProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return OracleValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()).convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return super.convertJDBCValueByType(dataValue); + String type = dataValue.getType(); + return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return super.convertJDBCValueStrByType(dataValue); + return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 4ed86f2b1..2a545ce00 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -1,9 +1,47 @@ package ai.chat2db.plugin.oracle.value.factory; +import ai.chat2db.plugin.oracle.type.OracleColumnTypeEnum; +import ai.chat2db.plugin.oracle.value.sub.*; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; + +import java.util.Map; + /** * @author: zgq * @date: 2024年06月03日 23:21 - */ + */ // TODO: 1.空间数据类型 2.XML数据类型 3.动态类型数据 4.ANSI、DB2 和 SQL/DS 数据 public class OracleValueProcessorFactory { + private static final Map PROCESSOR_MAP; + + static { + OracleClobProcessor oracleClobProcessor = new OracleClobProcessor(); + OracleTimeStampProcessor oracleTimeStampProcessor = new OracleTimeStampProcessor(); + PROCESSOR_MAP = Map.ofEntries( + //clob + Map.entry(OracleColumnTypeEnum.CLOB.name(), oracleClobProcessor), + Map.entry(OracleColumnTypeEnum.NCLOB.name(), oracleClobProcessor), + Map.entry(OracleColumnTypeEnum.LONG.name(), oracleClobProcessor), + //date + Map.entry(OracleColumnTypeEnum.DATE.name(), new OracleDateProcessor()), + //timestamp + Map.entry(OracleColumnTypeEnum.TIMESTAMP.name(), oracleTimeStampProcessor), + Map.entry(OracleColumnTypeEnum.TIMESTAMP_WITH_LOCAL_TIME_ZONE.getColumnType().getTypeName(), oracleTimeStampProcessor), + Map.entry(OracleColumnTypeEnum.TIMESTAMP_WITH_TIME_ZONE.getColumnType().getTypeName(), new OracleTimeStampTZProcessor()), + //INTERVAL + Map.entry("INTERVALDS", new OracleIntervalDSProcessor()), + Map.entry("INTERVALYM", new OracleIntervalYMProcessor()), + //number + Map.entry(OracleColumnTypeEnum.NUMBER.name(), new OracleNumberProcessor()), + //blob + Map.entry(OracleColumnTypeEnum.BLOB.name(), new OracleBlobProcessor()) + ); + + } + + public static DefaultValueProcessor getValueProcessor(String type) { + DefaultValueProcessor processor = PROCESSOR_MAP.get(type); + return processor == null ? new DefaultValueProcessor() : processor; + } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java new file mode 100644 index 000000000..6d7b80c85 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -0,0 +1,38 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import lombok.extern.slf4j.Slf4j; + +/** + * @author: zgq + * @date: 2024年06月05日 20:06 + */ +@Slf4j +public class OracleBlobProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getBlobHexString(); + } + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + try { + return dataValue.getBlobString(true); + } catch (Exception e) { + log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); + return super.convertJDBCValueByType(dataValue); + } + + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return dataValue.getBlobHexString(); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java new file mode 100644 index 000000000..3ec726d49 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java @@ -0,0 +1,34 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月04日 17:06 + */ +public class OracleClobProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getClobString(true); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(dataValue.getClobString(false)); + } + + private String wrap(String value) { + return EasyStringUtils.escapeAndQuoteString(value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java new file mode 100644 index 000000000..c807fa670 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java @@ -0,0 +1,56 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * @author: zgq + * @date: 2024年06月04日 16:33 + */ +public class OracleDateProcessor extends DefaultValueProcessor { + + /** + * @param dataValue + * @return + */ + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue()); + } + + /** + * @param dataValue + * @return + */ + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + Timestamp timestamp = dataValue.getTimestamp(); + LocalDateTime localDateTime = timestamp.toLocalDateTime(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String formattedDateTime = localDateTime.format(formatter); + if (formattedDateTime.endsWith("00:00:00")) { + formattedDateTime = formattedDateTime.substring(0, formattedDateTime.length() - 9); + } + return formattedDateTime; + + } + + /** + * @param dataValue + * @return + */ + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(convertJDBCValueByType(dataValue)); + } + + private String wrap(String value) { + return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java new file mode 100644 index 000000000..11b619b01 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java @@ -0,0 +1,35 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月05日 18:56 + */ +public class OracleIntervalDSProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return super.convertJDBCValueByType(dataValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); + } + + private String wrap(String value, int precision, int scale) { + return String.format(OracleDmlValueTemplate.INTEGER_DAY_TO_SECOND_TEMPLATE, value, precision, scale); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java new file mode 100644 index 000000000..f11ca212f --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java @@ -0,0 +1,36 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月05日 18:58 + */ +public class OracleIntervalYMProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue(), dataValue.getPrecision()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return super.convertJDBCValueByType(dataValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision()); + } + + public String wrap(String value, int precision) { + return String.format(OracleDmlValueTemplate.INTEGER_YEAR_TO_MONTH_TEMPLATE, value, precision); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java new file mode 100644 index 000000000..987bdf6d0 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java @@ -0,0 +1,31 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月05日 20:00 + */ +public class OracleNumberProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return super.convertSQLValueByType(dataValue); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getBigDecimalString(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return convertJDBCValueByType(dataValue); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java new file mode 100644 index 000000000..44a9d8dd7 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java @@ -0,0 +1,52 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * @author: zgq + * @date: 2024年06月05日 16:20 + */ +public class OracleTimeStampProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue(), dataValue.getScale()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + // TODO: datagrip对timestampLTZ的处理是不受时区影响的，但其实这个字段就是为了可以协同时区问题的，有待商讨 + Timestamp timestamp = dataValue.getTimestamp(); + int scale = dataValue.getScale(); + LocalDateTime localDateTime = timestamp.toLocalDateTime(); + StringBuilder templateBuilder = new StringBuilder("yyyy-MM-dd HH:mm:ss"); + if (scale != 0) { + templateBuilder.append("."); + templateBuilder.append("S".repeat(scale)); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(templateBuilder.toString()); + return localDateTime.format(formatter); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); + } + + private String wrap(String value, int scale) { + if (scale == 0) { + return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + } + return String.format(OracleDmlValueTemplate.TIMESTAMP_TEMPLATE, value, scale); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java new file mode 100644 index 000000000..7a2d95744 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -0,0 +1,41 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月05日 17:32 + */ +public class OracleTimeStampTZProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue(), dataValue.getScale()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + // TODO: return:2024-06-05 17:32:52.849 +8:00 but it actually is 2024-06-05 17:32:52.849000 +8:00 + return super.convertJDBCValueByType(dataValue); + + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); + } + + private String wrap(String value, int scale) { + if (scale == 0) { + return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, value); + } + return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_TEMPLATE, value, scale); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index eb9a70db7..e9080ae7d 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -6,5 +6,13 @@ */ public class OracleDmlValueTemplate { - public static final String DATE_TEMPLATE = "TO_DATE('%s', 'SYYYY-MM-DD HH24:MI:SS')"; + public static final String DATE_TEMPLATE = "TO_DATE('%s', 'YYYY-MM-DD HH24:MI:SS')"; + + public static final String TIMESTAMP_TEMPLATE = "TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d')"; + + public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d TZH:TZM')"; + public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')"; + + public static final String INTEGER_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; + public static final String INTEGER_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 844b522f3..f42c62575 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -1,10 +1,19 @@ package ai.chat2db.spi.model; import ai.chat2db.spi.util.ResultSetUtils; +import com.google.common.io.BaseEncoding; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.Getter; +import org.apache.tika.Tika; +import org.jetbrains.annotations.NotNull; +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; +import java.math.BigDecimal; import java.sql.*; /** @@ -60,14 +69,152 @@ public int getScale() { public int getInt() { - return ResultSetUtils.getInt(resultSet,columnIndex); + return ResultSetUtils.getInt(resultSet, columnIndex); } public Date getDate() { - return ResultSetUtils.getDate(resultSet,columnIndex); + return ResultSetUtils.getDate(resultSet, columnIndex); } public Timestamp getTimestamp() { - return ResultSetUtils.getTimestamp(resultSet,columnIndex); + return ResultSetUtils.getTimestamp(resultSet, columnIndex); + } + + public Clob getClob() { + return ResultSetUtils.getClob(resultSet, columnIndex); + } + + public Blob getBlob() { + return ResultSetUtils.getBlob(resultSet, columnIndex); + } + + public String getBlobString(boolean limitSize) { + //TODO: Identify more file types + Blob blob = getBlob(); + LOBInfo blobInfo = getBlobInfo(blob); + if (blobInfo.getSize() == 0) { + return ""; + } + try { + InputStream binaryStream = blob.getBinaryStream(); + Tika tika = new Tika(); + String contentType = tika.detect(binaryStream); + if ("image/jpeg".equals(contentType)) { + + BufferedImage bufferedImage = ImageIO.read(binaryStream); + return String.format("[%s] %dx%d JPEG image %d %s", getType(), bufferedImage.getWidth(), bufferedImage.getHeight(), blobInfo.getSize(), blobInfo.getUnit()); + } else if ("text/plain".equals(contentType)) { + if (isBigSize(blobInfo.unit) && limitSize) { + return String.format("[%s] %d %s", getType(), blobInfo.size, blobInfo.unit); + } else { + return new String(binaryStream.readAllBytes()); + } + } + } catch (SQLException | IOException e) { + throw new RuntimeException(e); + } + return getString(); + } + + private LOBInfo getBlobInfo(Blob blob) { + try { + long size = blob.length(); + return getLobInfo(size); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public String getClobString(boolean limitSize) { + Clob clob = getClob(); + LOBInfo cLobInfo = getCLobInfo(clob); + int size = cLobInfo.getSize(); + if (size == 0) { + return ""; + } + String unit = cLobInfo.getUnit(); + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %d %s", getType(), size, unit); + } + StringBuilder builder = new StringBuilder(size); + String line; + try (BufferedReader reader = new BufferedReader(clob.getCharacterStream())) { + while ((line = reader.readLine()) != null) { + builder.append(line).append("\n"); + } + } catch (IOException | SQLException e) { + throw new RuntimeException(e); + } + return builder.toString(); + } + + private boolean isBigSize(String unit) { + return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); + } + + public LOBInfo getCLobInfo(Clob clob) { + try { + long size = clob.length(); + return getLobInfo(size); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + } + + @NotNull + private LOBInfo getLobInfo(long size) { + if (size == 0) { + return new LOBInfo(LobUnit.K.unit, 0); + } + return calculateSizeAndUnit(size); + } + + @NotNull + private LOBInfo calculateSizeAndUnit(long size) { + if (size > LobUnit.G.size) { + return new LOBInfo(LobUnit.G.unit, (int) (size / LobUnit.G.size)); + } else if (size > LobUnit.M.size) { + return new LOBInfo(LobUnit.M.unit, (int) (size / LobUnit.M.size)); + } else if (size > LobUnit.K.size) { + return new LOBInfo(LobUnit.K.unit, (int) (size / LobUnit.K.size)); + } else { + return new LOBInfo(LobUnit.B.unit, (int) size); + } + } + + public String getBlobHexString() { + return "0x" + BaseEncoding.base16().encode(getBytes()); + } + + public BigDecimal getBigDecimal() { + return ResultSetUtils.getBigDecimal(resultSet, columnIndex); + } + + public String getBigDecimalString() { + BigDecimal bigDecimal = getBigDecimal(); + return bigDecimal==null?new String(getBytes()):bigDecimal.toPlainString(); + } + + @Data + @AllArgsConstructor + public static class LOBInfo { + private String unit; + private int size; + } + + @Getter + public enum LobUnit { + B("B", 1), + K("KB", 1024), + M("MB", 1024 * 1024), + G("GB", 1024 * 1024 * 1024); + private final String unit; + private final int size; + + LobUnit(String unit, int size) { + this.unit = unit; + this.size = size; + } } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java index eb434a277..f285bc759 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java @@ -1,5 +1,6 @@ package ai.chat2db.spi.model; +import com.google.common.io.BaseEncoding; import lombok.Data; /** @@ -11,11 +12,19 @@ public class SQLDataValue { private String value; private DataType dataType; - public String getDateTypeName(){ + public String getDateTypeName() { return dataType.getDataTypeName(); } - public int getPrecision(){ + public int getPrecision() { return dataType.getPrecision(); } + + public int getScale() { + return dataType.getScale(); + } + + public String getBlobHexString() { + return "0x" + BaseEncoding.base16().encode(value.getBytes()); + } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index 76710bcb2..14b686f6b 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -8,6 +8,7 @@ import com.google.common.collect.Lists; import java.io.InputStream; +import java.math.BigDecimal; import java.sql.*; import java.util.HashMap; import java.util.List; @@ -159,4 +160,28 @@ public static Timestamp getTimestamp(ResultSet resultSet, int columnIndex) { throw new RuntimeException(e); } } + + public static Clob getClob(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getClob(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static Blob getBlob(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getBlob(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static BigDecimal getBigDecimal(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getBigDecimal(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file From 591188703aa939bf09ad025adee7c73619b22066 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 12 Jun 2024 10:57:46 +0800 Subject: [PATCH 10/73] MysqlValueProcessor and OracleValueProcessor --- .../plugin/mysql/builder/MysqlSqlBuilder.java | 18 +++-- .../mysql/value/MysqlValueProcessor.java | 6 +- .../factory/MysqlValueProcessorFactory.java | 1 + .../mysql/value/sub/MysqlBinaryProcessor.java | 4 +- .../mysql/value/sub/MysqlBitProcessor.java | 13 +++- .../mysql/value/sub/MysqlTextProcessor.java | 8 +- .../value/sub/MysqlVarBinaryProcessor.java | 10 ++- .../oracle/value/sub/OracleBlobProcessor.java | 2 +- .../oracle/value/sub/OracleClobProcessor.java | 4 +- .../ai/chat2db/spi/model/JDBCDataValue.java | 78 +++++++++++++++---- 10 files changed, 104 insertions(+), 40 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index dc27b5afa..cd8e4684b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -2,13 +2,11 @@ import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; import ai.chat2db.plugin.mysql.type.MysqlIndexTypeEnum; -import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.jdbc.DefaultSqlBuilder; import ai.chat2db.spi.model.Database; import ai.chat2db.spi.model.Table; import ai.chat2db.spi.model.TableColumn; import ai.chat2db.spi.model.TableIndex; -import cn.hutool.core.util.ArrayUtil; import org.apache.commons.lang3.StringUtils; import java.util.*; @@ -288,14 +286,14 @@ private static String[] moveElement(String[] originalArray, int from, int to, St // 连续数据数量 if (from < to) { for (int i = to; i < originalArray.length - 1; i++) { - if (originalArray[i+1].equals(targetArray[findIndex(targetArray, originalArray[i]) +1])) { + if (originalArray[i + 1].equals(targetArray[findIndex(targetArray, originalArray[i]) + 1])) { continuousDataCount.set(continuousDataCount.incrementAndGet()); } else { break; } } if (continuousDataCount.get() > 0) { - System.arraycopy(originalArray, from + 1, newArray, from, to - from +1); + System.arraycopy(originalArray, from + 1, newArray, from, to - from + 1); isContinuousData = true; } else { System.arraycopy(originalArray, from + 1, newArray, from, to - from); @@ -303,12 +301,20 @@ private static String[] moveElement(String[] originalArray, int from, int to, St } else { System.arraycopy(originalArray, to, newArray, to + 1, from - to); } - if (isContinuousData){ - newArray[to+continuousDataCount.get()] = temp; + if (isContinuousData) { + newArray[to + continuousDataCount.get()] = temp; } else { newArray[to] = temp; } return newArray; } + + @Override + protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { + if (StringUtils.isNotBlank(databaseName)) { + script.append("`").append(databaseName).append("`").append('.'); + } + script.append("`").append(tableName).append("`"); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index 5e53f0e8e..dcb16dca4 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -27,11 +27,13 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return MysqlValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); + String type = dataValue.getType(); + return MysqlValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return MysqlValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueStrByType(dataValue); + String type = dataValue.getType(); + return MysqlValueProcessorFactory.getValueProcessor(type).convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index 8301e87ad..fe9601c47 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -35,6 +35,7 @@ public class MysqlValueProcessorFactory { Map.entry(MysqlColumnTypeEnum.MULTIPOLYGON.name(), mysqlGeometryProcessor), Map.entry(MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name(), mysqlGeometryProcessor), // binary + Map.entry(MysqlColumnTypeEnum.VARBINARY.name(), mysqlVarBinaryProcessor), Map.entry(MysqlColumnTypeEnum.BLOB.name(), mysqlVarBinaryProcessor), Map.entry(MysqlColumnTypeEnum.LONGBLOB.name(), mysqlVarBinaryProcessor), Map.entry(MysqlColumnTypeEnum.TINYBLOB.name(), mysqlVarBinaryProcessor), diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index 84ebeae4e..d35edc708 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -1,10 +1,8 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import com.google.common.io.BaseEncoding; /** * @author: zgq @@ -26,6 +24,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return convertJDBCValueByType(dataValue); + return dataValue.getBlobHexString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java index 2cba0e86a..238796e42 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java @@ -40,7 +40,18 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getString(convertJDBCValueByType(dataValue)); + int precision = dataValue.getPrecision(); + byte[] bytes = dataValue.getBytes(); + if (precision == 1) { + //bit(1) [1 -> true] [0 -> false] + if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { + return String.valueOf(dataValue.getBoolean()); + } + // tinyint(1) + return String.valueOf(dataValue.getInt()); + } + //bit(m) m: 2~64 + return wrap(EasyStringUtils.getBitString(bytes, precision)); } public String getString(String value) { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java index 0d4589047..2aabdf2f3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java @@ -25,17 +25,17 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return getClobString(dataValue, true, super::convertJDBCValueByType); + return getClobString(dataValue, super::convertJDBCValueByType); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(getClobString(dataValue, false, super::convertJDBCValueStrByType)); + return wrap(getClobString(dataValue, super::convertJDBCValueStrByType)); } - private String getClobString(JDBCDataValue dataValue, boolean limitSize, Function function) { + private String getClobString(JDBCDataValue dataValue, Function function) { try { - return dataValue.getClobString(limitSize); + return dataValue.getClobString(); } catch (Exception e) { log.warn("convertJDBCValue error database: {} , error dataType: {} ", Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index fc4e853fa..bff921153 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,8 +1,10 @@ package ai.chat2db.plugin.mysql.value.sub; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; import ai.chat2db.spi.sql.Chat2DBContext; +import ch.qos.logback.core.model.processor.DefaultProcessor; import lombok.extern.slf4j.Slf4j; /** @@ -10,7 +12,7 @@ * @date: 2024年06月03日 20:48 */ @Slf4j -public class MysqlVarBinaryProcessor extends MysqlBinaryProcessor { +public class MysqlVarBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { @@ -22,9 +24,9 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { try { - return dataValue.getBlobString(true); + return dataValue.getBlobString(); } catch (Exception e) { - log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", + log.warn("convertJDBCValue error database: {} , error dataType: {} ", Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); return super.convertJDBCValueByType(dataValue); } @@ -33,7 +35,7 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return super.convertJDBCValueStrByType(dataValue); + return dataValue.getBlobHexString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java index 6d7b80c85..e246fdb88 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -21,7 +21,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { try { - return dataValue.getBlobString(true); + return dataValue.getBlobString(); } catch (Exception e) { log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java index 3ec726d49..1409efd57 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java @@ -19,13 +19,13 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getClobString(true); + return dataValue.getClobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(dataValue.getClobString(false)); + return wrap(dataValue.getClobString()); } private String wrap(String value) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index f42c62575..420cf1ba0 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -14,7 +14,9 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.*; +import java.util.Objects; /** * @author: zgq @@ -26,6 +28,7 @@ public class JDBCDataValue { private ResultSet resultSet; private ResultSetMetaData metaData; private int columnIndex; + private boolean limitSize; public Object getObject() { try { @@ -88,34 +91,54 @@ public Blob getBlob() { return ResultSetUtils.getBlob(resultSet, columnIndex); } - public String getBlobString(boolean limitSize) { - //TODO: Identify more file types + public String getBlobString() { Blob blob = getBlob(); LOBInfo blobInfo = getBlobInfo(blob); + String unit = blobInfo.getUnit(); if (blobInfo.getSize() == 0) { return ""; } - try { - InputStream binaryStream = blob.getBinaryStream(); + + try (InputStream binaryStream = blob.getBinaryStream()) { Tika tika = new Tika(); String contentType = tika.detect(binaryStream); - if ("image/jpeg".equals(contentType)) { - - BufferedImage bufferedImage = ImageIO.read(binaryStream); - return String.format("[%s] %dx%d JPEG image %d %s", getType(), bufferedImage.getWidth(), bufferedImage.getHeight(), blobInfo.getSize(), blobInfo.getUnit()); - } else if ("text/plain".equals(contentType)) { - if (isBigSize(blobInfo.unit) && limitSize) { - return String.format("[%s] %d %s", getType(), blobInfo.size, blobInfo.unit); - } else { - return new String(binaryStream.readAllBytes()); + FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); + if (Objects.isNull(fileTypeEnum)) { + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); } + return getBlobHexString(); + } + switch (fileTypeEnum) { + case IMAGE: + if (limitSize) { + try (InputStream imageStream = blob.getBinaryStream()) { + BufferedImage bufferedImage = ImageIO.read(imageStream); + return String.format("[%s] %dx%d JPEG image %d %s", + getType(), bufferedImage.getWidth(), + bufferedImage.getHeight(), blobInfo.getSize(), unit); + } + } else { + return getBlobHexString(); + } + case STRING: + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); + } else { + return new String(binaryStream.readAllBytes()); + } + default: + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); + } + return getBlobHexString(); } } catch (SQLException | IOException e) { throw new RuntimeException(e); } - return getString(); } + private LOBInfo getBlobInfo(Blob blob) { try { long size = blob.length(); @@ -125,7 +148,7 @@ private LOBInfo getBlobInfo(Blob blob) { } } - public String getClobString(boolean limitSize) { + public String getClobString() { Clob clob = getClob(); LOBInfo cLobInfo = getCLobInfo(clob); int size = cLobInfo.getSize(); @@ -165,7 +188,7 @@ public LOBInfo getCLobInfo(Clob clob) { @NotNull private LOBInfo getLobInfo(long size) { if (size == 0) { - return new LOBInfo(LobUnit.K.unit, 0); + return new LOBInfo(LobUnit.B.unit, 0); } return calculateSizeAndUnit(size); } @@ -193,7 +216,7 @@ public BigDecimal getBigDecimal() { public String getBigDecimalString() { BigDecimal bigDecimal = getBigDecimal(); - return bigDecimal==null?new String(getBytes()):bigDecimal.toPlainString(); + return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); } @Data @@ -217,4 +240,25 @@ public enum LobUnit { this.size = size; } } + + @Getter + public enum FileTypeEnum { + IMAGE("image/jpeg"), + STRING("text/plain"), + ; + private final String description; + + FileTypeEnum(String description) { + this.description = description; + } + + public static FileTypeEnum fromDescription(String description) { + for (FileTypeEnum fileType : values()) { + if (fileType.description.equals(description)) { + return fileType; + } + } + return null; + } + } } From 4621d303eef4ee4371c82d9e6f70fc6b15969e89 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 12 Jun 2024 11:00:32 +0800 Subject: [PATCH 11/73] switch --- .../plugin/mysql/builder/MysqlSqlBuilder.java | 14 +++++++------- .../main/java/ai/chat2db/spi/sql/SQLExecutor.java | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index cd8e4684b..28a7132a3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -310,11 +310,11 @@ private static String[] moveElement(String[] originalArray, int from, int to, St } - @Override - protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { - if (StringUtils.isNotBlank(databaseName)) { - script.append("`").append(databaseName).append("`").append('.'); - } - script.append("`").append(tableName).append("`"); - } +// @Override +// protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { +// if (StringUtils.isNotBlank(databaseName)) { +// script.append("`").append(databaseName).append("`").append('.'); +// } +// script.append("`").append(tableName).append("`"); +// } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 6d867cc01..ca37da503 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -274,9 +274,9 @@ public ExecuteResult execute(final String sql, Connection connection, boolean li if (chat2dbAutoRowIdIndex == i) { continue; } - ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); - row.add((String) valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i))); -// row.add(valueHandler.getString(rs, i, limitRowSize)); +// ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); +// row.add((String) valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i))); + row.add(valueHandler.getString(rs, i, limitRowSize)); } } else { for (int i = 1; i <= col; i++) { From 153ea3aecbe1c43a3378ee6896232706e3e15d34 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Thu, 13 Jun 2024 15:17:20 +0800 Subject: [PATCH 12/73] fix(chat2db-dm): correct spelling of NUMERIC in DMColumnTypeEnum --- .../main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java index ea9876c92..cd422fbde 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java @@ -89,7 +89,7 @@ public enum DMColumnTypeEnum implements ColumnBuilder { TEXT("TEXT", false, false, true, false, false, false, true, true, false, false), - NUMBERIC("NUMBERIC", true, true, true, false, false, false, true, true, false, false), + NUMERIC("NUMERIC", true, true, true, false, false, false, true, true, false, false), NUMBER("NUMBER", true, true, true, false, false, false, true, true, false, false), @@ -222,7 +222,7 @@ private String buildDataType(TableColumn column, DMColumnTypeEnum type) { return script.toString(); } - if (Arrays.asList(DECIMAL,DEC, FLOAT, NUMBER, TIMESTAMP, NUMBERIC).contains(type)) { + if (Arrays.asList(DECIMAL, DEC, FLOAT, NUMBER, TIMESTAMP, NUMERIC).contains(type)) { StringBuilder script = new StringBuilder(); script.append(columnType); if (column.getColumnSize() != null && column.getDecimalDigits() == null) { From cd58f82f393e1426c187afc3c52f7af2439f722d Mon Sep 17 00:00:00 2001 From: luojun Date: Fri, 14 Jun 2024 20:49:42 +0800 Subject: [PATCH 13/73] =?UTF-8?q?[fix]=EF=BC=9Asql=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=85=BC=E5=AE=B9xml=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java index d05a7826b..316d16406 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java @@ -37,6 +37,8 @@ public String getString(ResultSet rs, int index, boolean limitSize) throws SQLEx return largeStringBlob(blob, limitSize); } else if (obj instanceof Timestamp || obj instanceof LocalDateTime) { return largeTime(obj); + } else if (obj instanceof SQLXML){ + return ((SQLXML) obj).getString(); } else { return obj.toString(); } From 2fcf18d0d94ae272a321feac78ac34b88290cb15 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 17 Jun 2024 17:51:46 +0800 Subject: [PATCH 14/73] =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java index 4a05533ab..6b2e719f2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java @@ -54,7 +54,7 @@ public List databases(Connection connection) { private List systemSchemas = Arrays.asList("pg_toast", "pg_temp_1", "pg_toast_temp_1", "pg_catalog", "information_schema"); - @Override +/* @Override public List schemas(Connection connection, String databaseName) { List schemas = SQLExecutor.getInstance().execute(connection, "SELECT catalog_name, schema_name FROM information_schema.schemata;", resultSet -> { @@ -70,7 +70,7 @@ public List schemas(Connection connection, String databaseName) { return databases; }); return SortUtils.sortSchema(schemas, systemSchemas); - } + }*/ private static final String SELECT_TABLE_INDEX = "SELECT tmp.INDISPRIMARY AS Index_primary, tmp.TABLE_SCHEM, tmp.TABLE_NAME, tmp.NON_UNIQUE, tmp.INDEX_QUALIFIER, tmp.INDEX_NAME AS Key_name, tmp.indisclustered, tmp.ORDINAL_POSITION AS Seq_in_index, TRIM ( BOTH '\"' FROM pg_get_indexdef ( tmp.CI_OID, tmp.ORDINAL_POSITION, FALSE ) ) AS Column_name,CASE tmp.AM_NAME WHEN 'btree' THEN CASE tmp.I_INDOPTION [ tmp.ORDINAL_POSITION - 1 ] & 1 :: SMALLINT WHEN 1 THEN 'D' ELSE'A' END ELSE NULL END AS Collation, tmp.CARDINALITY, tmp.PAGES, tmp.FILTER_CONDITION , tmp.AM_NAME AS Index_method, tmp.DESCRIPTION AS Index_comment FROM ( SELECT n.nspname AS TABLE_SCHEM, ct.relname AS TABLE_NAME, NOT i.indisunique AS NON_UNIQUE, NULL AS INDEX_QUALIFIER, ci.relname AS INDEX_NAME,i.INDISPRIMARY , i.indisclustered , ( information_schema._pg_expandarray ( i.indkey ) ).n AS ORDINAL_POSITION, ci.reltuples AS CARDINALITY, ci.relpages AS PAGES, pg_get_expr ( i.indpred, i.indrelid ) AS FILTER_CONDITION, ci.OID AS CI_OID, i.indoption AS I_INDOPTION, am.amname AS AM_NAME , d.description FROM pg_class ct JOIN pg_namespace n ON ( ct.relnamespace = n.OID ) JOIN pg_index i ON ( ct.OID = i.indrelid ) JOIN pg_class ci ON ( ci.OID = i.indexrelid ) JOIN pg_am am ON ( ci.relam = am.OID ) left outer join pg_description d on i.indexrelid = d.objoid WHERE n.nspname = '%s' AND ct.relname = '%s' ) AS tmp ;"; From cd16555c45afb948e27663fba68e91935a05d21a Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 18 Jun 2024 13:56:25 +0800 Subject: [PATCH 15/73] data export --- .../plugin/mysql/builder/MysqlSqlBuilder.java | 14 +- .../domain/api/enums/ExportFileSuffix.java | 1 + .../datasource/DatabaseExportDataParam.java | 7 + .../domain/core/impl/TaskServiceImpl.java | 1 + .../tools/common/util/EasyStringUtils.java | 12 +- .../controller/rdb/DatabaseController.java | 51 ++--- .../controller/rdb/data/BaseDataExporter.java | 70 +++++++ .../controller/rdb/data/BaseDataImporter.java | 8 + .../rdb/data/BaseExcelExporter.java | 105 ++++++++++ .../rdb/data/BaseExcelImporter.java | 10 + .../rdb/data/DataExportStrategy.java | 13 ++ .../rdb/data/DataImportStrategy.java | 4 + .../rdb/data/csv/CsvDataExporter.java | 26 +++ .../rdb/data/csv/CsvDataImporter.java | 12 ++ .../strategy/ExportDBData2CsvStrategy.java | 54 ----- .../strategy/ExportDBData2ExcelStrategy.java | 53 ----- .../strategy/ExportDBData2JsonStrategy.java | 61 ------ .../strategy/ExportDBData2SqlStrategy.java | 33 ---- .../export/strategy/ExportDBDataStrategy.java | 65 ------- .../rdb/data/factory/DataExportFactory.java | 33 ++++ .../rdb/data/factory/DataImportFactory.java | 35 ++++ .../rdb/data/json/JsonDataExporter.java | 112 +++++++++++ .../rdb/data/json/JsonDataImporter.java | 12 ++ .../rdb/data/service/DatabaseDataService.java | 13 ++ .../data/service/impl/DatabaseDataImpl.java | 150 ++++++++++++++ .../rdb/data/sql/SqlDataExporter.java | 184 ++++++++++++++++++ .../controller/rdb/data/task/TaskManager.java | 69 +++++++ .../controller/rdb/data/task/TaskState.java | 28 +++ .../rdb/data/xls/XlsDataExporter.java | 24 +++ .../rdb/data/xls/XlsDataImporter.java | 13 ++ .../rdb/data/xlsx/XlsxDataExporter.java | 25 +++ .../rdb/data/xlsx/XlsxDataImporter.java | 12 ++ .../factory/ExportDBDataStrategyFactory.java | 31 --- .../request/DatabaseExportDataRequest.java | 10 + .../api/controller/task/TaskController.java | 47 +++-- .../main/java/ai/chat2db/spi/SqlBuilder.java | 19 +- .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 70 +++++-- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 16 ++ 38 files changed, 1133 insertions(+), 370 deletions(-) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataExportStrategy.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataImportStrategy.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataImporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataImportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/DatabaseDataService.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/impl/DatabaseDataImpl.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskManager.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskState.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataImporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index 06a2ad37a..9a2ad6c7c 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -385,11 +385,11 @@ private static String[] moveElement(String[] originalArray, int from, int to, St } -// @Override -// protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { -// if (StringUtils.isNotBlank(databaseName)) { -// script.append("`").append(databaseName).append("`").append('.'); -// } -// script.append("`").append(tableName).append("`"); -// } + @Override + protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { + if (StringUtils.isNotBlank(databaseName)) { + script.append("`").append(databaseName).append("`").append('.'); + } + script.append("`").append(tableName).append("`"); + } } diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java index 9b5d854ee..4b55204d3 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java @@ -14,6 +14,7 @@ public enum ExportFileSuffix { WORD(".docx"), //excel EXCEL(".xlsx"), + XLS(".xls"), //markdown MARKDOWN(".md"), //html diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 4e562cedf..8f60eca06 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -4,6 +4,8 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 13:17 @@ -12,7 +14,12 @@ @AllArgsConstructor @NoArgsConstructor public class DatabaseExportDataParam { + private Long dataSourceId; private String databaseName; private String schemaName; private String exportType; + private List tableNames; + private String sqyType; + private Boolean containsHeader; + } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TaskServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TaskServiceImpl.java index 25ba9a9fe..42f3f8fcc 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TaskServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TaskServiceImpl.java @@ -44,6 +44,7 @@ public ActionResult updateStatus(TaskUpdateParam param) { taskDO.setId(param.getId()); taskDO.setTaskStatus(param.getTaskStatus()); taskDO.setContent(param.getContent()); + taskDO.setDownloadUrl(param.getDownloadUrl()); MapperUtils.getTaskMapper().updateById(taskDO); return ActionResult.isSuccess(); } diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 8033c61e2..7ae6d796b 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -193,17 +193,17 @@ public static String escapeAndQuoteString(String value) { } /** - * @param value "abcd" - * @param quoteChar '%' - * @return "%abcd%" + * @param value "abcd" + * @param quoteChar '%' + * @return "%abcd%" */ public static String quoteString(String value, char quoteChar) { - return StringUtils.wrap(value, quoteChar); + return quoteChar + value + quoteChar; } /** - * @param value "abcd" - * @return "'abcd'" + * @param value "abcd" + * @return "'abcd'" */ public static String quoteString(String value) { // (char)39 -> ' diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 6348db871..6e4571151 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -1,5 +1,6 @@ package ai.chat2db.server.web.api.controller.rdb; +import ai.chat2db.server.domain.api.enums.TaskStatusEnum; import ai.chat2db.server.domain.api.param.MetaDataQueryParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseCreateParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; @@ -14,8 +15,9 @@ import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; -import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; +import ai.chat2db.server.web.api.controller.rdb.data.service.DatabaseDataService; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskManager; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskState; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseCreateRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportDataRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportRequest; @@ -26,12 +28,12 @@ import ai.chat2db.spi.model.Sql; import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.io.PrintWriter; -import java.lang.reflect.Constructor; import java.util.Objects; /** @@ -40,6 +42,7 @@ @ConnectionInfoAspect @RequestMapping("/api/rdb/database") @RestController +@Slf4j public class DatabaseController { @Autowired private RdbWebConverter rdbWebConverter; @@ -49,6 +52,8 @@ public class DatabaseController { @Autowired public DatabaseConverter databaseConverter; + @Autowired + private DatabaseDataService databaseDataService; /** * Query the database_schema_list contained in the database @@ -59,8 +64,8 @@ public class DatabaseController { @GetMapping("/database_schema_list") public DataResult databaseSchemaList(@Valid DataSourceBaseRequest request) { MetaDataQueryParam queryParam = MetaDataQueryParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); DataResult result = databaseService.queryDatabaseSchema(queryParam); MetaSchemaVO schemaDto2vo = rdbWebConverter.metaSchemaDto2vo(result.getData()); return DataResult.of(schemaDto2vo); @@ -69,8 +74,8 @@ public DataResult databaseSchemaList(@Valid DataSourceBaseRequest @GetMapping("list") public ListResult databaseList(@Valid DataSourceBaseRequest request) { DatabaseQueryAllParam queryParam = DatabaseQueryAllParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); ListResult result = databaseService.queryAll(queryParam); return ListResult.of(rdbWebConverter.databaseDto2vo(result.getData())); } @@ -95,7 +100,7 @@ public ActionResult deleteDatabase(@Valid @RequestBody DataSourceBaseRequest req */ @PostMapping("/create_database_sql") public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest request) { - if(StringUtils.isBlank(request.getName())){ + if (StringUtils.isBlank(request.getName())) { request.setName(request.getDatabaseName()); } Database database = databaseConverter.request2param(request); @@ -111,12 +116,13 @@ public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest @PostMapping("/modify_database") public ActionResult modifyDatabase(@Valid @RequestBody UpdateDatabaseRequest request) { DatabaseCreateParam param = DatabaseCreateParam.builder().name(request.getDatabaseName()) - .name(request.getNewDatabaseName()).build(); + .name(request.getNewDatabaseName()).build(); return databaseService.modifyDatabase(param); } + @PostMapping("/export") - public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response){ - String fileName = Objects.isNull(request.getSchemaName())?request.getDatabaseName() : request.getSchemaName(); + public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response) { + String fileName = Objects.isNull(request.getSchemaName()) ? request.getDatabaseName() : request.getSchemaName(); response.setContentType("text/sql"); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".sql"); response.setCharacterEncoding("utf-8"); @@ -130,17 +136,18 @@ public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, Ht } @PostMapping("/export_data") - public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getExportType()); - response.setCharacterEncoding("utf-8"); - DatabaseExportDataParam param = databaseConverter.request2param(request); - try { - Constructor constructor = targetClass.getDeclaredConstructor(); - ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doExport(param, response); - } catch (Exception e) { - throw new RuntimeException(e); - } + public DataResult exportData(@Valid @RequestBody DatabaseExportDataRequest request) { + DatabaseExportDataParam databaseExportDataParam = databaseConverter.request2param(request); + return databaseDataService.doExportAsync(databaseExportDataParam); + } + @GetMapping("/export_data_status/{taskId}") + public DataResult exportDataStatus(@PathVariable("taskId") Long taskId) { + TaskState task = TaskManager.getTask(taskId); + String state = task.getState(); + if (Objects.equals(state, TaskStatusEnum.FINISH.name()) || Objects.equals(state, TaskStatusEnum.ERROR.name())) { + TaskManager.removeTask(taskId); + } + return DataResult.of(task.getExportStatus()); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataExporter.java new file mode 100644 index 000000000..dc5fa1a43 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataExporter.java @@ -0,0 +1,70 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.web.api.util.StringUtils; +import ai.chat2db.spi.sql.Chat2DBContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.io.*; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * @author: zgq + * @date: 2024年06月04日 10:51 + */ +@Slf4j +public abstract class BaseDataExporter implements DataExportStrategy { + + protected String contentType; + protected String suffix; + + public static int BATCH_SIZE = 1000; + + @Override + public void doExport(DatabaseExportDataParam databaseExportDataParam, File file) throws IOException, SQLException { + List tableNames = databaseExportDataParam.getTableNames(); + if (CollectionUtils.isEmpty(tableNames)) { + throw new IllegalArgumentException("tableNames should not be null or empty"); + } + try (Connection connection = Chat2DBContext.getConnection()) { + if (tableNames.size() == 1) { + String tableName = tableNames.get(0); + if (StringUtils.isEmpty(tableName)) { + throw new IllegalArgumentException("tableName should not be null or empty"); + } + singleExport(connection, databaseExportDataParam,file); + } else { + multiExport(databaseExportDataParam, connection, file); + } + } + + } + + + private void multiExport(DatabaseExportDataParam databaseExportDataParam, + Connection connection, File file) throws IOException { + try (OutputStream outputStream = new FileOutputStream(file); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { + List tableNames = databaseExportDataParam.getTableNames(); + for (String tableName : tableNames) { + String fileName = tableName + suffix; + zipOutputStream.putNextEntry(new ZipEntry(fileName)); + try (ByteArrayOutputStream byteArrayOutputStream = multiExport(connection, databaseExportDataParam, tableName)) { + byteArrayOutputStream.writeTo(zipOutputStream); + zipOutputStream.closeEntry(); + } + } + } + } + + + protected abstract void singleExport(Connection connectionInfo, DatabaseExportDataParam databaseExportDataParam, File file) throws IOException, SQLException; + + + protected abstract ByteArrayOutputStream multiExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, String tableName); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataImporter.java new file mode 100644 index 000000000..99fac6299 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseDataImporter.java @@ -0,0 +1,8 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +/** + * @author: zgq + * @date: 2024年06月04日 10:52 + */ +public abstract class BaseDataImporter implements DataImportStrategy{ +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelExporter.java new file mode 100644 index 000000000..2adc541c3 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelExporter.java @@ -0,0 +1,105 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.enums.TaskStatusEnum; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskManager; +import ai.chat2db.spi.ValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.SQLExecutor; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author: zgq + * @date: 2024年06月04日 10:56 + */ +@Slf4j +public abstract class BaseExcelExporter extends BaseDataExporter { + @Override + protected void singleExport(Connection connection, DatabaseExportDataParam exportParam, File outputFile) { + ExcelTypeEnum excelType = getExcelType(); + try (OutputStream outputStream = new FileOutputStream(outputFile)) { + + + String tableName = exportParam.getTableNames().get(0); + String querySql = getQuerySql(exportParam, tableName); + + log.info("开始导出：{}表数据，导出类型：{}", tableName, excelType); + + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> + writeExcelData(resultSet, excelType, outputStream, tableName, exportParam.getContainsHeader())); + + } catch (IOException e) { + TaskManager.updateStatus(TaskStatusEnum.ERROR); + throw new RuntimeException(e); + } + } + + @Override + protected ByteArrayOutputStream multiExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, String tableName) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ExcelTypeEnum excelType = getExcelType(); + + log.info("开始导出：{}表数据，导出类型：{}", tableName, excelType); + + String querySql = getQuerySql(databaseExportDataParam, tableName); + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> { + writeExcelData(resultSet, excelType, byteArrayOutputStream, tableName, databaseExportDataParam.getContainsHeader()); + }); + return byteArrayOutputStream; + } + + private void writeExcelData(ResultSet resultSet, ExcelTypeEnum excelType, OutputStream outputStream, String sheetName, Boolean containsHeader) { + try { + ExcelWriterSheetBuilder excelWriterSheetBuilder = EasyExcel.write(outputStream).excelType(excelType).sheet(sheetName); + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + List> dataList = new ArrayList<>(); + + if (containsHeader) { + List header = ResultSetUtils.getRsHeader(resultSet); + excelWriterSheetBuilder.head(header.stream().map(Collections::singletonList).collect(Collectors.toList())); + } + + while (resultSet.next()) { + List rowDataList = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); + rowDataList.add(valueProcessor.getJdbcValue(jdbcDataValue)); + } + dataList.add(rowDataList); + } + + excelWriterSheetBuilder.doWrite(dataList); + TaskManager.increaseCurrent(); + } catch (SQLException e) { + TaskManager.updateStatus(TaskStatusEnum.ERROR); + log.error("Error writing Excel data", e); + throw new RuntimeException(e); + } + } + + private String getQuerySql(DatabaseExportDataParam databaseExportDataParam, String tableName) { + String databaseName = databaseExportDataParam.getDatabaseName(); + String schemaName = databaseExportDataParam.getSchemaName(); + return Chat2DBContext.getSqlBuilder().buildTableQuerySql(databaseName, schemaName, tableName); + } + + protected abstract ExcelTypeEnum getExcelType(); +} + diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelImporter.java new file mode 100644 index 000000000..65ab11e58 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseExcelImporter.java @@ -0,0 +1,10 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月04日 10:57 + */ +public abstract class BaseExcelImporter extends BaseDataImporter{ +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataExportStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataExportStrategy.java new file mode 100644 index 000000000..de651c6fb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataExportStrategy.java @@ -0,0 +1,13 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; + +public interface DataExportStrategy { + + + void doExport(DatabaseExportDataParam databaseExportDataParam, File file) throws IOException, SQLException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataImportStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataImportStrategy.java new file mode 100644 index 000000000..96a453766 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataImportStrategy.java @@ -0,0 +1,4 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +public interface DataImportStrategy { +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataExporter.java new file mode 100644 index 000000000..470164757 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataExporter.java @@ -0,0 +1,26 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:05 + */ +@Component("csvExporter") +public class CsvDataExporter extends BaseExcelExporter { + + + public CsvDataExporter() { + this.contentType = "text/csv"; + this.suffix = ExportFileSuffix.CSV.getSuffix(); + } + + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataImporter.java new file mode 100644 index 000000000..28cb5f52a --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CsvDataImporter.java @@ -0,0 +1,12 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelImporter; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:04 + */ +@Component("csvImporter") +public class CsvDataImporter extends BaseExcelImporter { +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java deleted file mode 100644 index d091bf0ca..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ /dev/null @@ -1,54 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.support.ExcelTypeEnum; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { - - public ExportDBData2CsvStrategy() { - suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - int columnCount = metaData.getColumnCount(); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - EasyExcel.write(byteOut).excelType(ExcelTypeEnum.CSV).sheet(tableName).head(headList).doWrite(dataList); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java deleted file mode 100644 index b8fc38ee8..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ /dev/null @@ -1,53 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class ExportDBData2ExcelStrategy extends ExportDBDataStrategy { - - public ExportDBData2ExcelStrategy() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - EasyExcel.write(byteOut).sheet(tableName).head(headList).doWrite(dataList); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java deleted file mode 100644 index 47c8a59d8..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ /dev/null @@ -1,61 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.base.excption.BusinessException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.*; - -public class ExportDBData2JsonStrategy extends ExportDBDataStrategy { - - public ExportDBData2JsonStrategy() { - suffix = ExportFileSuffix.JSON.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("SELECT * FROM %s", tableName); - } else { - sql = String.format("SELECT * FROM %s.%s", schemaName, tableName); - } - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> data = new ArrayList<>(); - - while (resultSet.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - row.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - data.add(row); - } - - ObjectMapper objectMapper = new ObjectMapper(); - try { - String jsonString = objectMapper.writeValueAsString(data); - writer.println(jsonString); - } catch (IOException e) { - throw new BusinessException("data.export2Json.error",data.toArray(),e); - } - } - } - return byteOut; - } - - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java deleted file mode 100644 index 1e0577474..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ /dev/null @@ -1,33 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.sql.Chat2DBContext; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.SQLException; - -/** - * @author: zgq - * @date: 2024年03月24日 12:50 - */ -public class ExportDBData2SqlStrategy extends ExportDBDataStrategy { - - public ExportDBData2SqlStrategy() { - suffix = ExportFileSuffix.SQL.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql = Chat2DBContext.getDBManage().exportDatabaseData(connection, databaseName, schemaName, tableName); - writer.println(sql); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java deleted file mode 100644 index cb4a18209..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ /dev/null @@ -1,65 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.spi.sql.Chat2DBContext; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletResponse; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * @author: zgq - * @date: 2024年03月24日 12:46 - */ -@Getter -@AllArgsConstructor -@NoArgsConstructor -public abstract class ExportDBDataStrategy { - - public String suffix; - public String contentType; - - public void doExport(DatabaseExportDataParam param, HttpServletResponse response) { - String databaseName = param.getDatabaseName(); - String schemaName = param.getSchemaName(); - setResponseHeaders(param, response); - try (ServletOutputStream outputStream = response.getOutputStream(); - ZipOutputStream zipOut = new ZipOutputStream(outputStream); - Connection connection = Chat2DBContext.getConnection();) { - List tableNames = Chat2DBContext.getMetaData().tableNames(connection, databaseName, schemaName, null); - tableNames.addAll(Chat2DBContext.getMetaData().viewNames(connection, databaseName, schemaName)); - for (String tableName : tableNames) { - String fileName = tableName + getSuffix(); - zipOut.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = exportData(connection, databaseName, schemaName, tableName); - byteOut.writeTo(zipOut); - zipOut.closeEntry(); - byteOut.close(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - response.setContentType(contentType); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(param) + ExportFileSuffix.ZIP.getSuffix()); - } - - protected String getFileName(DatabaseExportDataParam param) { - return Objects.isNull(param.getSchemaName()) ? param.getDatabaseName() : param.getSchemaName(); - } - - protected abstract ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataExportFactory.java new file mode 100644 index 000000000..e82412728 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataExportFactory.java @@ -0,0 +1,33 @@ +package ai.chat2db.server.web.api.controller.rdb.data.factory; + +import ai.chat2db.server.tools.common.exception.ParamBusinessException; +import ai.chat2db.server.web.api.controller.rdb.data.DataExportStrategy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年06月04日 10:26 + */ +@Component +public class DataExportFactory { + + public static final String BEAN_SUFFIX = "Exporter"; + private final Map exports; + + @Autowired + public DataExportFactory(Map exports) { + this.exports = exports; + } + + public DataExportStrategy getExporter(String type) { + DataExportStrategy dataExportStrategy = exports.get(type.toLowerCase() + BEAN_SUFFIX); + if (Objects.isNull(dataExportStrategy)) { + throw new ParamBusinessException(type); + } + return dataExportStrategy; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataImportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataImportFactory.java new file mode 100644 index 000000000..8e8b4afa2 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/factory/DataImportFactory.java @@ -0,0 +1,35 @@ +package ai.chat2db.server.web.api.controller.rdb.data.factory; + +import ai.chat2db.server.tools.common.exception.ParamBusinessException; +import ai.chat2db.server.web.api.controller.rdb.data.DataImportStrategy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年06月04日 10:07 + */ +@Component +public class DataImportFactory { + + + private static final String BEAN_SUFFIX = "Importer"; + private final Map imports; + + @Autowired + public DataImportFactory(Map imports) { + this.imports = imports; + } + + public DataImportStrategy getImporter(String type) { + DataImportStrategy dataImportStrategy = imports.get(type.toLowerCase() + BEAN_SUFFIX); + if (Objects.isNull(dataImportStrategy)) { + throw new ParamBusinessException(type); + } + return dataImportStrategy; + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataExporter.java new file mode 100644 index 000000000..39112c059 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataExporter.java @@ -0,0 +1,112 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.web.api.controller.rdb.data.BaseDataExporter; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskManager; +import ai.chat2db.spi.ValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.SQLExecutor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSetMetaData; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年06月04日 10:33 + */ +@Component("jsonExporter") +@Slf4j +public class JsonDataExporter extends BaseDataExporter { + + public JsonDataExporter() { + this.suffix = ExportFileSuffix.JSON.getSuffix(); + this.contentType = "application/json"; + } + + + @Override + protected void singleExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, File file) { + String tableName = databaseExportDataParam.getTableNames().get(0); + String querySql = getQuerySql(databaseExportDataParam, tableName); + log.info("开始导出：{}表数据，导出类型：json", tableName); + try (PrintWriter writer = new PrintWriter(file, StandardCharsets.UTF_8);) { + writeJsonData(connection, querySql, writer); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected ByteArrayOutputStream multiExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, String tableName) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + log.info("开始导出：{}表数据，导出类型：json", tableName); + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteArrayOutputStream, StandardCharsets.UTF_8))) { + String querySql = getQuerySql(databaseExportDataParam, tableName); + writeJsonData(connection, querySql, writer); + } + return byteArrayOutputStream; + } + + private void writeJsonData(Connection connection, String querySql, PrintWriter writer) { + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> { + List> dataBatch = new ArrayList<>(); + ResultSetMetaData metaData = resultSet.getMetaData(); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + writer.println("["); + boolean firstBatch = true; + while (resultSet.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + row.put(metaData.getColumnName(i), valueProcessor.getJdbcValue(new JDBCDataValue(resultSet, metaData, i, false))); + } + dataBatch.add(row); + + if (dataBatch.size() >= BATCH_SIZE || resultSet.isLast()) { + if (!firstBatch) { + writer.println(","); + } + writeBatch(writer, objectMapper, dataBatch); + firstBatch = false; + } + } + writer.println("]"); + }); + TaskManager.increaseCurrent(); + } + + private void writeBatch(PrintWriter writer, ObjectMapper objectMapper, List> dataBatch) { + try { + String jsonBatch = objectMapper.writeValueAsString(dataBatch); + writer.println(jsonBatch.substring(1, jsonBatch.length() - 1)); + writer.flush(); + dataBatch.clear(); + } catch (JsonProcessingException e) { + throw new BusinessException("data.export.json.error", null, e); + } + } + + private String getQuerySql(DatabaseExportDataParam databaseExportDataParam, String tableName) { + String databaseName = databaseExportDataParam.getDatabaseName(); + String schemaName = databaseExportDataParam.getSchemaName(); + return Chat2DBContext.getSqlBuilder().buildTableQuerySql(databaseName, schemaName, tableName); + } + + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataImporter.java new file mode 100644 index 000000000..9dabcbf19 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JsonDataImporter.java @@ -0,0 +1,12 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.web.api.controller.rdb.data.BaseDataImporter; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:33 + */ +@Component("jsonImporter") +public class JsonDataImporter extends BaseDataImporter { +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/DatabaseDataService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/DatabaseDataService.java new file mode 100644 index 000000000..4a1df4974 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/DatabaseDataService.java @@ -0,0 +1,13 @@ +package ai.chat2db.server.web.api.controller.rdb.data.service; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; + +/** + * @author: zgq + * @date: 2024年06月08日 10:32 + */ +public interface DatabaseDataService { + + DataResult doExportAsync(DatabaseExportDataParam databaseExportDataParam); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/impl/DatabaseDataImpl.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/impl/DatabaseDataImpl.java new file mode 100644 index 000000000..7e37f84c7 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/service/impl/DatabaseDataImpl.java @@ -0,0 +1,150 @@ +package ai.chat2db.server.web.api.controller.rdb.data.service.impl; + +import ai.chat2db.server.domain.api.enums.TaskStatusEnum; +import ai.chat2db.server.domain.api.enums.TaskTypeEnum; +import ai.chat2db.server.domain.api.param.TaskCreateParam; +import ai.chat2db.server.domain.api.param.TaskUpdateParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.domain.api.service.TaskService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import ai.chat2db.server.web.api.controller.rdb.data.factory.DataExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.factory.DataImportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.service.DatabaseDataService; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskManager; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskState; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectInfo; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.io.FileUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** + * @author: zgq + * @date: 2024年06月08日 10:32 + */ +@Service +@Slf4j +public class DatabaseDataImpl implements DatabaseDataService { + + public static final String EXPORT_DATA_TASK_TEMPLATE = "export_%s_data"; + public static final String IMPORT_DATA_TASK_TEMPLATE = "import_%s_data"; + @Autowired + private DataExportFactory dataExportFactory; + @Autowired + private DataImportFactory dataImportFactory; + @Autowired + private TaskService taskService; + + @Override + public DataResult doExportAsync(DatabaseExportDataParam databaseExportDataParam) { + List tableNames = databaseExportDataParam.getTableNames(); + String databaseName = databaseExportDataParam.getDatabaseName(); + String schemaName = databaseExportDataParam.getSchemaName(); + Long dataSourceId = databaseExportDataParam.getDataSourceId(); + String taskName = buildTaskName(tableNames, databaseName, schemaName); + String fileName = URLEncoder.encode( + taskName + "_" + LocalDateTime.now().format(DatePattern.PURE_DATETIME_FORMATTER), + StandardCharsets.UTF_8); + String suffix = "."; + int size = tableNames.size(); + if (size > 1) { + suffix += "zip"; + } else { + suffix += databaseExportDataParam.getExportType().toLowerCase(); + } + File file = FileUtil.createTempFile(fileName, suffix, true); + file.deleteOnExit(); + LoginUser loginUser = ContextUtils.getLoginUser(); + ConnectInfo connectInfo = Chat2DBContext.getConnectInfo().copy(); + DataResult dataResult = createTask(tableNames.get(0), databaseName, schemaName, dataSourceId, taskName); + Long taskId = dataResult.getData(); + CompletableFuture.runAsync(() -> { + buildContext(loginUser, connectInfo); + TaskManager.addTask(taskId, TaskState.builder().state(TaskStatusEnum.PROCESSING.name()).total(size) + .current(0).build()); + try { + dataExportFactory.getExporter(databaseExportDataParam.getExportType()).doExport(databaseExportDataParam, file); + } catch (IOException | SQLException e) { + throw new RuntimeException(e); + } + }).whenComplete((v, ex) -> { + updateStatus(taskId, file, ex); + removeContext(); + TaskManager.removeTaskId(); + }); + return dataResult; + + } + + private void updateStatus(Long id, File file, Throwable throwable) { + TaskUpdateParam updateParam = new TaskUpdateParam(); + updateParam.setId(id); + updateParam.setTaskProgress("1"); + updateParam.setDownloadUrl(file.getAbsolutePath()); + if (throwable != null) { + log.error("export error", throwable); + updateParam.setTaskStatus(TaskStatusEnum.ERROR.name()); + } else { + updateParam.setTaskStatus(TaskStatusEnum.FINISH.name()); + } + taskService.updateStatus(updateParam); + } + + private void removeContext() { + Dbutils.removeSession(); + ContextUtils.removeContext(); + Chat2DBContext.removeContext(); + } + + private DataResult createTask(String tableName, String databaseName, String schemaName, Long datasourceId, String taskName) { + TaskCreateParam param = new TaskCreateParam(); + param.setTaskName(taskName); + param.setTaskType(TaskTypeEnum.DOWNLOAD_TABLE_DATA.name()); + param.setDatabaseName(databaseName); + param.setSchemaName(schemaName); + param.setTableName(tableName); + param.setDataSourceId(datasourceId); + param.setUserId(ContextUtils.getUserId()); + param.setTaskProgress("0.1"); + return taskService.create(param); + } + + private void buildContext(LoginUser loginUser, ConnectInfo connectInfo) { + ContextUtils.setContext(Context.builder() + .loginUser(loginUser) + .build()); + Dbutils.setSession(); + Chat2DBContext.putContext(connectInfo); + } + + private String buildTaskName(List tableNames, String databaseName, String schemaName) { + StringBuilder taskNameBuilder = new StringBuilder(); + if (StringUtils.isNotBlank(databaseName)) { + taskNameBuilder.append(databaseName).append("_"); + } + if (StringUtils.isNotBlank(schemaName)) { + taskNameBuilder.append(schemaName).append("_"); + } + if (tableNames.size() == 1) { + taskNameBuilder.append(StringUtils.join(tableNames, "_")); + } + return String.format(EXPORT_DATA_TASK_TEMPLATE, taskNameBuilder); + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java new file mode 100644 index 000000000..bc2a330ad --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java @@ -0,0 +1,184 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.web.api.controller.rdb.data.BaseDataExporter; +import ai.chat2db.server.web.api.controller.rdb.data.task.TaskManager; +import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.SQLExecutor; +import ai.chat2db.spi.util.ResultSetUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年06月04日 10:33 + */ +@Component("sqlExporter") +@Slf4j +public class SqlDataExporter extends BaseDataExporter { + + public SqlDataExporter() { + this.suffix = ExportFileSuffix.SQL.getSuffix(); + this.contentType = "text/sql"; + } + + /** + * @param connection + * @param databaseExportDataParam + * @param file + */ + @Override + protected void singleExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, File file) { + String tableName = databaseExportDataParam.getTableNames().get(0); + log.info("开始导出：{}表数据，导出类型：sql", tableName); + try (PrintWriter writer = new PrintWriter(file);) { + exportSql(connection, databaseExportDataParam, tableName, writer); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + protected ByteArrayOutputStream multiExport(Connection connection, DatabaseExportDataParam databaseExportDataParam, String tableName) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + log.info("开始导出：{}表数据，导出类型：sql", tableName); + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteArrayOutputStream, StandardCharsets.UTF_8))) { + exportSql(connection, databaseExportDataParam, tableName, writer); + } + return byteArrayOutputStream; + } + + private void exportSql(Connection connection, DatabaseExportDataParam databaseExportDataParam, String tableName, PrintWriter writer) { + String databaseName = databaseExportDataParam.getDatabaseName(); + String schemaName = databaseExportDataParam.getSchemaName(); + Boolean containsHeader = databaseExportDataParam.getContainsHeader(); + MetaData metaData = Chat2DBContext.getMetaData(); + String querySql = metaData.getSqlBuilder().buildTableQuerySql(databaseName, schemaName, tableName); + SqlBuilder sqlBuilder = metaData.getSqlBuilder(); + ValueProcessor valueProcessor = metaData.getValueProcessor(); + String sqyType = databaseExportDataParam.getSqyType(); + + switch (sqyType) { + case "single" -> exportSingleInsert(connection, querySql, containsHeader, sqlBuilder, + valueProcessor, databaseName, schemaName, tableName, writer); + case "multi" -> exportMultiInsert(connection, querySql, containsHeader, sqlBuilder, + valueProcessor, databaseName, schemaName, tableName, writer); + case "update" -> exportUpdate(connection, querySql, sqlBuilder, valueProcessor, + databaseName, schemaName, tableName, writer); + default -> throw new IllegalArgumentException("Unsupported sqyType: " + sqyType); + } + } + + private void exportSingleInsert(Connection connection, String querySql, Boolean containsHeader, + SqlBuilder sqlBuilder, ValueProcessor valueProcessor, + String databaseName, String schemaName, String tableName, PrintWriter writer) { + List sqlList = new ArrayList<>(BATCH_SIZE); + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> { + List header = containsHeader ? ResultSetUtils.getRsHeader(resultSet) : null; + while (resultSet.next()) { + List rowData = extractRowData(resultSet, valueProcessor); + String sql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, header, rowData); + sqlList.add(sql); + if (sqlList.size() >= BATCH_SIZE || resultSet.isLast()) { + writeSqlList(writer, sqlList); + } + } + }); + TaskManager.increaseCurrent(); + } + + private void exportMultiInsert(Connection connection, String querySql, Boolean containsHeader, + SqlBuilder sqlBuilder, ValueProcessor valueProcessor, + String databaseName, String schemaName, String tableName, PrintWriter writer) { + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> { + List> dataList = new ArrayList<>(BATCH_SIZE); + List header = containsHeader ? ResultSetUtils.getRsHeader(resultSet) : null; + while (resultSet.next()) { + dataList.add(extractRowData(resultSet, valueProcessor)); + } + String sql = sqlBuilder.buildMultiInsertSql(databaseName, schemaName, tableName, header, dataList); + writer.println(sql); + writer.flush(); + }); + TaskManager.increaseCurrent(); + } + + private void exportUpdate(Connection connection, String querySql, SqlBuilder sqlBuilder, + ValueProcessor valueProcessor, + String databaseName, String schemaName, String tableName, PrintWriter writer) { + List sqlList = new ArrayList<>(BATCH_SIZE); + SQLExecutor.getInstance().execute(connection, querySql, BATCH_SIZE, resultSet -> { + Map primaryKeyMap = getPrimaryKeyMap(connection, databaseName, schemaName, tableName); + while (resultSet.next()) { + Map row = extractRowDataAsMap(resultSet, valueProcessor, primaryKeyMap); + String sql = sqlBuilder.buildUpdateSql(databaseName, schemaName, tableName, row, primaryKeyMap); + sqlList.add(sql); + if (sqlList.size() >= BATCH_SIZE || resultSet.isLast()) { + writeSqlList(writer, sqlList); + } + } + }); + TaskManager.increaseCurrent(); + } + + private List extractRowData(ResultSet resultSet, ValueProcessor valueProcessor) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + List rowData = new ArrayList<>(metaData.getColumnCount()); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); + rowData.add(valueProcessor.getJdbcValueString(jdbcDataValue)); + } + return rowData; + } + + private Map extractRowDataAsMap(ResultSet resultSet, ValueProcessor valueProcessor, + Map primaryKeyMap) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + Map row = new HashMap<>(metaData.getColumnCount()); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); + String columnName = metaData.getColumnName(i); + String jdbcValueString = valueProcessor.getJdbcValueString(jdbcDataValue); + if (primaryKeyMap.containsKey(columnName)) { + primaryKeyMap.put(columnName, jdbcValueString); + } else { + row.put(columnName, jdbcValueString); + } + } + return row; + } + + private Map getPrimaryKeyMap(Connection connection, String databaseName, + String schemaName, String tableName) throws SQLException { + Map primaryKeyMap = new HashMap<>(); + try (ResultSet primaryKeys = connection.getMetaData().getPrimaryKeys(databaseName, schemaName, tableName)) { + while (primaryKeys.next()) { + primaryKeyMap.put(primaryKeys.getString("COLUMN_NAME"), ""); + } + } + return primaryKeyMap; + } + + private void writeSqlList(PrintWriter writer, List sqlList) { + sqlList.forEach(writer::println); + sqlList.clear(); + } + + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskManager.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskManager.java new file mode 100644 index 000000000..a59e4c434 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskManager.java @@ -0,0 +1,69 @@ +package ai.chat2db.server.web.api.controller.rdb.data.task; + +import ai.chat2db.server.domain.api.enums.TaskStatusEnum; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + + +public class TaskManager { + public static final ThreadLocal TASK_ID = new ThreadLocal<>(); + public static final Map taskMap = new ConcurrentHashMap<>(); + + + public static void increaseCurrent(int current) { + TaskState task = getTask(); + task.setCurrent(task.getCurrent() + current); + if (task.getCurrent() >= task.getTotal()) { + task.setState(TaskStatusEnum.FINISH.name()); + } + } + + public static void increaseCurrent() { + TaskState task = getTask(); + task.setCurrent(task.getCurrent() +1); + if (task.getCurrent() >= task.getTotal()) { + task.setState(TaskStatusEnum.FINISH.name()); + } + } + + public static void updateStatus(TaskStatusEnum status) { + TaskState task = getTask(); + task.setState(status.name()); + } + + + public static void addTask(Long taskId, TaskState taskState) { + setTaskId(taskId); + taskMap.put(taskId, taskState); + } + + public static TaskState getTask(Long taskId) { + TaskState taskState = taskMap.get(taskId); + if (Objects.isNull(taskState)) { + throw new IllegalArgumentException("taskId is not valid"); + } + return taskState; + } + + public static TaskState getTask() { + return getTask(getTaskId()); + } + + public static void removeTask(Long taskId) { + taskMap.remove(taskId); + } + + public static void setTaskId(Long taskId) { + TASK_ID.set(taskId); + } + + public static Long getTaskId() { + return TASK_ID.get(); + } + + public static void removeTaskId() { + TASK_ID.remove(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskState.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskState.java new file mode 100644 index 000000000..c93bdf81c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/task/TaskState.java @@ -0,0 +1,28 @@ +package ai.chat2db.server.web.api.controller.rdb.data.task; + +import lombok.Builder; +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年06月10日 15:51 + */ +@Data +@Builder +public class TaskState { + private String taskId; + private String state; + private int total; + private int current; + + + public String getExportStatus() { + StringBuilder statusBuilder = new StringBuilder(); + statusBuilder.append("导出状态: ").append(state) + .append(" 导出进度: ") + .append(current).append("/") + .append(total); + return statusBuilder.toString(); + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataExporter.java new file mode 100644 index 000000000..e643aef86 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataExporter.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:34 + */ +@Component("xlsExporter") +public class XlsDataExporter extends BaseExcelExporter { + + public XlsDataExporter() { + this.suffix = ExportFileSuffix.XLS.getSuffix(); + this.contentType="application/vnd.ms-excel"; + } + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLS; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataImporter.java new file mode 100644 index 000000000..bd232b6c7 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XlsDataImporter.java @@ -0,0 +1,13 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelImporter; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:34 + */ +@Component("xlsImporter") +public class XlsDataImporter extends BaseExcelImporter { + +} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataExporter.java new file mode 100644 index 000000000..ce4ac61b1 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataExporter.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:34 + */ +@Component("xlsxExporter") +public class XlsxDataExporter extends BaseExcelExporter { + + public XlsxDataExporter() { + this.suffix = ExportFileSuffix.EXCEL.getSuffix(); + this.contentType="application/vnd.ms-excel"; + } + + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataImporter.java new file mode 100644 index 000000000..43b6dc4cc --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XlsxDataImporter.java @@ -0,0 +1,12 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.BaseExcelImporter; +import org.springframework.stereotype.Component; + +/** + * @author: zgq + * @date: 2024年06月04日 10:34 + */ +@Component("xlsxImporter") +public class XlsxDataImporter extends BaseExcelImporter { +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java deleted file mode 100644 index 638557b17..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.factory; - -import ai.chat2db.server.domain.api.enums.ExportTypeEnum; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.*; -import lombok.SneakyThrows; - -import java.util.Map; - -/** - * @author: zgq - * @date: 2024年03月24日 12:53 - */ -public class ExportDBDataStrategyFactory { - - public static final Map> SERVICE_MAP = Map.of( - ExportTypeEnum.SQL.getCode(), ExportDBData2SqlStrategy.class, - ExportTypeEnum.CSV.getCode(), ExportDBData2CsvStrategy.class, - ExportTypeEnum.EXCEL.getCode(), ExportDBData2ExcelStrategy.class, - ExportTypeEnum.JSON.getCode(), ExportDBData2JsonStrategy.class - ); - - @SneakyThrows - public static Class get(String type) { - Class dataResult = SERVICE_MAP.get(type); - if (dataResult == null) { - throw new ClassNotFoundException("no ExportUI was found"); - } else { - return dataResult; - } - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 41686b503..0f8a03fc1 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,11 +1,14 @@ package ai.chat2db.server.web.api.controller.rdb.request; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; +import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 12:36 @@ -16,4 +19,11 @@ public class DatabaseExportDataRequest extends DataSourceBaseRequest { @NotNull private String exportType; + @NotEmpty + private List tableNames; + /** + * single：单行插入，multi：多行插入，update：更新语句 + */ + private String sqyType; + private Boolean containsHeader; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/TaskController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/TaskController.java index b6c784117..54d3aa91b 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/TaskController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/TaskController.java @@ -8,19 +8,17 @@ import ai.chat2db.server.tools.base.wrapper.result.web.WebPageResult; import ai.chat2db.server.tools.common.util.ContextUtils; import ai.chat2db.server.web.api.aspect.ConnectionInfoAspect; +import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; -import org.springframework.core.io.UrlResource; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; -import java.net.MalformedURLException; +import java.io.*; @ConnectionInfoAspect @RequestMapping("/api/task") @@ -43,35 +41,42 @@ public WebPageResult list() { } @GetMapping("/download/{id}") - public ResponseEntity download(@PathVariable Long id) { + public void download(@PathVariable Long id, HttpServletResponse response) { DataResult task = taskService.get(id); - if(task.getData() == null){ + Task data = task.getData(); + if (data == null) { log.error("task is null"); throw new RuntimeException("task is null"); } - if(ContextUtils.getUserId() != task.getData().getUserId()){ + if (!ContextUtils.getUserId().equals(data.getUserId())) { log.error("task is not belong to user"); throw new RuntimeException("task is not belong to user"); } - Resource resource = null; - try { - resource = new UrlResource("file://"+task.getData().getDownloadUrl()); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + File file = new File(data.getDownloadUrl()); - if (resource.exists() || resource.isReadable()) { - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"") - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); - } else { - throw new RuntimeException("Could not read the file!"); + if (!file.exists() || !file.isFile()) { + log.error("File not found or is not a file: {}", file.getAbsolutePath()); + throw new RuntimeException("File not found or accessible"); } - } + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getName() + "\""); + response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); + + try (InputStream inputStream = new FileInputStream(file)) { + OutputStream outputStream = response.getOutputStream(); + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + outputStream.flush(); + } + } catch (IOException e) { + log.error("Error occurred while processing file download", e); + throw new RuntimeException("Error in file download", e); + } + } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java index 9089634cb..4e59080cf 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java @@ -1,8 +1,12 @@ package ai.chat2db.spi; -import ai.chat2db.spi.model.*; +import ai.chat2db.spi.model.Database; +import ai.chat2db.spi.model.OrderBy; +import ai.chat2db.spi.model.QueryResult; +import ai.chat2db.spi.model.Schema; import java.util.List; +import java.util.Map; public interface SqlBuilder { @@ -12,7 +16,7 @@ public interface SqlBuilder { * @param table * @return */ - String buildCreateTableSql(T table); + String buildCreateTableSql(T table); /** @@ -83,9 +87,18 @@ public interface SqlBuilder { /** * DML SQL + * * @param table * @param type * @return */ - String getTableDmlSql(T table,String type); + String getTableDmlSql(T table, String type); + + String buildTableQuerySql(String databaseName, String schemaName, String tableName); + + String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList); + + String buildMultiInsertSql(String databaseName, String schemaName, String tableName, List columnList, List> valueLists); + + String buildUpdateSql(String databaseName, String schemaName, String tableName, Map row,Map primaryKeyMap); } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 8f3964c86..33df0264f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -13,16 +13,26 @@ import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class DefaultSqlBuilder implements SqlBuilder
{ + @Override + public String buildTableQuerySql(String databaseName, String schemaName, String tableName) { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("SELECT * FROM "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + return sqlBuilder.toString(); + } + @Override public String buildCreateTableSql(Table table) { return null; @@ -198,19 +208,17 @@ private String getInsertSql(String name, List columnList) { * Generates the base part of the INSERT SQL statement. * Optionally includes column names if provided. * - * @param schemaName Name of the database schema. - * @param tableName Name of the table to insert into. - * @param columnList Optional list of column names. + * @param databaseName + * @param schemaName Name of the database schema. + * @param tableName Name of the table to insert into. + * @param columnList Optional list of column names. * @return The base part of the INSERT SQL statement. */ - private String generateBaseInsertSql(String schemaName, String tableName, List columnList) { + protected String buildBaseInsertSql(String databaseName, String schemaName, String tableName, List columnList) { StringBuilder script = new StringBuilder(); - if (StringUtils.isNotBlank(schemaName)) { - script.append(schemaName).append('.'); - } - - script.append(tableName); + script.append("INSERT INTO "); + buildTableName(databaseName, schemaName, tableName, script); if (CollectionUtils.isNotEmpty(columnList)) { script.append(" (") @@ -218,10 +226,21 @@ private String generateBaseInsertSql(String schemaName, String tableName, List columnList, List valueList) { - String baseSql = generateBaseInsertSql(schemaName, tableName, columnList); + public String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList) { + String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); return baseSql + "(" + String.join(",", valueList) + ");"; } @@ -245,14 +264,37 @@ public String generateSingleInsertSql(String schemaName, String tableName, List< * @param valueLists List of lists, each inner list represents values for a row. * @return The complete multi-row INSERT SQL statement. */ - public String generateMultiInsertSql(String schemaName, String tableName, List columnList, List> valueLists) { - String baseSql = generateBaseInsertSql(schemaName, tableName, columnList); + public String buildMultiInsertSql(String databaseName, String schemaName, String tableName, List columnList, List> valueLists) { + String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); String valuesPart = valueLists.stream() .map(values -> "(" + String.join(",", values) + ")") .collect(Collectors.joining(",\n")); return baseSql + valuesPart + ";"; } + + @Override + public String buildUpdateSql(String databaseName, String schemaName, String tableName, Map row, Map primaryKeyMap) { + StringBuilder script = new StringBuilder(); + script.append("UPDATE "); + buildTableName(databaseName, schemaName, tableName, script); + + script.append(" SET "); + List setClauses = row.entrySet().stream() + .map(entry -> entry.getKey() + " = " + entry.getValue()) + .collect(Collectors.toList()); + script.append(String.join(",", setClauses)); + + if (MapUtils.isNotEmpty(primaryKeyMap)) { + script.append(" WHERE "); + List whereClauses = primaryKeyMap.entrySet().stream() + .map(entry -> entry.getKey() + " = " + entry.getValue()) + .collect(Collectors.toList()); + script.append(String.join(" AND ", whereClauses)); + } + return script + ";"; + } + private List getPrimaryColumns(List
headerList) { if (CollectionUtils.isEmpty(headerList)) { return Lists.newArrayList(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 8db1999ad..235f1328a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -715,4 +715,20 @@ private ExecuteResult execute(String sql, Integer offset, Integer count) { } return executeResult; } + + public void execute(Connection connection, String sql, int batchSize, ResultSetConsumer consumer) { + log.info("execute:{}", sql); + try (Statement stmt = connection.createStatement()) { + stmt.setFetchSize(batchSize); + boolean query = stmt.execute(sql); + // Represents the query + if (query) { + try (ResultSet rs = stmt.getResultSet()) { + consumer.accept(rs); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } } From 56b76665f4edf152b553968898394c8d5b91a87c Mon Sep 17 00:00:00 2001 From: robin <850379744@qq.com> Date: Thu, 20 Jun 2024 17:22:13 +0800 Subject: [PATCH 16/73] =?UTF-8?q?=E9=80=9A=E4=B9=89=E5=8D=83=E9=97=AE?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/controller/ai/tongyi/model/TongyiChatCompletions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/tongyi/model/TongyiChatCompletions.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/tongyi/model/TongyiChatCompletions.java index 84e50ae71..f0066b9f2 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/tongyi/model/TongyiChatCompletions.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/tongyi/model/TongyiChatCompletions.java @@ -35,7 +35,7 @@ public class TongyiChatCompletions { */ @JsonCreator private TongyiChatCompletions( - @JsonProperty(value = "id") String id, + @JsonProperty(value = "request_id") String id, @JsonProperty(value = "output") TongyiChatOutput choices, @JsonProperty(value = "usage") TongyiChatCompletionsUsage usage) { this.id = id; From 9421c3d3f4ffb84939149ea36e8346e80204c84e Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Fri, 21 Jun 2024 17:21:28 +0800 Subject: [PATCH 17/73] batch export --- .../plugin/clickhouse/ClickHouseDBManage.java | 26 +- .../ai/chat2db/plugin/db2/DB2DBManage.java | 37 +- .../java/ai/chat2db/plugin/dm/DMDBManage.java | 99 ++-- .../java/ai/chat2db/plugin/h2/H2DBManage.java | 13 +- .../plugin/hive/HiveCommandExecutor.java | 5 +- .../chat2db/plugin/mysql/MysqlDBManage.java | 59 ++- .../chat2db/plugin/mysql/MysqlMetaData.java | 11 +- .../plugin/mysql/MysqlValueHandler.java | 90 ++-- .../mysql/value/GeometryValueHandler.java | 138 +++--- .../chat2db-plugins/chat2db-oceanbase/pom.xml | 5 + .../plugin/oceanbase/OceanBaseDBManage.java | 3 +- .../plugin/oceanbase/OceanBaseMetaData.java | 3 +- .../chat2db/plugin/oracle/OracleDBManage.java | 89 ++-- .../plugin/postgresql/PostgreSQLDBManage.java | 43 +- .../chat2db/plugin/sqlite/SqliteDBManage.java | 39 +- .../sqlserver/SqlServerCommandExecutor.java | 5 +- .../plugin/sqlserver/SqlServerDBManage.java | 67 +-- .../domain/core/impl/DatabaseServiceImpl.java | 16 +- .../core/impl/DlTemplateServiceImpl.java | 5 +- .../rdb/RdbDmlExportController.java | 5 +- .../controller/task/biz/TaskBizService.java | 5 +- .../java/ai/chat2db/spi/CommandExecutor.java | 2 +- .../main/java/ai/chat2db/spi/DBManage.java | 20 +- .../main/java/ai/chat2db/spi/MetaData.java | 2 - .../java/ai/chat2db/spi/ValueHandler.java | 34 +- .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 17 +- .../chat2db/spi/jdbc/DefaultMetaService.java | 6 - .../chat2db/spi/jdbc/DefaultValueHandler.java | 224 ++++----- .../ai/chat2db/spi/model/AsyncContext.java | 34 ++ .../java/ai/chat2db/spi/sql/SQLExecutor.java | 451 +++++++----------- .../ai/chat2db/spi/util/ResultSetUtils.java | 84 +++- 31 files changed, 872 insertions(+), 765 deletions(-) create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java index 75d782d2d..745ab65e6 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; import org.apache.commons.lang3.StringUtils; @@ -11,43 +12,50 @@ public class ClickHouseDBManage extends DefaultDBManage implements DBManage { @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTablesOrViewsOrDictionaries(connection, sqlBuilder, databaseName, schemaName,containData); - exportFunctions(connection, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTablesOrViewsOrDictionaries(connection, databaseName, schemaName,asyncContext); + exportFunctions(connection, asyncContext); } - private void exportFunctions(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportFunctions(Connection connection, AsyncContext asyncContext) throws SQLException { String sql ="SELECT name,create_query from system.functions where origin='SQLUserDefined'"; try(ResultSet resultSet=connection.createStatement().executeQuery(sql)){ while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(resultSet.getString("name")).append(";") .append("\n") .append(resultSet.getString("create_query")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTablesOrViewsOrDictionaries(Connection connection, StringBuilder sqlBuilder, String databaseName, String schemaName, boolean containData) throws SQLException { + private void exportTablesOrViewsOrDictionaries(Connection connection,String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql =String.format("SELECT create_table_query, has_own_data,engine,name from system.`tables` WHERE `database`='%s'", databaseName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { while (resultSet.next()) { + String ddl = resultSet.getString("create_table_query"); boolean dataFlag = resultSet.getInt("has_own_data") == 1; String tableType = resultSet.getString("engine"); String tableOrViewName = resultSet.getString("name"); if (Objects.equals("View", tableType)) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP VIEW IF EXISTS ").append(databaseName).append(".").append(tableOrViewName) .append(";").append("\n").append(ddl).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } else if (Objects.equals("Dictionary", tableType)) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP DICTIONARY IF EXISTS ").append(databaseName).append(".").append(tableOrViewName) .append(";").append("\n").append(ddl).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } else { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE IF EXISTS ").append(databaseName).append(".").append(tableOrViewName) .append(";").append("\n").append(ddl).append(";").append("\n"); - if (containData && dataFlag) { - exportTableData(connection,schemaName, tableOrViewName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData() && dataFlag) { + exportTableData(connection,schemaName, tableOrViewName, asyncContext); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java index 8546a962f..d5dc9445f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java @@ -3,6 +3,7 @@ import ai.chat2db.plugin.db2.constant.SQLConstant; import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; @@ -18,25 +19,23 @@ public class DB2DBManage extends DefaultDBManage implements DBManage { @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, schemaName, sqlBuilder, containData); - exportViews(connection, schemaName, sqlBuilder); - exportProceduresAndFunctions(connection, schemaName, sqlBuilder); - exportTriggers(connection, schemaName, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, schemaName, asyncContext); + exportViews(connection, schemaName, asyncContext); + exportProceduresAndFunctions(connection, schemaName, asyncContext); + exportTriggers(connection, schemaName, asyncContext); } - private void exportTables(Connection connection, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection, schemaName, resultSet.getString("TABLE_NAME"), sqlBuilder, containData); + exportTable(connection, schemaName, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, SQLConstant.TABLE_DDL_FUNCTION_SQL, resultSet -> null); } catch (Exception e) { @@ -45,42 +44,50 @@ private void exportTable(Connection connection, String schemaName, String tableN String sql = String.format("select %s.GENERATE_TABLE_DDL('%s', '%s') as sql from %s;", schemaName, schemaName, tableName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("sql")).append("\n"); - if (containData) { - exportTableData(connection, schemaName, tableName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection, schemaName, tableName, asyncContext); } } } } - private void exportViews(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select TEXT from syscat.views where VIEWSCHEMA='%s';", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); String ddl = resultSet.getString("TEXT"); sqlBuilder.append(ddl).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportProceduresAndFunctions(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportProceduresAndFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select TEXT from syscat.routines where ROUTINESCHEMA='%s';", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); String ddl = resultSet.getString("TEXT"); sqlBuilder.append(ddl).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select * from SYSCAT.TRIGGERS where TRIGSCHEMA = '%s';", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); String ddl = resultSet.getString("TEXT"); sqlBuilder.append(ddl).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java index cc2f0d74c..38282f301 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java @@ -5,6 +5,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; @@ -17,6 +18,7 @@ public class DMDBManage extends DefaultDBManage implements DBManage { private String format(String tableName) { return "\"" + tableName + "\""; } + private static String ROUTINES_SQL = "SELECT OWNER, NAME, TEXT FROM ALL_SOURCE WHERE TYPE = '%s' AND OWNER = '%s' AND NAME = '%s' ORDER BY LINE"; private static String TRIGGER_SQL_LIST = "SELECT OWNER, TRIGGER_NAME FROM ALL_TRIGGERS WHERE OWNER = '%s'"; @@ -24,38 +26,38 @@ private String format(String tableName) { private static String TRIGGER_SQL = "SELECT OWNER, TRIGGER_NAME, TABLE_OWNER, TABLE_NAME, TRIGGERING_TYPE, TRIGGERING_EVENT, STATUS, TRIGGER_BODY " + "FROM ALL_TRIGGERS WHERE OWNER = '%s' AND TRIGGER_NAME = '%s'"; + @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, sqlBuilder, schemaName, containData); - exportViews(connection, schemaName, sqlBuilder); - exportProcedures(connection, schemaName, sqlBuilder); - exportTriggers(connection,schemaName, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, schemaName, asyncContext); + exportViews(connection, schemaName, asyncContext); + exportProcedures(connection, schemaName, asyncContext); + exportTriggers(connection, schemaName, asyncContext); } - private void exportTables(Connection connection, StringBuilder sqlBuilder, String schemaName, boolean containData) throws SQLException { - String sql =String.format("SELECT TABLE_NAME FROM ALL_TABLES where OWNER='%s' and TABLESPACE_NAME='MAIN'", schemaName); + private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("SELECT TABLE_NAME FROM ALL_TABLES where OWNER='%s' and TABLESPACE_NAME='MAIN'", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); - exportTable(connection, tableName, schemaName, sqlBuilder, containData); + exportTable(connection, tableName, schemaName, asyncContext); } } } - private void exportTable(Connection connection, String tableName, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = """ - SELECT - (SELECT comments FROM user_tab_comments WHERE table_name = '%s') AS comments, - (SELECT dbms_metadata.get_ddl('TABLE', '%s', '%s') FROM dual) AS ddl - FROM dual; - """; + SELECT + (SELECT comments FROM user_tab_comments WHERE table_name = '%s') AS comments, + (SELECT dbms_metadata.get_ddl('TABLE', '%s', '%s') FROM dual) AS ddl + FROM dual; + """; try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(String.format(sql, tableName, tableName, schemaName))) { String formatSchemaName = format(schemaName); String formatTableName = format(tableName); if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE IF EXISTS ").append(formatSchemaName).append(".").append(formatTableName) .append(";").append("\n") .append(resultSet.getString("ddl")).append("\n"); @@ -64,79 +66,88 @@ private void exportTable(Connection connection, String tableName, String schemaN sqlBuilder.append("COMMENT ON TABLE ").append(formatSchemaName).append(".").append(formatTableName) .append(" IS ").append("'").append(comment).append("';"); } - exportTableColumnComment(connection, schemaName, tableName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + exportTableColumnComment(connection, schemaName, tableName, asyncContext); } - if (containData) { - exportTableData(connection, schemaName, tableName, sqlBuilder); + if (asyncContext.isContainsData()) { + exportTableData(connection, schemaName, tableName, asyncContext); } } } - private void exportTableColumnComment(Connection connection, String schemaName, String tableName, StringBuilder sqlBuilder) throws SQLException { - String sql =String.format("select COLNAME,COMMENT$ from SYS.SYSCOLUMNCOMMENTS\n" + - "where SCHNAME = '%s' and TVNAME = '%s'and TABLE_TYPE = 'TABLE';", schemaName,tableName); - try(ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - while (resultSet.next()) { - String columnName = resultSet.getString("COLNAME"); - String comment = resultSet.getString("COMMENT$"); - sqlBuilder.append("COMMENT ON COLUMN ").append(format(schemaName)).append(".").append(format(tableName)) - .append(".").append(format(columnName)).append(" IS ").append("'").append(comment).append("';").append("\n"); - } - } + private void exportTableColumnComment(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("select COLNAME,COMMENT$ from SYS.SYSCOLUMNCOMMENTS\n" + + "where SCHNAME = '%s' and TVNAME = '%s'and TABLE_TYPE = 'TABLE';", schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); + String columnName = resultSet.getString("COLNAME"); + String comment = resultSet.getString("COMMENT$"); + sqlBuilder.append("COMMENT ON COLUMN ").append(format(schemaName)).append(".").append(format(tableName)) + .append(".").append(format(columnName)).append(" IS ").append("'").append(comment).append("';").append("\n"); + asyncContext.write(sqlBuilder.toString()); + } + } } - private void exportViews(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"VIEW"})) { while (resultSet.next()) { String viewName = resultSet.getString("TABLE_NAME"); - exportView(connection, viewName, schemaName, sqlBuilder); + exportView(connection, viewName, schemaName, asyncContext); } } } - private void exportView(Connection connection, String viewName, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportView(Connection connection, String viewName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT DBMS_METADATA.GET_DDL('VIEW','%s','%s') as ddl FROM DUAL;", viewName, schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportProcedures(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getProcedures(null, schemaName, null)) { while (resultSet.next()) { String procedureName = resultSet.getString("PROCEDURE_NAME"); - exportProcedure(connection, schemaName,procedureName, sqlBuilder); + exportProcedure(connection, schemaName, procedureName, asyncContext); } } } - private void exportProcedure(Connection connection, String schemaName, String procedureName, StringBuilder sqlBuilder) throws SQLException { - String sql = String.format(ROUTINES_SQL,"PROC", schemaName,procedureName); + private void exportProcedure(Connection connection, String schemaName, String procedureName, AsyncContext asyncContext) throws SQLException { + String sql = String.format(ROUTINES_SQL, "PROC", schemaName, procedureName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("TEXT")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { - String sql =String.format(TRIGGER_SQL_LIST, schemaName); + private void exportTriggers(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { + String sql = String.format(TRIGGER_SQL_LIST, schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String triggerName = resultSet.getString("TRIGGER_NAME"); - exportTrigger(connection,schemaName, triggerName, sqlBuilder); + exportTrigger(connection, schemaName, triggerName, asyncContext); } } } - private void exportTrigger(Connection connection, String schemaName, String triggerName, StringBuilder sqlBuilder) throws SQLException { - String sql = String.format(TRIGGER_SQL, schemaName,triggerName); + private void exportTrigger(Connection connection, String schemaName, String triggerName, AsyncContext asyncContext) throws SQLException { + String sql = String.format(TRIGGER_SQL, schemaName, triggerName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("TRIGGER_BODY")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } @@ -157,7 +168,7 @@ public void connectDatabase(Connection connection, String database) { @Override public void dropTable(Connection connection, String databaseName, String schemaName, String tableName) { - String sql = "DROP TABLE IF EXISTS " +tableName; - SQLExecutor.getInstance().execute(connection,sql, resultSet -> null); + String sql = "DROP TABLE IF EXISTS " + tableName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2DBManage.java b/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2DBManage.java index 7f3bf75ec..aa02e2c30 100644 --- a/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2DBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2DBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; @@ -16,23 +17,23 @@ public class H2DBManage extends DefaultDBManage implements DBManage { @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportSchema(connection, schemaName, sqlBuilder, containData); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportSchema(connection, schemaName, asyncContext); } - private void exportSchema(Connection connection, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportSchema(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SCRIPT NODATA NOPASSWORDS NOSETTINGS DROP SCHEMA %s;", schemaName); - if (containData) { + if (asyncContext.isContainsData()) { sql = sql.replace("NODATA", ""); } try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String script = resultSet.getString("SCRIPT"); if (!(script.startsWith("CREATE USER")||script.startsWith("--"))) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(script); sqlBuilder.append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveCommandExecutor.java b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveCommandExecutor.java index 4d152b9fc..db7a23057 100644 --- a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveCommandExecutor.java +++ b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveCommandExecutor.java @@ -1,6 +1,5 @@ package ai.chat2db.plugin.hive; -import ai.chat2db.spi.ValueHandler; import ai.chat2db.spi.model.Command; import ai.chat2db.spi.model.ExecuteResult; import ai.chat2db.spi.model.Header; @@ -51,9 +50,9 @@ public ExecuteResult executeUpdate(String sql, Connection connection, int n) thr */ @Override public ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, - Integer count, ValueHandler valueHandler) + Integer count) throws SQLException { - return super.execute(sql, connection, limitRowSize, offset, count, valueHandler); + return super.execute(sql, connection, limitRowSize, offset, count); } public static String formatTableName(String tableName) { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index 4e6dff6be..b59acb1e6 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.sql.SQLExecutor; import org.springframework.util.StringUtils; @@ -11,113 +12,121 @@ public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, databaseName, sqlBuilder, containData); - exportViews(connection, databaseName, sqlBuilder); - exportProcedures(connection, sqlBuilder); - exportTriggers(connection, sqlBuilder); - exportFunctions(connection, databaseName, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, databaseName, asyncContext); + exportViews(connection, databaseName, asyncContext); + exportProcedures(connection, asyncContext); + exportTriggers(connection, asyncContext); + exportFunctions(connection, databaseName, asyncContext); } - private void exportFunctions(Connection connection, String databaseName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunctions(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getFunctions(databaseName, null, null)) { while (resultSet.next()) { - exportFunction(connection, resultSet.getString("FUNCTION_NAME"), sqlBuilder); + exportFunction(connection, resultSet.getString("FUNCTION_NAME"), asyncContext); } } } - private void exportFunction(Connection connection, String functionName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunction(Connection connection, String functionName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SHOW CREATE FUNCTION %s;", functionName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n") .append(resultSet.getString("Create Function")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTables(Connection connection, String databaseName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTables(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection, resultSet.getString("TABLE_NAME"), sqlBuilder, containData); + exportTable(connection, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String tableName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create table %s ", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") .append(resultSet.getString("Create Table")).append(";").append("\n"); - if (containData) { - exportTableData(connection, null,tableName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection, null,tableName, asyncContext); } } } } - private void exportViews(Connection connection, String databaseName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"VIEW"})) { while (resultSet.next()) { - exportView(connection, resultSet.getString("TABLE_NAME"), sqlBuilder); + exportView(connection, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportView(Connection connection, String viewName, StringBuilder sqlBuilder) throws SQLException { + private void exportView(Connection connection, String viewName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create view %s ", viewName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP VIEW IF EXISTS ").append(format(viewName)).append(";").append("\n") .append(resultSet.getString("Create View")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportProcedures(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedures(Connection connection, AsyncContext asyncContext) throws SQLException { String sql = "SHOW PROCEDURE STATUS WHERE Db = DATABASE()"; try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { - exportProcedure(connection, resultSet.getString("Name"), sqlBuilder); + exportProcedure(connection, resultSet.getString("Name"), asyncContext); } } } - private void exportProcedure(Connection connection, String procedureName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedure(Connection connection, String procedureName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create procedure %s ", procedureName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP PROCEDURE IF EXISTS ").append(format(procedureName)).append(";").append("\n") .append("delimiter ;;").append("\n").append(resultSet.getString("Create Procedure")).append(";;") .append("\n").append("delimiter ;").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, AsyncContext asyncContext) throws SQLException { String sql = "SHOW TRIGGERS"; try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String triggerName = resultSet.getString("Trigger"); - exportTrigger(connection, triggerName, sqlBuilder); + exportTrigger(connection, triggerName, asyncContext); } } } - private void exportTrigger(Connection connection, String triggerName, StringBuilder sqlBuilder) throws SQLException { + private void exportTrigger(Connection connection, String triggerName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create trigger %s ", triggerName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TRIGGER IF EXISTS ").append(format(triggerName)).append(";").append("\n") .append("delimiter ;;").append("\n").append(resultSet.getString("SQL Original Statement")).append(";;") .append("\n").append("delimiter ;").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java index b241197db..3bec09617 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlMetaData.java @@ -4,9 +4,8 @@ import ai.chat2db.plugin.mysql.type.*; import ai.chat2db.plugin.mysql.value.MysqlValueProcessor; import ai.chat2db.spi.MetaData; -import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.SqlBuilder; -import ai.chat2db.spi.ValueHandler; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.jdbc.DefaultMetaService; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; @@ -348,10 +347,10 @@ public String getMetaDataName(String... names) { return Arrays.stream(names).filter(name -> StringUtils.isNotBlank(name)).map(name -> "`" + name + "`").collect(Collectors.joining(".")); } - @Override - public ValueHandler getValueHandler() { - return new MysqlValueHandler(); - } +// @Override +// public ValueHandler getValueHandler() { +// return new MysqlValueHandler(); +// } @Override public ValueProcessor getValueProcessor() { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlValueHandler.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlValueHandler.java index ed70d6f43..9f4fa11d3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlValueHandler.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlValueHandler.java @@ -1,45 +1,45 @@ -package ai.chat2db.plugin.mysql; - -import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; -import ai.chat2db.plugin.mysql.value.GeometryValueHandler; -import ai.chat2db.spi.ValueHandler; -import ai.chat2db.spi.jdbc.DefaultValueHandler; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Map; - -public class MysqlValueHandler extends DefaultValueHandler { - - private static final Map VALUE_HANDLER_MAP = Map.of( - MysqlColumnTypeEnum.GEOMETRY.name(), new GeometryValueHandler() - ); - - @Override - public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { - try { - Object obj = rs.getObject(index); - if (obj == null) { - return null; - } - String columnTypeName = rs.getMetaData().getColumnTypeName(index); - if (MysqlColumnTypeEnum.GEOMETRY.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.POINT.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.LINESTRING.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.POLYGON.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTIPOINT.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTILINESTRING.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.MULTIPOLYGON.name().equalsIgnoreCase(columnTypeName) - || MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name().equalsIgnoreCase(columnTypeName) - ) { - ValueHandler handler = VALUE_HANDLER_MAP.get(MysqlColumnTypeEnum.GEOMETRY.name()); - return handler.getString(rs, index, limitSize); - } else { - return super.getString(rs, index, limitSize); - } - }catch (Exception e){ - return rs.getString(index); - } - } - -} +//package ai.chat2db.plugin.mysql; +// +//import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; +//import ai.chat2db.plugin.mysql.value.GeometryValueHandler; +//import ai.chat2db.spi.ValueHandler; +//import ai.chat2db.spi.jdbc.DefaultValueHandler; +// +//import java.sql.ResultSet; +//import java.sql.SQLException; +//import java.util.Map; +// +//public class MysqlValueHandler extends DefaultValueHandler { +// +// private static final Map VALUE_HANDLER_MAP = Map.of( +// MysqlColumnTypeEnum.GEOMETRY.name(), new GeometryValueHandler() +// ); +// +// @Override +// public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { +// try { +// Object obj = rs.getObject(index); +// if (obj == null) { +// return null; +// } +// String columnTypeName = rs.getMetaData().getColumnTypeName(index); +// if (MysqlColumnTypeEnum.GEOMETRY.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.POINT.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.LINESTRING.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.POLYGON.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.MULTIPOINT.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.MULTILINESTRING.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.MULTIPOLYGON.name().equalsIgnoreCase(columnTypeName) +// || MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name().equalsIgnoreCase(columnTypeName) +// ) { +// ValueHandler handler = VALUE_HANDLER_MAP.get(MysqlColumnTypeEnum.GEOMETRY.name()); +// return handler.getString(rs, index, limitSize); +// } else { +// return super.getString(rs, index, limitSize); +// } +// }catch (Exception e){ +// return rs.getString(index); +// } +// } +// +//} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/GeometryValueHandler.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/GeometryValueHandler.java index d8a083cbc..8283a4ee2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/GeometryValueHandler.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/GeometryValueHandler.java @@ -1,69 +1,69 @@ -package ai.chat2db.plugin.mysql.value; - -import ai.chat2db.spi.ValueHandler; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.io.WKBReader; - -import java.io.InputStream; -import java.io.ByteArrayOutputStream; -import java.sql.ResultSet; -import java.sql.SQLException; - -public class GeometryValueHandler implements ValueHandler { - @Override - public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { - try { - InputStream inputStream = rs.getBinaryStream(index); - Geometry dbGeometry = null; - if (inputStream != null) { - - //convert the stream to a byte[] array - //so it can be passed to the WKBReader - byte[] buffer = new byte[255]; - - int bytesRead = 0; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - while ((bytesRead = inputStream.read(buffer)) != -1) { - baos.write(buffer, 0, bytesRead); - } - - byte[] geometryAsBytes = baos.toByteArray(); - - if (geometryAsBytes.length < 5) { - throw new Exception("Invalid geometry inputStream - less than five bytes"); - } - - //first four bytes of the geometry are the SRID, - //followed by the actual WKB. Determine the SRID - //here - byte[] sridBytes = new byte[4]; - System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); - boolean bigEndian = (geometryAsBytes[4] == 0x00); - - int srid = 0; - if (bigEndian) { - for (int i = 0; i < sridBytes.length; i++) { - srid = (srid << 8) + (sridBytes[i] & 0xff); - } - } else { - for (int i = 0; i < sridBytes.length; i++) { - srid += (sridBytes[i] & 0xff) << (8 * i); - } - } - - //use the JTS WKBReader for WKB parsing - WKBReader wkbReader = new WKBReader(); - - //copy the byte array, removing the first four - //SRID bytes - byte[] wkb = new byte[geometryAsBytes.length - 4]; - System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); - dbGeometry = wkbReader.read(wkb); - dbGeometry.setSRID(srid); - } - return dbGeometry.toString(); - } catch (Exception e) { - return rs.getString(index); - } - } -} +//package ai.chat2db.plugin.mysql.value; +// +//import ai.chat2db.spi.ValueHandler; +//import org.locationtech.jts.geom.Geometry; +//import org.locationtech.jts.io.WKBReader; +// +//import java.io.InputStream; +//import java.io.ByteArrayOutputStream; +//import java.sql.ResultSet; +//import java.sql.SQLException; +// +//public class GeometryValueHandler implements ValueHandler { +// @Override +// public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { +// try { +// InputStream inputStream = rs.getBinaryStream(index); +// Geometry dbGeometry = null; +// if (inputStream != null) { +// +// //convert the stream to a byte[] array +// //so it can be passed to the WKBReader +// byte[] buffer = new byte[255]; +// +// int bytesRead = 0; +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// while ((bytesRead = inputStream.read(buffer)) != -1) { +// baos.write(buffer, 0, bytesRead); +// } +// +// byte[] geometryAsBytes = baos.toByteArray(); +// +// if (geometryAsBytes.length < 5) { +// throw new Exception("Invalid geometry inputStream - less than five bytes"); +// } +// +// //first four bytes of the geometry are the SRID, +// //followed by the actual WKB. Determine the SRID +// //here +// byte[] sridBytes = new byte[4]; +// System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); +// boolean bigEndian = (geometryAsBytes[4] == 0x00); +// +// int srid = 0; +// if (bigEndian) { +// for (int i = 0; i < sridBytes.length; i++) { +// srid = (srid << 8) + (sridBytes[i] & 0xff); +// } +// } else { +// for (int i = 0; i < sridBytes.length; i++) { +// srid += (sridBytes[i] & 0xff) << (8 * i); +// } +// } +// +// //use the JTS WKBReader for WKB parsing +// WKBReader wkbReader = new WKBReader(); +// +// //copy the byte array, removing the first four +// //SRID bytes +// byte[] wkb = new byte[geometryAsBytes.length - 4]; +// System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); +// dbGeometry = wkbReader.read(wkb); +// dbGeometry.setSRID(srid); +// } +// return dbGeometry.toString(); +// } catch (Exception e) { +// return rs.getString(index); +// } +// } +//} diff --git a/chat2db-server/chat2db-plugins/chat2db-oceanbase/pom.xml b/chat2db-server/chat2db-plugins/chat2db-oceanbase/pom.xml index f27394b1a..056ced1b4 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oceanbase/pom.xml +++ b/chat2db-server/chat2db-plugins/chat2db-oceanbase/pom.xml @@ -15,6 +15,11 @@ ai.chat2db chat2db-spi + + ai.chat2db + chat2db-mysql + 2.0.0-SNAPSHOT + chat2db-oceanbase diff --git a/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseDBManage.java index 0753639c5..ba3a4c89f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseDBManage.java @@ -1,8 +1,9 @@ package ai.chat2db.plugin.oceanbase; +import ai.chat2db.plugin.mysql.MysqlDBManage; import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; -public class OceanBaseDBManage extends DefaultDBManage implements DBManage { +public class OceanBaseDBManage extends MysqlDBManage implements DBManage { } diff --git a/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseMetaData.java index 2c1020fa8..bdb0e7098 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oceanbase/src/main/java/ai/chat2db/plugin/oceanbase/OceanBaseMetaData.java @@ -1,9 +1,10 @@ package ai.chat2db.plugin.oceanbase; +import ai.chat2db.plugin.mysql.MysqlMetaData; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.jdbc.DefaultMetaService; -public class OceanBaseMetaData extends DefaultMetaService implements MetaData { +public class OceanBaseMetaData extends MysqlMetaData implements MetaData { } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index 4e401d510..b6f256c49 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; @@ -30,72 +31,73 @@ public class OracleDBManage extends DefaultDBManage implements DBManage { private static String FUNCTION_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('FUNCTION', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; @Override - public String exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTableData(connection, tableName, sqlBuilder); - return sqlBuilder.toString(); - } - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, schemaName, sqlBuilder, containData); - exportViews(connection, sqlBuilder, schemaName); - exportProcedures(connection, schemaName, sqlBuilder); - exportTriggers(connection, schemaName, sqlBuilder); - exportFunctions(connection, schemaName, sqlBuilder); - return sqlBuilder.toString(); - } - - private void exportTables(Connection connection, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + exportTableData(connection, tableName, asyncContext); + } + public void exportDatabase(Connection connection, String databaseName, String schemaName,AsyncContext asyncContext) throws SQLException { + exportTables(connection, schemaName, asyncContext); + exportViews(connection, asyncContext, schemaName); + exportProcedures(connection, schemaName, asyncContext); + exportTriggers(connection, schemaName, asyncContext); + exportFunctions(connection, schemaName, asyncContext); + } + + private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); - exportTable(connection, schemaName, tableName, sqlBuilder, containData); + exportTable(connection, schemaName, tableName, asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TABLE_DDL_SQL, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE ").append(schemaName).append(".").append(tableName).append(";") .append(resultSet.getString("ddl")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } - exportTableComments(connection, tableName, sqlBuilder); - exportTableColumnsComments(connection, tableName, sqlBuilder); - if (containData) { - exportTableData(connection, tableName, sqlBuilder); + exportTableComments(connection, tableName, asyncContext); + exportTableColumnsComments(connection, tableName, asyncContext); + if (asyncContext.isContainsData()) { + exportTableData(connection, tableName, asyncContext); } } } - private void exportTableComments(Connection connection, String tableName, StringBuilder sqlBuilder) throws SQLException { + private void exportTableComments(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TABLE_COMMENT_SQL, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("table_comment_ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } - sqlBuilder.append("\n"); } - private void exportTableColumnsComments(Connection connection, String tableName, StringBuilder sqlBuilder) throws SQLException { + private void exportTableColumnsComments(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TABLE_COLUMN_COMMENT_SQL, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("column_comment_ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } - sqlBuilder.append("\n"); } - private void exportTableData(Connection connection, String tableName, StringBuilder sqlBuilder) throws SQLException { + private void exportTableData(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT * FROM %s", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); for (int i = 1; i <= columnCount; i++) { String columnValue = resultSet.getString(i); @@ -114,80 +116,89 @@ private void exportTableData(Connection connection, String tableName, StringBuil } sqlBuilder.append(");"); sqlBuilder.append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportViews(Connection connection, StringBuilder sqlBuilder, String schemaName) throws SQLException { + private void exportViews(Connection connection, AsyncContext asyncContext, String schemaName) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"VIEW"})) { while (resultSet.next()) { String viewName = resultSet.getString("TABLE_NAME"); - exportView(connection, sqlBuilder, schemaName, viewName); + exportView(connection, asyncContext, schemaName, viewName); } } } - private void exportView(Connection connection, StringBuilder sqlBuilder, String schemaName, String viewName) throws SQLException { + private void exportView(Connection connection, AsyncContext asyncContext, String schemaName, String viewName) throws SQLException { String sql = String.format(VIEW_DDL_SQL, schemaName, viewName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportProcedures(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format(PROCEDURE_LIST_DDL,schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String procedureName = resultSet.getString("object_name"); - exportProcedure(connection, schemaName, procedureName, sqlBuilder); + exportProcedure(connection, schemaName, procedureName, asyncContext); } } } - private void exportProcedure(Connection connection, String schemaName, String procedureName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedure(Connection connection, String schemaName, String procedureName, AsyncContext asyncContext) throws SQLException { String sql = String.format(PROCEDURE_DDL_SQL, schemaName, procedureName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT TRIGGER_NAME FROM all_triggers where OWNER='%s'", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String triggerName = resultSet.getString("TRIGGER_NAME"); - exportTrigger(connection, schemaName, triggerName, sqlBuilder); + exportTrigger(connection, schemaName, triggerName, asyncContext); } } } - private void exportTrigger(Connection connection, String schemaName, String triggerName, StringBuilder sqlBuilder) throws SQLException { + private void exportTrigger(Connection connection, String schemaName, String triggerName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TRIGGER_DDL_SQL, schemaName, triggerName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportFunctions(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getFunctions(null, schemaName, null)) { while (resultSet.next()) { String functionName = resultSet.getString("FUNCTION_NAME"); - exportFunction(connection, schemaName, functionName, sqlBuilder); + exportFunction(connection, schemaName, functionName, asyncContext); } } } - private void exportFunction(Connection connection, String schemaName, String functionName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunction(Connection connection, String schemaName, String functionName, AsyncContext asyncContext) throws SQLException { String sql = String.format(FUNCTION_DDL_SQL, schemaName, functionName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java index 123e7128f..4c5086565 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; @@ -15,24 +16,24 @@ public class PostgreSQLDBManage extends DefaultDBManage implements DBManage { - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTypes(connection, sqlBuilder); - exportTables(connection, databaseName, schemaName, sqlBuilder, containData); - exportViews(connection, schemaName, sqlBuilder); - exportFunctions(connection, schemaName, sqlBuilder); - exportTriggers(connection, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTypes(connection, asyncContext); + exportTables(connection, databaseName, schemaName, asyncContext); + exportViews(connection, schemaName, asyncContext); + exportFunctions(connection, schemaName, asyncContext); + exportTriggers(connection, asyncContext); } - private void exportTypes(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportTypes(Connection connection, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(ENUM_TYPE_DDL_SQL)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTables(Connection connection, String databaseName, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE","PARTITIONED TABLE"})) { ArrayList tableNames = new ArrayList<>(); @@ -41,58 +42,66 @@ private void exportTables(Connection connection, String databaseName, String sch tableNames.add(tableName); } for (String tableName : tableNames) { - exportTable(connection, schemaName, tableName, sqlBuilder); + exportTable(connection, schemaName, tableName, asyncContext); } - if (containData) { + if (asyncContext.isContainsData()) { for (String tableName : tableNames) { - exportTableData(connection, schemaName, tableName, sqlBuilder); + exportTableData(connection, schemaName, tableName, asyncContext); } } } } - private void exportTable(Connection connection, String schemaName, String tableName, StringBuilder sqlBuilder) throws SQLException { + private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql =String.format( "select pg_get_tabledef('%s','%s',true,'COMMENTS') as ddl;", schemaName,tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("\n").append("DROP TABLE IF EXISTS ").append(tableName).append(";").append("\n") .append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportViews(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT table_name, view_definition FROM information_schema.views WHERE table_schema = '%s'",schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); String viewName = resultSet.getString("table_name"); String viewDefinition = resultSet.getString("view_definition"); sqlBuilder.append("CREATE OR REPLACE VIEW ").append(viewName).append(" AS ").append(viewDefinition).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportFunctions(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT proname, pg_get_functiondef(oid) AS function_definition FROM pg_proc " + "WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); String functionName = resultSet.getString("proname"); String functionDefinition = resultSet.getString("function_definition"); sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(schemaName).append(".").append(functionName).append(";\n"); sqlBuilder.append(functionDefinition).append(";\n\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, AsyncContext asyncContext) throws SQLException { String sql = "SELECT pg_get_triggerdef(oid) AS trigger_definition FROM pg_trigger"; try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("trigger_definition")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java index da9659266..306814445 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import java.sql.Connection; import java.sql.ResultSet; @@ -10,31 +11,31 @@ public class SqliteDBManage extends DefaultDBManage implements DBManage { @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, databaseName, schemaName,sqlBuilder, containData); - exportViews(connection, databaseName, sqlBuilder); - exportTriggers(connection, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, databaseName, schemaName,asyncContext); + exportViews(connection, databaseName, asyncContext); + exportTriggers(connection, asyncContext); } - private void exportTables(Connection connection, String databaseName, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection,schemaName, resultSet.getString("TABLE_NAME"), sqlBuilder, containData); + exportTable(connection,schemaName, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT sql FROM sqlite_master WHERE type='table' AND name='%s'", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") .append(resultSet.getString("sql")).append(";").append("\n"); - if (containData) { - exportTableData(connection,schemaName, tableName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection,schemaName, tableName, asyncContext); } } } @@ -44,39 +45,43 @@ private String format(String tableName) { return "\""+tableName+"\""; } - private void exportViews(Connection connection, String databaseName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"VIEW"})) { while (resultSet.next()) { - exportView(connection, resultSet.getString("TABLE_NAME"), sqlBuilder); + exportView(connection, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportView(Connection connection, String viewName, StringBuilder sqlBuilder) throws SQLException { + private void exportView(Connection connection, String viewName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT * FROM sqlite_master WHERE type = 'view' and name='%s';", viewName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP VIEW IF EXISTS ").append(format(viewName)).append(";").append("\n") .append(resultSet.getString("sql")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, AsyncContext asyncContext) throws SQLException { String sql = "SELECT * FROM sqlite_master WHERE type = 'trigger';"; try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String triggerName = resultSet.getString("name"); - exportTrigger(connection, triggerName, sqlBuilder); + exportTrigger(connection, triggerName, asyncContext); } } } - private void exportTrigger(Connection connection, String triggerName, StringBuilder sqlBuilder) throws SQLException { + private void exportTrigger(Connection connection, String triggerName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT * FROM sqlite_master WHERE type = 'trigger' and name='%s';", triggerName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("sql")).append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerCommandExecutor.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerCommandExecutor.java index f30bf5f02..6ba58c5f1 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerCommandExecutor.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerCommandExecutor.java @@ -1,6 +1,5 @@ package ai.chat2db.plugin.sqlserver; -import ai.chat2db.spi.ValueHandler; import ai.chat2db.spi.model.Command; import ai.chat2db.spi.model.ExecuteResult; import ai.chat2db.spi.sql.SQLExecutor; @@ -45,8 +44,8 @@ public ExecuteResult executeUpdate(String sql, Connection connection, int n) thr * */ public ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, - Integer count, ValueHandler valueHandler) + Integer count) throws SQLException { - return super.execute(removeSpecialGO(sql), connection, limitRowSize, offset, count, valueHandler); + return super.execute(removeSpecialGO(sql), connection, limitRowSize, offset, count); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 8e9bff5d6..906576b0f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -2,6 +2,7 @@ import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.SQLExecutor; import java.sql.*; @@ -42,34 +43,30 @@ public class SqlServerDBManage extends DefaultDBManage implements DBManage { + "WHERE xtype = 'TR' "; @Override - public String exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTableData(connection, tableName, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + exportTableData(connection, tableName, asyncContext); } @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTables(connection, sqlBuilder, schemaName, containData); - exportViews(connection, databaseName, schemaName, sqlBuilder); - exportFunctions(connection, schemaName, sqlBuilder); - exportProcedures(connection, schemaName, sqlBuilder); - exportTriggers(connection, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, schemaName, asyncContext); + exportViews(connection, databaseName, schemaName, asyncContext); + exportFunctions(connection, schemaName, asyncContext); + exportProcedures(connection, schemaName, asyncContext); + exportTriggers(connection, asyncContext); } - private void exportTables(Connection connection, StringBuilder sqlBuilder, String schemaName, boolean containData) throws SQLException { + private void exportTables(Connection connection, String schemaName,AsyncContext asyncContext) throws SQLException { String sql ="SELECT name FROM SysObjects Where XType='U'"; try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String tableName = resultSet.getString("name"); - exportTable(connection, tableName, schemaName, sqlBuilder, containData); + exportTable(connection, tableName, schemaName, asyncContext); } } } - private void exportTable(Connection connection, String tableName, String schemaName, StringBuilder sqlBuilder, boolean containData) throws SQLException { + private void exportTable(Connection connection, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, tableDDLFunction.replace("tableSchema", schemaName), resultSet -> null); @@ -79,23 +76,26 @@ private void exportTable(Connection connection, String tableName, String schemaN String sql = String.format("SELECT %s.ufn_GetCreateTableScript('%s', '%s') as ddl",schemaName,schemaName,tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TABLE IF EXISTS ").append(tableName).append(";").append("\n") .append(resultSet.getString("ddl")).append("\n"); - if (containData) { - exportTableData(connection, tableName, sqlBuilder); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection, tableName, asyncContext); } else { - sqlBuilder.append("go").append("\n"); + asyncContext.write("go \n"); } } } } - private void exportTableData(Connection connection, String tableName, StringBuilder sqlBuilder) throws SQLException { + private void exportTableData(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select * from %s", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { ResultSetMetaData metaData = resultSet.getMetaData(); while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); for (int i = 1; i <= metaData.getColumnCount(); i++) { String value = resultSet.getString(i); @@ -109,75 +109,84 @@ private void exportTableData(Connection connection, String tableName, StringBuil } } sqlBuilder.append(");\n"); + asyncContext.write(sqlBuilder.toString()); } - sqlBuilder.append("\n"); } - sqlBuilder.append("go").append("\n"); + asyncContext.write("go \n"); } - private void exportViews(Connection connection, String databaseName, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportViews(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT TABLE_NAME, VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS " + "WHERE TABLE_SCHEMA = '%s' AND TABLE_CATALOG = '%s'; ", schemaName, databaseName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP VIEW IF EXISTS ").append(resultSet.getString("TABLE_NAME")).append(";\n").append("go").append("\n") .append(resultSet.getString("VIEW_DEFINITION")).append(";").append("\n") .append("go").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportFunctions(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT name FROM sys.objects WHERE type = 'FN' and SCHEMA_ID = SCHEMA_ID('%s')", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String functionName = resultSet.getString("name"); - exportFunction(connection, functionName, schemaName, sqlBuilder); + exportFunction(connection, functionName, schemaName, asyncContext); } } } - private void exportFunction(Connection connection, String functionName, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportFunction(Connection connection, String functionName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT OBJECT_DEFINITION(OBJECT_ID('%s.%s')) as ddl", schemaName, functionName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("ddl") .replace("CREATE FUNCTION", "CREATE OR ALTER FUNCTION")) .append("\n").append("go").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportProcedures(Connection connection, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT name FROM sys.procedures WHERE SCHEMA_ID = SCHEMA_ID('%s')", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String procedureName = resultSet.getString("name"); - exportProcedure(connection, procedureName, schemaName, sqlBuilder); + + exportProcedure(connection, procedureName, schemaName, asyncContext); } } } - private void exportProcedure(Connection connection, String procedureName, String schemaName, StringBuilder sqlBuilder) throws SQLException { + private void exportProcedure(Connection connection, String procedureName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT definition FROM sys.sql_modules WHERE object_id = (OBJECT_ID('%s.%s'));", schemaName, procedureName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("definition") .replace("CREATE PROCEDURE", "CREATE OR ALTER PROCEDURE")) .append("\n").append("go").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } - private void exportTriggers(Connection connection, StringBuilder sqlBuilder) throws SQLException { + private void exportTriggers(Connection connection, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(TRIGGER_SQL_LIST)) { while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append(resultSet.getString("triggerDefinition") .replace("CREATE TRIGGER", "CREATE OR ALTER TRIGGER")) .append("\n").append("go").append("\n"); + asyncContext.write(sqlBuilder.toString()); } } } diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java index c819924c7..4ebd494f5 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.function.Consumer; import ai.chat2db.server.domain.api.param.datasource.DatabaseCreateParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; @@ -18,10 +19,7 @@ import ai.chat2db.server.tools.base.wrapper.result.DataResult; import ai.chat2db.server.tools.base.wrapper.result.ListResult; import ai.chat2db.spi.MetaData; -import ai.chat2db.spi.model.Database; -import ai.chat2db.spi.model.MetaSchema; -import ai.chat2db.spi.model.Schema; -import ai.chat2db.spi.model.Sql; +import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.Chat2DBContext; import cn.hutool.core.thread.ThreadUtil; import lombok.extern.slf4j.Slf4j; @@ -179,10 +177,14 @@ public ActionResult modifySchema(SchemaOperationParam param) { @Override public String exportDatabase(DatabaseExportParam param) throws SQLException { - return Chat2DBContext.getDBManage().exportDatabase(Chat2DBContext.getConnection(), + AsyncContext asyncContext = new AsyncContext(); + asyncContext.setContainsData(param.getContainData()); + asyncContext.setConsumer(aLong -> log.info("exportDatabase success")); + Chat2DBContext.getDBManage().exportDatabase(Chat2DBContext.getConnection(), param.getDatabaseName(), - param.getSchemaName(), - param.getContainData()); + param.getSchemaName(), asyncContext); + + return "exportDatabase success"; } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java index 999c8dcc7..2b45ef966 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java @@ -13,7 +13,6 @@ import ai.chat2db.server.tools.common.util.EasyCollectionUtils; import ai.chat2db.spi.CommandExecutor; import ai.chat2db.spi.SqlBuilder; -import ai.chat2db.spi.ValueHandler; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; @@ -135,9 +134,7 @@ public DataResult count(DlCountParam param) { sql = PagerUtils.count(sql, dbType); ExecuteResult executeResult; try { - ValueHandler valueHandler = Chat2DBContext.getMetaData().getValueHandler(); - executeResult = Chat2DBContext.getMetaData().getCommandExecutor().execute(sql, Chat2DBContext.getConnection(), true, null, null, - valueHandler); + executeResult = Chat2DBContext.getMetaData().getCommandExecutor().execute(sql, Chat2DBContext.getConnection(), true, null, null); } catch (SQLException e) { log.warn("Execute sql: {} exception", sql, e); executeResult = ExecuteResult.builder() diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java index 73acdf4a9..cbb263180 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java @@ -31,7 +31,6 @@ import ai.chat2db.server.tools.common.util.EasyEnumUtils; import ai.chat2db.server.web.api.aspect.ConnectionInfoAspect; import ai.chat2db.server.web.api.controller.rdb.request.DataExportRequest; -import ai.chat2db.spi.jdbc.DefaultValueHandler; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.util.JdbcUtils; @@ -135,7 +134,7 @@ private void doExportCsv(String sql, HttpServletResponse response, String fileNa List> writeDataList = Lists.newArrayList(); writeDataList.add(dataList); excelWrapper.getExcelWriter().write(writeDataList, excelWrapper.getWriteSheet()); - }, false, new DefaultValueHandler()); + }, false); } finally { if (excelWrapper.getExcelWriter() != null) { excelWrapper.getExcelWriter().finish(); @@ -166,7 +165,7 @@ private void doExportInsert(String sql, HttpServletResponse response, String fil sqlInsertStatement.setValues(valuesClause); printWriter.println(SQLUtils.toSQLString(sqlInsertStatement, dbType, INSERT_FORMAT_OPTION) + ";"); - }, false, new DefaultValueHandler()); + }, false); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java index 2623c580e..d52e8cecf 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java @@ -19,7 +19,6 @@ import ai.chat2db.server.web.api.controller.rdb.factory.ExportServiceFactory; import ai.chat2db.server.web.api.controller.rdb.request.DataExportRequest; import ai.chat2db.server.web.api.controller.rdb.vo.TableVO; -import ai.chat2db.spi.jdbc.DefaultValueHandler; import ai.chat2db.spi.model.Table; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; @@ -260,7 +259,7 @@ private void doExportCsv(String sql, File file) { List> writeDataList = Lists.newArrayList(); writeDataList.add(dataList); excelWrapper.getExcelWriter().write(writeDataList, excelWrapper.getWriteSheet()); - }, false, new DefaultValueHandler()); + }, false); } finally { if (excelWrapper.getExcelWriter() != null) { excelWrapper.getExcelWriter().finish(); @@ -288,7 +287,7 @@ private void doExportInsert(String sql, File file, DbType dbType, sqlInsertStatement.setValues(valuesClause); printWriter.println(SQLUtils.toSQLString(sqlInsertStatement, dbType, INSERT_FORMAT_OPTION) + ";"); - }, false, new DefaultValueHandler()); + }, false); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/CommandExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/CommandExecutor.java index 35b878956..1787949a1 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/CommandExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/CommandExecutor.java @@ -38,6 +38,6 @@ public interface CommandExecutor { * */ ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, - Integer count, ValueHandler valueHandler) + Integer count) throws SQLException; } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java index eae7e20af..7e811be2e 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java @@ -1,5 +1,6 @@ package ai.chat2db.spi; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.sql.ConnectInfo; import jakarta.validation.constraints.NotEmpty; @@ -121,7 +122,22 @@ void dropProcedure(Connection connection, @NotEmpty String databaseName, String */ void updateProcedure(Connection connection, @NotEmpty String databaseName, String schemaName, @NotNull Procedure procedure) throws SQLException; - String exportDatabase(Connection connection, String databaseName, String schemaName,boolean containData) throws SQLException; + /** + * Export database + * + * @param databaseName + * @param schemaName + * @return + */ + void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException; - String exportDatabaseData(Connection connection, String databaseName, String schemaName,String tableName) throws SQLException; + /** + * Export database data + * + * @param databaseName + * @param schemaName + * @param tableName + * @return + */ + void exportDatabaseData(Connection connection, String databaseName, String schemaName,String tableName,AsyncContext asyncContext) throws SQLException; } \ No newline at end of file diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java index eae653013..073092a61 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/MetaData.java @@ -241,8 +241,6 @@ List indexes(Connection connection, @NotEmpty String databaseName, S * Get column builder. * */ - ValueHandler getValueHandler(); - ValueProcessor getValueProcessor(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueHandler.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueHandler.java index 80fdc342d..3e1b32093 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueHandler.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueHandler.java @@ -1,17 +1,17 @@ -package ai.chat2db.spi; - -import java.sql.ResultSet; -import java.sql.SQLException; - -public interface ValueHandler { - - /** - * Process column values in the result set - * @param rs - * @param index - * @param limitSize - * @return - * @throws SQLException - */ - String getString(ResultSet rs, int index, boolean limitSize)throws SQLException; -} +//package ai.chat2db.spi; +// +//import java.sql.ResultSet; +//import java.sql.SQLException; +// +//public interface ValueHandler { +// +// /** +// * Process column values in the result set +// * @param rs +// * @param index +// * @param limitSize +// * @return +// * @throws SQLException +// */ +// String getString(ResultSet rs, int index, boolean limitSize)throws SQLException; +//} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 8cec4db76..10369b460 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -9,6 +9,7 @@ import ai.chat2db.server.tools.base.excption.BusinessException; import ai.chat2db.server.tools.common.exception.ConnectionException; import ai.chat2db.spi.DBManage; +import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.model.SSHInfo; import ai.chat2db.spi.sql.ConnectInfo; @@ -146,13 +147,11 @@ public void updateProcedure(Connection connection, String databaseName, String s } @Override - public String exportDatabase(Connection connection, String databaseName, String schemaName, boolean containData) throws SQLException { - return null; + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + } - public String exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - StringBuilder sqlBuilder = new StringBuilder(); - exportTableData(connection, schemaName,tableName, sqlBuilder); - return sqlBuilder.toString(); + public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + exportTableData(connection, schemaName,tableName, asyncContext); } @Override @@ -161,7 +160,7 @@ public void dropTable(Connection connection, String databaseName, String schemaN SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } - public void exportTableData(Connection connection,String schemaName, String tableName, StringBuilder sqlBuilder) throws SQLException { + public void exportTableData(Connection connection,String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql; if (Objects.isNull(schemaName)) { sql = String.format("select * from %s", tableName); @@ -171,6 +170,7 @@ public void exportTableData(Connection connection,String schemaName, String tabl try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { ResultSetMetaData metaData = resultSet.getMetaData(); while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); for (int i = 1; i <= metaData.getColumnCount(); i++) { String value = resultSet.getString(i); @@ -184,8 +184,9 @@ public void exportTableData(Connection connection,String schemaName, String tabl } } sqlBuilder.append(");\n"); + asyncContext.write(sqlBuilder.toString()); } - sqlBuilder.append("\n"); + asyncContext.write("\n"); } } } \ No newline at end of file diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java index 1c1c836c6..07f2ece5b 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java @@ -159,12 +159,6 @@ public String getMetaDataName(String... names) { return Arrays.stream(names).filter(name -> StringUtils.isNotBlank(name)).collect(Collectors.joining(".")); } - @Override - public ValueHandler getValueHandler() { - return new DefaultValueHandler(); - } - - @Override public ValueProcessor getValueProcessor() { return new DefaultValueProcessor(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java index 316d16406..46e2f718c 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueHandler.java @@ -1,112 +1,112 @@ -package ai.chat2db.spi.jdbc; - -import ai.chat2db.server.tools.common.util.I18nUtils; -import ai.chat2db.spi.ValueHandler; -import cn.hutool.core.io.unit.DataSizeUtil; -import lombok.extern.slf4j.Slf4j; - -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.sql.*; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -@Slf4j -public class DefaultValueHandler implements ValueHandler { - - private static final long MAX_RESULT_SIZE = 10* 1024 * 1024; - - @Override - public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { - try { - Object obj = rs.getObject(index); - if (obj == null) { - return null; - } - if (obj instanceof BigDecimal bigDecimal) { - return bigDecimal.toPlainString(); - } else if (obj instanceof Double d) { - return BigDecimal.valueOf(d).toPlainString(); - } else if (obj instanceof Float f) { - return BigDecimal.valueOf(f).toPlainString(); - } else if (obj instanceof Clob) { - return largeString(rs, index, limitSize); - } else if (obj instanceof byte[]) { - return largeString(rs, index, limitSize); - } else if (obj instanceof Blob blob) { - return largeStringBlob(blob, limitSize); - } else if (obj instanceof Timestamp || obj instanceof LocalDateTime) { - return largeTime(obj); - } else if (obj instanceof SQLXML){ - return ((SQLXML) obj).getString(); - } else { - return obj.toString(); - } - } catch (Exception e) { - log.warn("Failed to parse number:{},", index, e); - return rs.getString(index); - } - } - - private String largeStringBlob(Blob blob, boolean limitSize) throws SQLException { - if (blob == null) { - return null; - } - int length = Math.toIntExact(blob.length()); - if (limitSize && length > MAX_RESULT_SIZE) { - length = Math.toIntExact(MAX_RESULT_SIZE); - } - byte[] data = blob.getBytes(1, length); - String result = new String(data, StandardCharsets.UTF_8); - - if (length > MAX_RESULT_SIZE) { - return "[ " + DataSizeUtil.format(MAX_RESULT_SIZE) + " of " + DataSizeUtil.format(length) - + " ," - + I18nUtils.getMessage("execute.exportCsv") + " ] " + result; - } - return result; - } - - private String largeTime(Object obj) throws SQLException { - Object timeField = obj; // Assuming a time field of type Object - - LocalDateTime localDateTime; - - if (obj instanceof Timestamp) { - // Convert a time field of type Object to a LocalDateTime object - localDateTime = ((Timestamp) timeField).toLocalDateTime(); - } else if(obj instanceof LocalDateTime){ - localDateTime = (LocalDateTime) timeField; - } else { - try { - localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")); - }catch (Exception e){ - localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm")); - } - } - // Create a DateTimeFormatter instance and specify the output date and time format - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - // Format date time - String formattedDateTime = dtf.format(localDateTime); - return formattedDateTime; - } - - private static String largeString(ResultSet rs, int index, boolean limitSize) throws SQLException { - String result = rs.getString(index); - if (result == null) { - return null; - - } - if (!limitSize) { - return result; - } - - if (result.length() > MAX_RESULT_SIZE) { - return "[ " + DataSizeUtil.format(MAX_RESULT_SIZE) + " of " + DataSizeUtil.format(result.length()) + " ," - + I18nUtils.getMessage("execute.exportCsv") + " ] " + result.substring(0, - Math.toIntExact(MAX_RESULT_SIZE)); - } - return result; - } -} +//package ai.chat2db.spi.jdbc; +// +//import ai.chat2db.server.tools.common.util.I18nUtils; +//import ai.chat2db.spi.ValueHandler; +//import cn.hutool.core.io.unit.DataSizeUtil; +//import lombok.extern.slf4j.Slf4j; +// +//import java.math.BigDecimal; +//import java.nio.charset.StandardCharsets; +//import java.sql.*; +//import java.time.LocalDateTime; +//import java.time.format.DateTimeFormatter; +// +//@Slf4j +//public class DefaultValueHandler implements ValueHandler { +// +// private static final long MAX_RESULT_SIZE = 10* 1024 * 1024; +// +// @Override +// public String getString(ResultSet rs, int index, boolean limitSize) throws SQLException { +// try { +// Object obj = rs.getObject(index); +// if (obj == null) { +// return null; +// } +// if (obj instanceof BigDecimal bigDecimal) { +// return bigDecimal.toPlainString(); +// } else if (obj instanceof Double d) { +// return BigDecimal.valueOf(d).toPlainString(); +// } else if (obj instanceof Float f) { +// return BigDecimal.valueOf(f).toPlainString(); +// } else if (obj instanceof Clob) { +// return largeString(rs, index, limitSize); +// } else if (obj instanceof byte[]) { +// return largeString(rs, index, limitSize); +// } else if (obj instanceof Blob blob) { +// return largeStringBlob(blob, limitSize); +// } else if (obj instanceof Timestamp || obj instanceof LocalDateTime) { +// return largeTime(obj); +// } else if (obj instanceof SQLXML){ +// return ((SQLXML) obj).getString(); +// } else { +// return obj.toString(); +// } +// } catch (Exception e) { +// log.warn("Failed to parse number:{},", index, e); +// return rs.getString(index); +// } +// } +// +// private String largeStringBlob(Blob blob, boolean limitSize) throws SQLException { +// if (blob == null) { +// return null; +// } +// int length = Math.toIntExact(blob.length()); +// if (limitSize && length > MAX_RESULT_SIZE) { +// length = Math.toIntExact(MAX_RESULT_SIZE); +// } +// byte[] data = blob.getBytes(1, length); +// String result = new String(data, StandardCharsets.UTF_8); +// +// if (length > MAX_RESULT_SIZE) { +// return "[ " + DataSizeUtil.format(MAX_RESULT_SIZE) + " of " + DataSizeUtil.format(length) +// + " ," +// + I18nUtils.getMessage("execute.exportCsv") + " ] " + result; +// } +// return result; +// } +// +// private String largeTime(Object obj) throws SQLException { +// Object timeField = obj; // Assuming a time field of type Object +// +// LocalDateTime localDateTime; +// +// if (obj instanceof Timestamp) { +// // Convert a time field of type Object to a LocalDateTime object +// localDateTime = ((Timestamp) timeField).toLocalDateTime(); +// } else if(obj instanceof LocalDateTime){ +// localDateTime = (LocalDateTime) timeField; +// } else { +// try { +// localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")); +// }catch (Exception e){ +// localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm")); +// } +// } +// // Create a DateTimeFormatter instance and specify the output date and time format +// DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); +// +// // Format date time +// String formattedDateTime = dtf.format(localDateTime); +// return formattedDateTime; +// } +// +// private static String largeString(ResultSet rs, int index, boolean limitSize) throws SQLException { +// String result = rs.getString(index); +// if (result == null) { +// return null; +// +// } +// if (!limitSize) { +// return result; +// } +// +// if (result.length() > MAX_RESULT_SIZE) { +// return "[ " + DataSizeUtil.format(MAX_RESULT_SIZE) + " of " + DataSizeUtil.format(result.length()) + " ," +// + I18nUtils.getMessage("execute.exportCsv") + " ] " + result.substring(0, +// Math.toIntExact(MAX_RESULT_SIZE)); +// } +// return result; +// } +//} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java new file mode 100644 index 000000000..5bacc7898 --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -0,0 +1,34 @@ +package ai.chat2db.spi.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.PrintWriter; +import java.util.function.Consumer; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AsyncContext { + + private PrintWriter writer; + + private boolean containsData; + + private Consumer consumer; + + public void addProgress(Long progress) { + if (consumer != null) { + consumer.accept(progress); + } + } + + public void write(String message) { + if (writer != null) { + writer.write(message); + } + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 235f1328a..3adc08e65 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -17,11 +17,10 @@ import ai.chat2db.server.tools.common.util.I18nUtils; import ai.chat2db.spi.CommandExecutor; import ai.chat2db.spi.MetaData; -import ai.chat2db.spi.ValueHandler; import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DataTypeEnum; import ai.chat2db.spi.enums.SqlTypeEnum; -import ai.chat2db.spi.jdbc.DefaultValueHandler; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.*; import ai.chat2db.spi.util.JdbcUtils; import ai.chat2db.spi.util.ResultSetUtils; @@ -33,7 +32,6 @@ import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @@ -93,15 +91,16 @@ public void execute(Connection connection, String sql, ResultSetConsumer consume } } - public void execute(Connection connection, String sql, Consumer> headerConsumer, - Consumer> rowConsumer, ValueHandler valueHandler) { - execute(connection, sql, headerConsumer, rowConsumer, true, valueHandler); - } +// public void execute(Connection connection, String sql, Consumer> headerConsumer, +// Consumer> rowConsumer, ValueHandler valueHandler) { +// execute(connection, sql, headerConsumer, rowConsumer, true, valueHandler); +// } public void execute(Connection connection, String sql, Consumer> headerConsumer, - Consumer> rowConsumer, boolean limitSize, ValueHandler valueHandler) { + Consumer> rowConsumer, boolean limitSize) { Assert.notNull(sql, "SQL must not be null"); log.info("execute:{}", sql); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); try (Statement stmt = connection.createStatement();) { boolean query = stmt.execute(sql); // Represents the query @@ -114,20 +113,13 @@ public void execute(Connection connection, String sql, Consumer> he int col = resultSetMetaData.getColumnCount(); // Get header information - List
headerList = Lists.newArrayListWithExpectedSize(col); - for (int i = 1; i <= col; i++) { - headerList.add(Header.builder() - .dataType(JdbcUtils.resolveDataType( - resultSetMetaData.getColumnTypeName(i), resultSetMetaData.getColumnType(i)).getCode()) - .name(ResultSetUtils.getColumnName(resultSetMetaData, i)) - .build()); - } + List
headerList = generateHeaderList(resultSetMetaData); headerConsumer.accept(headerList); while (rs.next()) { List row = Lists.newArrayListWithExpectedSize(col); for (int i = 1; i <= col; i++) { - row.add(valueHandler.getString(rs, i, limitSize)); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i, limitSize))); } rowConsumer.accept(row); } @@ -140,34 +132,28 @@ public void execute(Connection connection, String sql, Consumer> he } } - /** - * Execute SQL - * - * @param sql - * @return - * @throws SQLException - */ - public ExecuteResult execute(final String sql, Connection connection, ValueHandler valueHandler) - throws SQLException { - return execute(sql, connection, true, null, null, valueHandler); - } +// /** +// * Execute SQL +// * +// * @param sql +// * @return +// * @throws SQLException +// */ +// public ExecuteResult execute(final String sql, Connection connection, ValueHandler valueHandler) +// throws SQLException { +// return execute(sql, connection, true, null, null, valueHandler); +// } @Override public ExecuteResult executeUpdate(String sql, Connection connection, int n) - throws SQLException { + throws SQLException { Assert.notNull(sql, "SQL must not be null"); log.info("execute:{}", sql); - // connection.setAutoCommit(false); ExecuteResult executeResult = ExecuteResult.builder().sql(sql).success(Boolean.TRUE).build(); try (Statement stmt = connection.createStatement()) { int affectedRows = stmt.executeUpdate(sql); if (affectedRows != n) { log.info("Update error {} update affectedRows = {}", sql, affectedRows); -// executeResult.setSuccess(false); -// executeResult.setMessage("Update error " + sql + " update affectedRows = " + affectedRows -// + ", Each SQL statement should update no more than one record. Please use a unique key for " -// + "updates."); - // connection.rollback(); } } return executeResult; @@ -177,7 +163,7 @@ public ExecuteResult executeUpdate(String sql, Connection connection, int n) public List executeSelectTable(Command command) { MetaData metaData = Chat2DBContext.getMetaData(); String tableName = metaData.getMetaDataName(command.getDatabaseName(), command.getSchemaName(), - command.getTableName()); + command.getTableName()); String sql = "select * from " + tableName; command.setScript(sql); return execute(command); @@ -192,161 +178,119 @@ public List executeSelectTable(Command command) { * @param limitRowSize Flag to indicate if row size should be limited. * @param offset The starting point of rows to fetch in the result set. * @param count The number of rows to fetch from the result set. - * @param valueHandler Handles the processing of the result set values. * @return ExecuteResult containing the result of the execution. * @throws SQLException If there is any SQL related error. */ - public ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, - Integer count, ValueHandler valueHandler) - throws SQLException { + public ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, Integer count) + throws SQLException { Assert.notNull(sql, "SQL must not be null"); log.info("execute:{}", sql); - - String type = Chat2DBContext.getConnectInfo().getDbType(); ExecuteResult executeResult = ExecuteResult.builder().sql(sql).success(Boolean.TRUE).build(); try (Statement stmt = connection.createStatement()) { stmt.setFetchSize(EasyToolsConstant.MAX_PAGE_SIZE); - // if (!DataSourceTypeEnum.MONGODB.getCode().equals(type)) { - // stmt.setQueryTimeout(30); - // } if (offset != null && count != null) { stmt.setMaxRows(offset + count); } - TimeInterval timeInterval = new TimeInterval(); boolean query = stmt.execute(sql); executeResult.setDescription(I18nUtils.getMessage("sqlResult.success")); // Represents the query if (query) { - ResultSet rs = null; - try { - rs = stmt.getResultSet(); - // Get how many columns - ResultSetMetaData resultSetMetaData = rs.getMetaData(); - int col = resultSetMetaData.getColumnCount(); - - // Get header information - List
headerList = Lists.newArrayListWithExpectedSize(col); - executeResult.setHeaderList(headerList); - int chat2dbAutoRowIdIndex = -1;// Row paging ID automatically generated by chat2db - - boolean isMongoMap = false; - for (int i = 1; i <= col; i++) { - String name = ResultSetUtils.getColumnName(resultSetMetaData, i); - // The returned map is from mongodb, and you need to parse the map yourself - if (DataSourceTypeEnum.MONGODB.getCode().equals(type) && i == 1 && "map".equals(name)) { - isMongoMap = true; - break; - } - if ("CAHT2DB_AUTO_ROW_ID".equals(name)) { - chat2dbAutoRowIdIndex = i; - continue; - } - String dataType = JdbcUtils.resolveDataType( - resultSetMetaData.getColumnTypeName(i), resultSetMetaData.getColumnType(i)).getCode(); - headerList.add(Header.builder() - .dataType(dataType) - .name(name) - .build()); - } + executeResult = generateQueryExecuteResult(stmt, limitRowSize, offset, count); + } else { + // Modification or other + executeResult.setUpdateCount(stmt.getUpdateCount()); + } + executeResult.setDuration(timeInterval.interval()); + } + return executeResult; + } - // Get data information - List> dataList = Lists.newArrayList(); - executeResult.setDataList(dataList); + private ExecuteResult generateQueryExecuteResult(Statement stmt, boolean limitRowSize, Integer offset, + Integer count) throws SQLException { + ExecuteResult executeResult = ExecuteResult.builder().success(Boolean.TRUE).build(); + executeResult.setDescription(I18nUtils.getMessage("sqlResult.success")); + ResultSet rs = null; + try { + rs = stmt.getResultSet(); + // Get how many columns + ResultSetMetaData resultSetMetaData = rs.getMetaData(); + int col = resultSetMetaData.getColumnCount(); + // Get header information + List
headerList = generateHeaderList(resultSetMetaData); + + + int chat2dbAutoRowIdIndex = getChat2dbAutoRowIdIndex(headerList); + // Get data information + List> dataList = generateDataList(rs, col, chat2dbAutoRowIdIndex, limitRowSize, + offset, count); + + executeResult.setHeaderList(headerList); + executeResult.setDataList(dataList); + } finally { + JdbcUtils.closeResultSet(rs); + } + return executeResult; + } - Map headerListMap = null; - List> dataListMap = null; - if (isMongoMap) { - headerListMap = Maps.newLinkedHashMap(); - dataListMap = Lists.newArrayList(); - } + private List> generateDataList(ResultSet rs, int col, int chat2dbAutoRowIdIndex, + boolean limitRowSize, Integer offset, Integer count) throws SQLException { + List> dataList = Lists.newArrayList(); - if (offset == null || offset < 0) { - offset = 0; - } - int rowNumber = 0; - int rowCount = 1; - while (rs.next()) { - if (rowNumber++ < offset) { - continue; - } - if (!isMongoMap) { - List row = Lists.newArrayListWithExpectedSize(col); - dataList.add(row); - for (int i = 1; i <= col; i++) { - if (chat2dbAutoRowIdIndex == i) { - continue; - } -// ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); -// row.add((String) valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i))); - row.add(valueHandler.getString(rs, i, limitRowSize)); - } - } else { - for (int i = 1; i <= col; i++) { - Object o = rs.getObject(i); - Map row = Maps.newHashMap(); - dataListMap.add(row); - LinkedHashMap data = DocumentUtils.convertToMap(o); - if (data != null) { - for (String string : data.keySet()) { - headerListMap.computeIfAbsent(string, k -> Header.builder() - .dataType("string") - .name(string) - .build()); - row.put(string, Objects.toString(data.get(string))); - } - } else { - headerListMap.computeIfAbsent("_unknown", k -> Header.builder() - .dataType("string") - .name("_unknown") - .build()); - row.put("_unknown", Objects.toString(o)); - } - } - } - if (count != null && count > 0 && rowCount++ >= count) { - break; - } - } + if (offset == null || offset < 0) { + offset = 0; + } + int rowNumber = 0; + int rowCount = 1; + while (rs.next()) { + if (rowNumber++ < offset) { + continue; + } + List row = Lists.newArrayListWithExpectedSize(col); + dataList.add(row); + for (int i = 1; i <= col; i++) { + if (chat2dbAutoRowIdIndex == i) { + continue; + } + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i,false))); + } + if (count != null && count > 0 && rowCount++ >= count) { + break; + } + } + return dataList; + } - if (isMongoMap) { - headerList.addAll(headerListMap.values().stream().toList()); - for (Map stringStringMap : dataListMap) { - List dataTempList = Lists.newArrayList(); - dataList.add(dataTempList); - for (Header value : headerListMap.values()) { - dataTempList.add(stringStringMap.get(value.getName())); - } - } - } + private int getChat2dbAutoRowIdIndex(List
headerList) { - executeResult.setDuration(timeInterval.interval()); - } finally { - JdbcUtils.closeResultSet(rs); - } - } else { - executeResult.setDuration(timeInterval.interval()); - // Modification or other - executeResult.setUpdateCount(stmt.getUpdateCount()); + for (int i = 0; i < headerList.size(); i++) { + Header header = headerList.get(i); + if ("CAHT2DB_AUTO_ROW_ID".equals(header.getName())) { + headerList.remove(i); + return i; } } - return executeResult; + return -1; } - /** - * Execute SQL - * - * @param connection - * @param sql - * @return - * @throws SQLException - */ - public ExecuteResult execute(Connection connection, String sql, ValueHandler valueHandler) throws SQLException { - return execute(sql, connection, true, null, null, valueHandler); + + private List
generateHeaderList(ResultSetMetaData resultSetMetaData) throws SQLException { + int col = resultSetMetaData.getColumnCount(); + List
headerList = Lists.newArrayListWithExpectedSize(col); + for (int i = 1; i <= col; i++) { + headerList.add(Header.builder() + .dataType(JdbcUtils.resolveDataType( + resultSetMetaData.getColumnTypeName(i), resultSetMetaData.getColumnType(i)).getCode()) + .name(ResultSetUtils.getColumnName(resultSetMetaData, i)) + .build()); + } + return headerList; } + public ExecuteResult execute(Connection connection, String sql) throws SQLException { - return execute(sql, connection, true, null, null, new DefaultValueHandler()); + return execute(sql, connection, true, null, null); } /** @@ -402,34 +346,8 @@ public List schemas(Connection connection, String databaseName, String s * @return */ public List
tables(Connection connection, String databaseName, String schemaName, String tableName, - String types[]) { - - try { - DatabaseMetaData metadata = connection.getMetaData(); - ResultSet resultSet = metadata.getTables(databaseName, schemaName, tableName, - types); - // // If connection is mysql - // if ("MySQL".equalsIgnoreCase(metadata.getDatabaseProductName())) { - // // Get the comment of mysql table - // List
tables = ResultSetUtils.toObjectList(resultSet, Table.class); - // if (CollectionUtils.isNotEmpty(tables)) { - // for (Table table : tables) { - // String sql = "show table status where name = '" + table.getName() + "'"; - // try (Statement stmt = connection.createStatement()) { - // boolean query = stmt.execute(sql); - // if (query) { - // try (ResultSet rs = stmt.getResultSet();) { - // while (rs.next()) { - // table.setComment(rs.getString("Comment")); - // } - // } - // } - // } - // } - // - // return tables; - // } - // } + String types[]) { + try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, tableName, types)) { return ResultSetUtils.toObjectList(resultSet, Table.class); } catch (SQLException e) { throw new RuntimeException(e); @@ -447,7 +365,7 @@ public List
tables(Connection connection, String databaseName, String sch * @return */ public List tableNames(Connection connection, String databaseName, String schemaName, String tableName, - String[] types) { + String[] types) { List tableNames = new ArrayList<>(); try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, tableName, types)) { while (resultSet.next()) { @@ -470,10 +388,10 @@ public List tableNames(Connection connection, String databaseName, Strin * @return */ public List columns(Connection connection, String databaseName, String schemaName, String - tableName, - String columnName) { + tableName, + String columnName) { try (ResultSet resultSet = connection.getMetaData().getColumns(databaseName, schemaName, tableName, - columnName)) { + columnName)) { return ResultSetUtils.toObjectList(resultSet, TableColumn.class); } catch (Exception e) { throw new RuntimeException(e); @@ -492,28 +410,29 @@ public List columns(Connection connection, String databaseName, Str public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { List tableIndices = Lists.newArrayList(); try (ResultSet resultSet = connection.getMetaData().getIndexInfo(databaseName, schemaName, tableName, - false, - false)) { + false, + false)) { List tableIndexColumns = ResultSetUtils.toObjectList(resultSet, TableIndexColumn.class); tableIndexColumns.stream().filter(c -> c.getIndexName() != null).collect( - Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() - .stream().forEach(entry -> { - TableIndex tableIndex = new TableIndex(); - TableIndexColumn column = entry.getValue().get(0); - tableIndex.setName(entry.getKey()); - tableIndex.setTableName(column.getTableName()); - tableIndex.setSchemaName(column.getSchemaName()); - tableIndex.setDatabaseName(column.getDatabaseName()); - tableIndex.setUnique(!column.getNonUnique()); - tableIndex.setColumnList(entry.getValue()); - tableIndices.add(tableIndex); - }); + Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() + .stream().forEach(entry -> { + TableIndex tableIndex = new TableIndex(); + TableIndexColumn column = entry.getValue().get(0); + tableIndex.setName(entry.getKey()); + tableIndex.setTableName(column.getTableName()); + tableIndex.setSchemaName(column.getSchemaName()); + tableIndex.setDatabaseName(column.getDatabaseName()); + tableIndex.setUnique(!column.getNonUnique()); + tableIndex.setColumnList(entry.getValue()); + tableIndices.add(tableIndex); + }); } catch (SQLException e) { throw new RuntimeException(e); } return tableIndices; } + /** * Get all functions available in a catalog. * @@ -523,7 +442,7 @@ public List indexes(Connection connection, String databaseName, Stri * @return List */ public List functions(Connection connection, String databaseName, - String schemaName) { + String schemaName) { try (ResultSet resultSet = connection.getMetaData().getFunctions(databaseName, schemaName, null);) { return ResultSetUtils.toObjectList(resultSet, ai.chat2db.spi.model.Function.class); } catch (Exception e) { @@ -578,13 +497,12 @@ public String getDbVersion(Connection connection) { @Override public List execute(Command command) { + if(StringUtils.isBlank(command.getScript())){ + return Collections.emptyList(); + } // parse sql String type = Chat2DBContext.getConnectInfo().getDbType(); DbType dbType = JdbcUtils.parse2DruidDbType(type); - // if ("SQLSERVER".equalsIgnoreCase(type)) { - // RemoveSpecialGO(param); - // } - List sqlList = SqlUtils.parse(command.getScript(), dbType); if (CollectionUtils.isEmpty(sqlList)) { @@ -600,35 +518,14 @@ public List execute(Command command) { } private ExecuteResult executeSQL(String originalSql, DbType dbType, Command param) { - int pageNo = 1; - int pageSize = 0; - Integer offset = null; - Integer count = null; - String sqlType = SqlTypeEnum.UNKNOWN.getCode(); - // parse sql - String type = Chat2DBContext.getConnectInfo().getDbType(); - boolean supportDruid = !DataSourceTypeEnum.MONGODB.getCode().equals(type); - // Parse sql pagination - SQLStatement sqlStatement = null; - if (supportDruid) { - try { - sqlStatement = SQLUtils.parseSingleStatement(originalSql, dbType); - } catch (Exception e) { - log.warn("Failed to parse sql: {}", originalSql, e); - } - } - - // Mongodb is currently unable to recognize it, so every time a page is transmitted - if (!supportDruid || (sqlStatement instanceof SQLSelectStatement)) { - pageNo = Optional.ofNullable(param.getPageNo()).orElse(1); - pageSize = Optional.ofNullable(param.getPageSize()).orElse(EasyToolsConstant.MAX_PAGE_SIZE); - offset = (pageNo - 1) * pageSize; - count = pageSize; - sqlType = SqlTypeEnum.SELECT.getCode(); - } - + int pageNo = Optional.ofNullable(param.getPageNo()).orElse(1); + int pageSize = Optional.ofNullable(param.getPageSize()).orElse(EasyToolsConstant.MAX_PAGE_SIZE); + Integer offset = (pageNo - 1) * pageSize; + Integer count = pageSize; + SqlTypeEnum sqlType = getSqlType(dbType, originalSql); ExecuteResult executeResult = null; - if (SqlTypeEnum.SELECT.getCode().equals(sqlType) && !SqlUtils.hasPageLimit(originalSql, dbType)) { + + if (SqlTypeEnum.SELECT.equals(sqlType) && !SqlUtils.hasPageLimit(originalSql, dbType)) { String pageLimit = Chat2DBContext.getSqlBuilder().pageLimit(originalSql, offset, pageNo, pageSize); if (StringUtils.isNotBlank(pageLimit)) { executeResult = execute(pageLimit, 0, count); @@ -638,40 +535,62 @@ private ExecuteResult executeSQL(String originalSql, DbType dbType, Command para executeResult = execute(originalSql, offset, count); } - executeResult.setSqlType(sqlType); + executeResult.setSqlType(sqlType.getCode()); executeResult.setOriginalSql(originalSql); - boolean supportJsqlParser = !DataSourceTypeEnum.MONGODB.getCode().equals(type); - if (supportJsqlParser) { + SqlUtils.buildCanEditResult(originalSql, dbType, executeResult); + // Add row number + addRowNumber(executeResult, pageNo, pageSize); + // Total number of fuzzy rows + setPageInfo(executeResult, sqlType, pageNo, pageSize); + return executeResult; + } + + private SqlTypeEnum getSqlType(DbType dbType, String originalSql) { + SqlTypeEnum sqlType = SqlTypeEnum.UNKNOWN; + // parse sql + String type = Chat2DBContext.getConnectInfo().getDbType(); + boolean supportDruid = !DataSourceTypeEnum.MONGODB.getCode().equals(type); + SQLStatement sqlStatement = null; + if (supportDruid) { try { - SqlUtils.buildCanEditResult(originalSql, dbType, executeResult); + sqlStatement = SQLUtils.parseSingleStatement(originalSql, dbType); } catch (Exception e) { - log.warn("buildCanEditResult error", e); + log.warn("Failed to parse sql: {}", originalSql, e); } } - if (SqlTypeEnum.SELECT.getCode().equals(sqlType)) { + // Mongodb is currently unable to recognize it, so every time a page is transmitted + if (!supportDruid || (sqlStatement instanceof SQLSelectStatement)) { + sqlType = SqlTypeEnum.SELECT; + } + return sqlType; + } + + private void setPageInfo(ExecuteResult executeResult, SqlTypeEnum sqlType, int pageNo, int pageSize) { + if (SqlTypeEnum.SELECT.equals(sqlType)) { executeResult.setPageNo(pageNo); executeResult.setPageSize(pageSize); executeResult.setHasNextPage( - CollectionUtils.size(executeResult.getDataList()) >= executeResult.getPageSize()); + CollectionUtils.size(executeResult.getDataList()) >= executeResult.getPageSize()); } else { executeResult.setPageNo(pageNo); executeResult.setPageSize(CollectionUtils.size(executeResult.getDataList())); executeResult.setHasNextPage(Boolean.FALSE); } + executeResult.setFuzzyTotal(calculateFuzzyTotal(pageNo, pageSize, executeResult)); + } + + private void addRowNumber(ExecuteResult executeResult, int pageNo, int pageSize) { List
headers = executeResult.getHeaderList(); - // if (executeResult.getSuccess() && executeResult.isCanEdit() && CollectionUtils.isNotEmpty(headers)) { - // headers = setColumnInfo(headers, executeResult.getTableName(), param.getSchemaName(), - // param.getDatabaseName()); - // } Header rowNumberHeader = Header.builder() - .name(I18nUtils.getMessage("sqlResult.rowNumber")) - .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER - .getCode()).build(); - + .name(I18nUtils.getMessage("sqlResult.rowNumber")) + .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER + .getCode()).build(); executeResult.setHeaderList(EasyCollectionUtils.union(Arrays.asList(rowNumberHeader), headers)); + + // Add row number if (executeResult.getDataList() != null) { int rowNumberIncrement = 1 + Math.max(pageNo - 1, 0) * pageSize; for (int i = 0; i < executeResult.getDataList().size(); i++) { @@ -682,11 +601,9 @@ private ExecuteResult executeSQL(String originalSql, DbType dbType, Command para executeResult.getDataList().set(i, newRow); } } - // Total number of fuzzy rows - executeResult.setFuzzyTotal(calculateFuzzyTotal(pageNo, pageSize, executeResult)); - return executeResult; } + private String calculateFuzzyTotal(int pageNo, int pageSize, ExecuteResult executeResult) { int dataSize = CollectionUtils.size(executeResult.getDataList()); if (pageSize <= 0) { @@ -696,22 +613,20 @@ private String calculateFuzzyTotal(int pageNo, int pageSize, ExecuteResult execu if (dataSize < pageSize) { return Integer.toString(fuzzyTotal); } - return Integer.toString(fuzzyTotal) + "+"; + return fuzzyTotal + "+"; } private ExecuteResult execute(String sql, Integer offset, Integer count) { ExecuteResult executeResult; try { - ValueHandler valueHandler = Chat2DBContext.getMetaData().getValueHandler(); - executeResult = SQLExecutor.getInstance().execute(sql, Chat2DBContext.getConnection(), true, offset, count, - valueHandler); + executeResult = SQLExecutor.getInstance().execute(sql, Chat2DBContext.getConnection(), true, offset, count); } catch (SQLException e) { log.error("Execute sql: {} exception", sql, e); executeResult = ExecuteResult.builder() - .sql(sql) - .success(Boolean.FALSE) - .message(e.getMessage()) - .build(); + .sql(sql) + .success(Boolean.FALSE) + .message(e.getMessage()) + .build(); } return executeResult; } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index 14b686f6b..e26d496ea 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -1,15 +1,21 @@ package ai.chat2db.spi.util; +import ai.chat2db.server.tools.common.util.I18nUtils; +import cn.hutool.core.io.unit.DataSizeUtil; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; import java.io.InputStream; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.*; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -19,6 +25,7 @@ * @author jipengfei * @version : ResultSetUtils.java */ +@Slf4j public class ResultSetUtils { @@ -107,10 +114,81 @@ public static int getColumnScale(ResultSetMetaData resultSetMetaData, int column public static String getString(ResultSet rs, int columnIndex){ try { - return rs.getString(columnIndex); - } catch (SQLException e) { - throw new RuntimeException(e); + Object obj = rs.getObject(columnIndex); + if (obj == null) { + return null; + } + if (obj instanceof BigDecimal bigDecimal) { + return bigDecimal.toPlainString(); + } else if (obj instanceof Double d) { + return BigDecimal.valueOf(d).toPlainString(); + } else if (obj instanceof Float f) { + return BigDecimal.valueOf(f).toPlainString(); + } else if (obj instanceof Clob) { + return largeString(rs, columnIndex); + } else if (obj instanceof byte[]) { + return largeString(rs, columnIndex); + } else if (obj instanceof Blob blob) { + return largeStringBlob(blob); + } else if (obj instanceof Timestamp || obj instanceof LocalDateTime) { + return largeTime(obj); + } else if (obj instanceof SQLXML){ + return ((SQLXML) obj).getString(); + } else { + return obj.toString(); + } + } catch (Exception e) { + log.warn("Failed to parse number:{},", columnIndex, e); + try { + return rs.getString(columnIndex); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + } + + private static String largeStringBlob(Blob blob) throws SQLException { + if (blob == null) { + return null; + } + int length = Math.toIntExact(blob.length()); + byte[] data = blob.getBytes(1, length); + String result = new String(data, StandardCharsets.UTF_8); + return result; + } + + private static String largeTime(Object obj) throws SQLException { + Object timeField = obj; // Assuming a time field of type Object + + LocalDateTime localDateTime; + + if (obj instanceof Timestamp) { + // Convert a time field of type Object to a LocalDateTime object + localDateTime = ((Timestamp) timeField).toLocalDateTime(); + } else if(obj instanceof LocalDateTime){ + localDateTime = (LocalDateTime) timeField; + } else { + try { + localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")); + }catch (Exception e){ + localDateTime = LocalDateTime.parse(timeField.toString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm")); + } + } + // Create a DateTimeFormatter instance and specify the output date and time format + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + // Format date time + String formattedDateTime = dtf.format(localDateTime); + return formattedDateTime; + } + + private static String largeString(ResultSet rs, int index) throws SQLException { + String result = rs.getString(index); + if (result == null) { + return null; + } + return result; } public static InputStream getBinaryStream(ResultSet rs, int columnIndex) { From 2bfbbaeb87f54a1e758989317660da18e51f70f0 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 21 Jun 2024 18:29:12 +0800 Subject: [PATCH 18/73] fix-exportTableData --- .../plugin/clickhouse/ClickHouseDBManage.java | 2 +- .../ai/chat2db/plugin/db2/DB2DBManage.java | 10 +-- .../java/ai/chat2db/plugin/dm/DMDBManage.java | 30 ++++---- .../chat2db/plugin/mysql/MysqlDBManage.java | 19 ++--- .../chat2db/plugin/oracle/OracleDBManage.java | 44 ++---------- .../plugin/postgresql/PostgreSQLDBManage.java | 32 +++++---- .../chat2db/plugin/sqlite/SqliteDBManage.java | 6 +- .../plugin/sqlserver/SqlServerDBManage.java | 65 +++++++++-------- .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 69 +++++++++---------- 9 files changed, 128 insertions(+), 149 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java index 745ab65e6..d05b760b2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java @@ -55,7 +55,7 @@ private void exportTablesOrViewsOrDictionaries(Connection connection,String data .append(";").append("\n").append(ddl).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); if (asyncContext.isContainsData() && dataFlag) { - exportTableData(connection,schemaName, tableOrViewName, asyncContext); + exportTableData(connection, databaseName,schemaName, tableOrViewName, asyncContext); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java index d5dc9445f..d8354c4c0 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java @@ -20,22 +20,22 @@ public class DB2DBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - exportTables(connection, schemaName, asyncContext); + exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, schemaName, asyncContext); exportProceduresAndFunctions(connection, schemaName, asyncContext); exportTriggers(connection, schemaName, asyncContext); } - private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection, schemaName, resultSet.getString("TABLE_NAME"), asyncContext); + exportTable(connection, databaseName, schemaName, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, SQLConstant.TABLE_DDL_FUNCTION_SQL, resultSet -> null); } catch (Exception e) { @@ -48,7 +48,7 @@ private void exportTable(Connection connection, String schemaName, String tableN sqlBuilder.append(resultSet.getString("sql")).append("\n"); asyncContext.write(sqlBuilder.toString()); if (asyncContext.isContainsData()) { - exportTableData(connection, schemaName, tableName, asyncContext); + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java index 38282f301..5b1bcbdfd 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java @@ -1,8 +1,5 @@ package ai.chat2db.plugin.dm; -import java.sql.*; -import java.util.Objects; - import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; import ai.chat2db.spi.model.AsyncContext; @@ -13,6 +10,11 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + @Slf4j public class DMDBManage extends DefaultDBManage implements DBManage { private String format(String tableName) { @@ -29,30 +31,30 @@ private String format(String tableName) { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - exportTables(connection, schemaName, asyncContext); + exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, schemaName, asyncContext); exportProcedures(connection, schemaName, asyncContext); exportTriggers(connection, schemaName, asyncContext); } - private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT TABLE_NAME FROM ALL_TABLES where OWNER='%s' and TABLESPACE_NAME='MAIN'", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); - exportTable(connection, tableName, schemaName, asyncContext); + exportTable(connection, databaseName, tableName, schemaName, asyncContext); } } } - private void exportTable(Connection connection, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = """ - SELECT - (SELECT comments FROM user_tab_comments WHERE table_name = '%s') AS comments, - (SELECT dbms_metadata.get_ddl('TABLE', '%s', '%s') FROM dual) AS ddl - FROM dual; - """; + SELECT + (SELECT comments FROM user_tab_comments WHERE table_name = '%s') AS comments, + (SELECT dbms_metadata.get_ddl('TABLE', '%s', '%s') FROM dual) AS ddl + FROM dual; + """; try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(String.format(sql, tableName, tableName, schemaName))) { String formatSchemaName = format(schemaName); String formatTableName = format(tableName); @@ -70,14 +72,14 @@ private void exportTable(Connection connection, String tableName, String schemaN exportTableColumnComment(connection, schemaName, tableName, asyncContext); } if (asyncContext.isContainsData()) { - exportTableData(connection, schemaName, tableName, asyncContext); + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } } private void exportTableColumnComment(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select COLNAME,COMMENT$ from SYS.SYSCOLUMNCOMMENTS\n" + - "where SCHNAME = '%s' and TVNAME = '%s'and TABLE_TYPE = 'TABLE';", schemaName, tableName); + "where SCHNAME = '%s' and TVNAME = '%s'and TABLE_TYPE = 'TABLE';", schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index b59acb1e6..2d7a49b96 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -7,13 +7,14 @@ import ai.chat2db.spi.sql.SQLExecutor; import org.springframework.util.StringUtils; -import java.sql.*; -import java.util.Objects; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - exportTables(connection, databaseName, asyncContext); + exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, databaseName, asyncContext); exportProcedures(connection, asyncContext); exportTriggers(connection, asyncContext); @@ -35,22 +36,22 @@ private void exportFunction(Connection connection, String functionName, AsyncCon if (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n") - .append(resultSet.getString("Create Function")).append(";").append("\n"); + .append(resultSet.getString("Create Function")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); } } } - private void exportTables(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection, resultSet.getString("TABLE_NAME"), asyncContext); + exportTable(connection, databaseName, schemaName, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create table %s ", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { @@ -59,7 +60,7 @@ private void exportTable(Connection connection, String tableName, AsyncContext a .append(resultSet.getString("Create Table")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); if (asyncContext.isContainsData()) { - exportTableData(connection, null,tableName, asyncContext); + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } } @@ -143,7 +144,7 @@ public void updateProcedure(Connection connection, String databaseName, String s } catch (Exception e) { connection.rollback(); throw new RuntimeException(e); - }finally { + } finally { connection.setAutoCommit(true); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index b6f256c49..717163749 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -30,29 +30,25 @@ public class OracleDBManage extends DefaultDBManage implements DBManage { private static String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', trigger_name) AS ddl FROM all_triggers WHERE owner = '%s' AND trigger_name = '%s'"; private static String FUNCTION_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('FUNCTION', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; - @Override - public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - exportTableData(connection, tableName, asyncContext); - } public void exportDatabase(Connection connection, String databaseName, String schemaName,AsyncContext asyncContext) throws SQLException { - exportTables(connection, schemaName, asyncContext); + exportTables(connection,databaseName, schemaName, asyncContext); exportViews(connection, asyncContext, schemaName); exportProcedures(connection, schemaName, asyncContext); exportTriggers(connection, schemaName, asyncContext); exportFunctions(connection, schemaName, asyncContext); } - private void exportTables(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); - exportTable(connection, schemaName, tableName, asyncContext); + exportTable(connection,databaseName, schemaName, tableName, asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TABLE_DDL_SQL, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { @@ -64,7 +60,7 @@ private void exportTable(Connection connection, String schemaName, String tableN exportTableComments(connection, tableName, asyncContext); exportTableColumnsComments(connection, tableName, asyncContext); if (asyncContext.isContainsData()) { - exportTableData(connection, tableName, asyncContext); + exportTableData(connection,databaseName,schemaName, tableName, asyncContext); } } } @@ -91,36 +87,6 @@ private void exportTableColumnsComments(Connection connection, String tableName, } } - private void exportTableData(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { - String sql = String.format("SELECT * FROM %s", tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - while (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); - for (int i = 1; i <= columnCount; i++) { - String columnValue = resultSet.getString(i); - if (Objects.isNull(columnValue)) { - sqlBuilder.append("NULL"); - } else if (metaData.getColumnTypeName(i).equalsIgnoreCase("DATE")) { - // 处理日期值格式 - columnValue = "TO_DATE('" + columnValue + "', 'YYYY-MM-DD HH24:MI:SS')"; - sqlBuilder.append(columnValue); - } else { - sqlBuilder.append("'").append(columnValue).append("'"); - } - if (i < columnCount) { - sqlBuilder.append(", "); - } - } - sqlBuilder.append(");"); - sqlBuilder.append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } - } - private void exportViews(Connection connection, AsyncContext asyncContext, String schemaName) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"VIEW"})) { while (resultSet.next()) { diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java index 4c5086565..3e31c7770 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java @@ -8,11 +8,13 @@ import ai.chat2db.spi.sql.SQLExecutor; import org.apache.commons.lang3.StringUtils; -import java.sql.*; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; -import java.util.Objects; -import static ai.chat2db.plugin.postgresql.consts.SQLConst.*; +import static ai.chat2db.plugin.postgresql.consts.SQLConst.ENUM_TYPE_DDL_SQL; public class PostgreSQLDBManage extends DefaultDBManage implements DBManage { @@ -25,17 +27,18 @@ public void exportDatabase(Connection connection, String databaseName, String sc } private void exportTypes(Connection connection, AsyncContext asyncContext) throws SQLException { - try (ResultSet resultSet = connection.createStatement().executeQuery(ENUM_TYPE_DDL_SQL)) { - while (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("ddl")).append("\n"); - asyncContext.write(sqlBuilder.toString()); - } + try (ResultSet resultSet = connection.createStatement().executeQuery(ENUM_TYPE_DDL_SQL)) { + while (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append(resultSet.getString("ddl")).append("\n"); + asyncContext.write(sqlBuilder.toString()); + } } } + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, null, - new String[]{"TABLE", "SYSTEM TABLE","PARTITIONED TABLE"})) { + new String[]{"TABLE", "SYSTEM TABLE", "PARTITIONED TABLE"})) { ArrayList tableNames = new ArrayList<>(); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); @@ -46,14 +49,14 @@ private void exportTables(Connection connection, String databaseName, String sch } if (asyncContext.isContainsData()) { for (String tableName : tableNames) { - exportTableData(connection, schemaName, tableName, asyncContext); + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } } } private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - String sql =String.format( "select pg_get_tabledef('%s','%s',true,'COMMENTS') as ddl;", schemaName,tableName); + String sql = String.format("select pg_get_tabledef('%s','%s',true,'COMMENTS') as ddl;", schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); @@ -65,10 +68,9 @@ private void exportTable(Connection connection, String schemaName, String tableN } - private void exportViews(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { - String sql = String.format("SELECT table_name, view_definition FROM information_schema.views WHERE table_schema = '%s'",schemaName); + String sql = String.format("SELECT table_name, view_definition FROM information_schema.views WHERE table_schema = '%s'", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); @@ -82,7 +84,7 @@ private void exportViews(Connection connection, String schemaName, AsyncContext private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT proname, pg_get_functiondef(oid) AS function_definition FROM pg_proc " + - "WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')", schemaName); + "WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java index 306814445..6b959ad37 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java @@ -20,13 +20,13 @@ public void exportDatabase(Connection connection, String databaseName, String sc private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection,schemaName, resultSet.getString("TABLE_NAME"), asyncContext); + exportTable(connection, databaseName,schemaName, resultSet.getString("TABLE_NAME"), asyncContext); } } } - private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT sql FROM sqlite_master WHERE type='table' AND name='%s'", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { @@ -35,7 +35,7 @@ private void exportTable(Connection connection, String schemaName, String tableN .append(resultSet.getString("sql")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); if (asyncContext.isContainsData()) { - exportTableData(connection,schemaName, tableName, asyncContext); + exportTableData(connection, databaseName,schemaName, tableName, asyncContext); } } } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 906576b0f..9ee14ad37 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -1,11 +1,21 @@ package ai.chat2db.plugin.sqlserver; import ai.chat2db.spi.DBManage; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.jdbc.DefaultDBManage; import ai.chat2db.spi.model.AsyncContext; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.SQLExecutor; - -import java.sql.*; +import ai.chat2db.spi.util.ResultSetUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; public class SqlServerDBManage extends DefaultDBManage implements DBManage { @@ -44,36 +54,37 @@ public class SqlServerDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - exportTableData(connection, tableName, asyncContext); + exportTableData(connection,databaseName,schemaName, tableName, asyncContext); } + @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - exportTables(connection, schemaName, asyncContext); + exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, databaseName, schemaName, asyncContext); exportFunctions(connection, schemaName, asyncContext); exportProcedures(connection, schemaName, asyncContext); exportTriggers(connection, asyncContext); } - private void exportTables(Connection connection, String schemaName,AsyncContext asyncContext) throws SQLException { - String sql ="SELECT name FROM SysObjects Where XType='U'"; + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + String sql = "SELECT name FROM SysObjects Where XType='U'"; try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String tableName = resultSet.getString("name"); - exportTable(connection, tableName, schemaName, asyncContext); + exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } } - private void exportTable(Connection connection, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { + private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, tableDDLFunction.replace("tableSchema", schemaName), resultSet -> null); } catch (Exception e) { //log.error("Failed to create function", e); } - String sql = String.format("SELECT %s.ufn_GetCreateTableScript('%s', '%s') as ddl",schemaName,schemaName,tableName); + String sql = String.format("SELECT %s.ufn_GetCreateTableScript('%s', '%s') as ddl", schemaName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); @@ -81,7 +92,7 @@ private void exportTable(Connection connection, String tableName, String schemaN .append(resultSet.getString("ddl")).append("\n"); asyncContext.write(sqlBuilder.toString()); if (asyncContext.isContainsData()) { - exportTableData(connection, tableName, asyncContext); + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } else { asyncContext.write("go \n"); } @@ -90,29 +101,26 @@ private void exportTable(Connection connection, String tableName, String schemaN } - private void exportTableData(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { - String sql = String.format("select * from %s", tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + public void exportTableData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) { + SqlBuilder sqlBuilder = Chat2DBContext.getSqlBuilder(); + String tableQuerySql = sqlBuilder.buildTableQuerySql(databaseName, schemaName, tableName); + SQLExecutor.getInstance().execute(connection, tableQuerySql, 1000, resultSet -> { ResultSetMetaData metaData = resultSet.getMetaData(); + List columnList = ResultSetUtils.getRsHeader(resultSet); + List valueList = new ArrayList<>(); while (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); for (int i = 1; i <= metaData.getColumnCount(); i++) { - String value = resultSet.getString(i); - if (Objects.isNull(value)) { - sqlBuilder.append("NULL"); - } else { - sqlBuilder.append("'").append(value).append("'"); - } - if (i < metaData.getColumnCount()) { - sqlBuilder.append(", "); - } + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); + String valueString = valueProcessor.getJdbcValueString(jdbcDataValue); + valueList.add(valueString); } - sqlBuilder.append(");\n"); - asyncContext.write(sqlBuilder.toString()); + String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, columnList, valueList); + asyncContext.write(insertSql); + valueList.clear(); } - } - asyncContext.write("go \n"); + asyncContext.write("go \n"); + }); } private void exportViews(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { @@ -190,6 +198,7 @@ private void exportTriggers(Connection connection, AsyncContext asyncContext) th } } } + @Override public void connectDatabase(Connection connection, String database) { try { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 10369b460..7ab7ad71c 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -1,25 +1,29 @@ package ai.chat2db.spi.jdbc; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.Objects; - import ai.chat2db.server.tools.base.excption.BusinessException; import ai.chat2db.server.tools.common.exception.ConnectionException; import ai.chat2db.spi.DBManage; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.AsyncContext; +import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.model.SSHInfo; +import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.IDriverManager; import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.ssh.SSHManager; -import com.jcraft.jsch.JSchException; +import ai.chat2db.spi.util.ResultSetUtils; import com.jcraft.jsch.Session; import org.apache.commons.lang3.StringUtils; +import java.sql.Connection; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + /** * @author jipengfei * @version : DefaultDBManage.java @@ -51,10 +55,10 @@ public Connection getConnection(ConnectInfo connectInfo) { } try { connection = IDriverManager.getConnection(url, connectInfo.getUser(), connectInfo.getPassword(), - connectInfo.getDriverConfig(), connectInfo.getExtendMap()); + connectInfo.getDriverConfig(), connectInfo.getExtendMap()); - }catch (Exception e1) { - close(connection,session,ssh); + } catch (Exception e1) { + close(connection, session, ssh); throw new BusinessException("connection.error", null, e1); } connectInfo.setSession(session); @@ -64,7 +68,8 @@ public Connection getConnection(ConnectInfo connectInfo) { } return connection; } - private void close(Connection connection,Session session,SSHInfo ssh){ + + private void close(Connection connection, Session session, SSHInfo ssh) { if (connection != null) { try { connection.close(); @@ -150,8 +155,9 @@ public void updateProcedure(Connection connection, String databaseName, String s public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { } - public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - exportTableData(connection, schemaName,tableName, asyncContext); + + public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } @Override @@ -160,33 +166,26 @@ public void dropTable(Connection connection, String databaseName, String schemaN SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } - public void exportTableData(Connection connection,String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - }else{ - sql = String.format("select * from %s.%s",schemaName,tableName); - } - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + public void exportTableData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) { + SqlBuilder sqlBuilder = Chat2DBContext.getSqlBuilder(); + String tableQuerySql = sqlBuilder.buildTableQuerySql(databaseName, schemaName, tableName); + SQLExecutor.getInstance().execute(connection, tableQuerySql, 1000, resultSet -> { ResultSetMetaData metaData = resultSet.getMetaData(); + List columnList = ResultSetUtils.getRsHeader(resultSet); + List valueList = new ArrayList<>(); while (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("INSERT INTO ").append(tableName).append(" VALUES ("); for (int i = 1; i <= metaData.getColumnCount(); i++) { - String value = resultSet.getString(i); - if (Objects.isNull(value)) { - sqlBuilder.append("NULL"); - } else { - sqlBuilder.append("'").append(value).append("'"); - } - if (i < metaData.getColumnCount()) { - sqlBuilder.append(", "); - } + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); + String valueString = valueProcessor.getJdbcValueString(jdbcDataValue); + valueList.add(valueString); } - sqlBuilder.append(");\n"); - asyncContext.write(sqlBuilder.toString()); + String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, columnList, valueList); + asyncContext.write(insertSql); + valueList.clear(); } asyncContext.write("\n"); - } + }); + } } \ No newline at end of file From 8edfa686d41067200cf656b9b84e87fdf1a29190 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Fri, 21 Jun 2024 19:54:09 +0800 Subject: [PATCH 19/73] batch export --- .../chat2db/plugin/mysql/MysqlDBManage.java | 5 ++++ .../domain/core/impl/DatabaseServiceImpl.java | 22 +++++++++++++++- .../java/ai/chat2db/spi/model/AsyncCall.java | 17 ++++++++++++ .../ai/chat2db/spi/model/AsyncContext.java | 26 ++++++++++++++++--- 4 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index 2d7a49b96..4970935ee 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -15,10 +15,15 @@ public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { exportTables(connection, databaseName, schemaName, asyncContext); + asyncContext.setProgress(50); exportViews(connection, databaseName, asyncContext); + asyncContext.setProgress(60); exportProcedures(connection, asyncContext); + asyncContext.setProgress(70); exportTriggers(connection, asyncContext); + asyncContext.setProgress(90); exportFunctions(connection, databaseName, asyncContext); + asyncContext.finish(); } private void exportFunctions(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java index 4ebd494f5..d62c1178f 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java @@ -179,7 +179,27 @@ public ActionResult modifySchema(SchemaOperationParam param) { public String exportDatabase(DatabaseExportParam param) throws SQLException { AsyncContext asyncContext = new AsyncContext(); asyncContext.setContainsData(param.getContainData()); - asyncContext.setConsumer(aLong -> log.info("exportDatabase success")); + asyncContext.setCall(new AsyncCall() { + @Override + public void setProgress(int progress) { + + } + + @Override + public void info(String message) { + + } + + @Override + public void error(String message) { + + } + + @Override + public void finish() { + + } + }); Chat2DBContext.getDBManage().exportDatabase(Chat2DBContext.getConnection(), param.getDatabaseName(), param.getSchemaName(), asyncContext); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java new file mode 100644 index 000000000..18ea9889e --- /dev/null +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java @@ -0,0 +1,17 @@ +package ai.chat2db.spi.model; + +public interface AsyncCall { + + + void setProgress(int progress); + + + void info(String message); + + + void error(String message); + + + void finish(); + +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 5bacc7898..c2f5aa862 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -18,11 +18,29 @@ public class AsyncContext { private boolean containsData; - private Consumer consumer; + private AsyncCall call; - public void addProgress(Long progress) { - if (consumer != null) { - consumer.accept(progress); + public void setProgress(Integer progress) { + if (call != null) { + call.setProgress(progress); + } + } + + public void info(String message) { + if (call != null) { + call.info(message); + } + } + + public void error(String message) { + if (call != null) { + call.error(message); + } + } + + public void finish() { + if (call != null) { + call.finish(); } } From 5d80e3837a37c1030b41cbd09c8575efea1aa933 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Sun, 23 Jun 2024 21:23:24 +0800 Subject: [PATCH 20/73] batch export --- .../chat2db-server-tools-common/pom.xml | 5 ---- chat2db-server/chat2db-spi/pom.xml | 5 ++++ .../main/java/ai/chat2db/spi/SqlBuilder.java | 9 ++++++++ .../ai/chat2db/spi/model/AsyncContext.java | 9 +++++++- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 23 ++++++++++++------- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml index 704b53b7c..2958b662d 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/pom.xml @@ -18,11 +18,6 @@ ai.chat2db chat2db-server-tools-base - - org.apache.tika - tika-core - 2.7.0 - org.apache.commons commons-lang3 diff --git a/chat2db-server/chat2db-spi/pom.xml b/chat2db-server/chat2db-spi/pom.xml index efea9f6ee..69c2c7b68 100644 --- a/chat2db-server/chat2db-spi/pom.xml +++ b/chat2db-server/chat2db-spi/pom.xml @@ -90,6 +90,11 @@ ob-sql-parser 1.2.1 + + org.apache.tika + tika-core + 2.7.0 + diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java index 4e59080cf..07dc5e01f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/SqlBuilder.java @@ -94,8 +94,17 @@ public interface SqlBuilder { */ String getTableDmlSql(T table, String type); + /** + * + * @param databaseName + * @param schemaName + * @param tableName + * @return + */ String buildTableQuerySql(String databaseName, String schemaName, String tableName); + + String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList); String buildMultiInsertSql(String databaseName, String schemaName, String tableName, List columnList, List> valueLists); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index c2f5aa862..19bcf9101 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -4,6 +4,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; import java.io.PrintWriter; import java.util.function.Consumer; @@ -11,7 +12,7 @@ @Data @AllArgsConstructor @NoArgsConstructor -@Builder +@SuperBuilder public class AsyncContext { private PrintWriter writer; @@ -39,9 +40,14 @@ public void error(String message) { } public void finish() { + if (writer != null) { + writer.flush(); + writer.close(); + } if (call != null) { call.finish(); } + } public void write(String message) { @@ -49,4 +55,5 @@ public void write(String message) { writer.write(message); } } + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 3adc08e65..b947eb079 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -1,11 +1,6 @@ package ai.chat2db.spi.sql; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -253,7 +248,7 @@ private List> generateDataList(ResultSet rs, int col, int chat2dbAu continue; } ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); - row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i,false))); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, false))); } if (count != null && count > 0 && rowCount++ >= count) { break; @@ -497,7 +492,7 @@ public String getDbVersion(Connection connection) { @Override public List execute(Command command) { - if(StringUtils.isBlank(command.getScript())){ + if (StringUtils.isBlank(command.getScript())) { return Collections.emptyList(); } // parse sql @@ -646,4 +641,16 @@ public void execute(Connection connection, String sql, int batchSize, ResultSetC throw new RuntimeException(e); } } + + public void executeBatchInsert(Connection connection, List sqlCacheList) { + try (Statement stmt = connection.createStatement()) { + for (String sql : sqlCacheList) { + stmt.addBatch(sql); + } + stmt.executeBatch(); + stmt.clearBatch(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } From 737c1fb284f68250ffe01d20e8e7cb569a962cbe Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Tue, 25 Jun 2024 20:53:33 +0800 Subject: [PATCH 21/73] batch export --- .../ai/chat2db/plugin/db2/DB2DBManage.java | 2 +- .../java/ai/chat2db/plugin/dm/DMDBManage.java | 2 +- .../chat2db/plugin/mysql/MysqlDBManage.java | 23 +++++++++----- .../chat2db/plugin/oracle/OracleDBManage.java | 2 +- .../plugin/postgresql/PostgreSQLDBManage.java | 17 +++++------ .../chat2db/plugin/sqlite/SqliteDBManage.java | 2 +- .../plugin/sqlserver/SqlServerDBManage.java | 6 +--- .../main/java/ai/chat2db/spi/DBManage.java | 2 +- .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 30 +++++++++++++++++-- .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 2 +- 10 files changed, 58 insertions(+), 30 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java index d8354c4c0..39114b593 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java @@ -35,7 +35,7 @@ private void exportTables(Connection connection, String databaseName, String sch } - private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, SQLConstant.TABLE_DDL_FUNCTION_SQL, resultSet -> null); } catch (Exception e) { diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java index 5b1bcbdfd..812970901 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/DMDBManage.java @@ -48,7 +48,7 @@ private void exportTables(Connection connection, String databaseName, String sch } - private void exportTable(Connection connection, String databaseName, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String tableName, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = """ SELECT (SELECT comments FROM user_tab_comments WHERE table_name = '%s') AS comments, diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index 4970935ee..d7d183dd3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -5,15 +5,18 @@ import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.sql.SQLExecutor; +import cn.hutool.core.date.DateUtil; import org.springframework.util.StringUtils; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Date; public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + asyncContext.write(String.format(EXPORT_TITLE, DateUtil.formatDate(new Date()))); exportTables(connection, databaseName, schemaName, asyncContext); asyncContext.setProgress(50); exportViews(connection, databaseName, asyncContext); @@ -39,6 +42,7 @@ private void exportFunction(Connection connection, String functionName, AsyncCon String sql = String.format("SHOW CREATE FUNCTION %s;", functionName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + asyncContext.write(String.format(FUNCTION_TITLE, functionName)); StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n") .append(resultSet.getString("Create Function")).append(";").append("\n"); @@ -48,22 +52,26 @@ private void exportFunction(Connection connection, String functionName, AsyncCon } private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + asyncContext.write("SET FOREIGN_KEY_CHECKS=0;"); try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { - exportTable(connection, databaseName, schemaName, resultSet.getString("TABLE_NAME"), asyncContext); + String tableName = resultSet.getString("TABLE_NAME"); + exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } } - private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("show create table %s ", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); + asyncContext.write(String.format(EXPORT_TITLE, tableName)); sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") .append(resultSet.getString("Create Table")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); + asyncContext.write("\n"); if (asyncContext.isContainsData()) { exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } @@ -84,9 +92,10 @@ private void exportView(Connection connection, String viewName, AsyncContext asy String sql = String.format("show create view %s ", viewName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + asyncContext.write(String.format(VIEW_TITLE, viewName)); StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP VIEW IF EXISTS ").append(format(viewName)).append(";").append("\n") - .append(resultSet.getString("Create View")).append(";").append("\n"); + .append(resultSet.getString("Create View")).append(";").append("\n\n"); asyncContext.write(sqlBuilder.toString()); } } @@ -105,10 +114,11 @@ private void exportProcedure(Connection connection, String procedureName, AsyncC String sql = String.format("show create procedure %s ", procedureName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + asyncContext.write(String.format(PROCEDURE_TITLE, procedureName)); StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP PROCEDURE IF EXISTS ").append(format(procedureName)).append(";").append("\n") .append("delimiter ;;").append("\n").append(resultSet.getString("Create Procedure")).append(";;") - .append("\n").append("delimiter ;").append("\n"); + .append("\n").append("delimiter ;").append("\n\n"); asyncContext.write(sqlBuilder.toString()); } } @@ -128,10 +138,11 @@ private void exportTrigger(Connection connection, String triggerName, AsyncConte String sql = String.format("show create trigger %s ", triggerName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { + asyncContext.write(String.format(TRIGGER_TITLE, triggerName)); StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("DROP TRIGGER IF EXISTS ").append(format(triggerName)).append(";").append("\n") .append("delimiter ;;").append("\n").append(resultSet.getString("SQL Original Statement")).append(";;") - .append("\n").append("delimiter ;").append("\n"); + .append("\n").append("delimiter ;").append("\n\n"); asyncContext.write(sqlBuilder.toString()); } } @@ -140,12 +151,10 @@ private void exportTrigger(Connection connection, String triggerName, AsyncConte @Override public void updateProcedure(Connection connection, String databaseName, String schemaName, Procedure procedure) throws SQLException { try { - connection.setAutoCommit(false); String sql = "DROP PROCEDURE " + procedure.getProcedureName(); SQLExecutor.getInstance().execute(connection, sql, resultSet -> {}); String procedureBody = procedure.getProcedureBody(); SQLExecutor.getInstance().execute(connection, procedureBody, resultSet -> {}); - connection.commit(); } catch (Exception e) { connection.rollback(); throw new RuntimeException(e); diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index 717163749..3554eb7ad 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -48,7 +48,7 @@ private void exportTables(Connection connection, String databaseName, String sch } - private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format(TABLE_DDL_SQL, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java index 3e31c7770..d09c5c03b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java @@ -38,24 +38,20 @@ private void exportTypes(Connection connection, AsyncContext asyncContext) throw private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, null, - new String[]{"TABLE", "SYSTEM TABLE", "PARTITIONED TABLE"})) { + new String[]{"TABLE", "SYSTEM TABLE", "PARTITIONED TABLE"})) { ArrayList tableNames = new ArrayList<>(); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); tableNames.add(tableName); } for (String tableName : tableNames) { - exportTable(connection, schemaName, tableName, asyncContext); - } - if (asyncContext.isContainsData()) { - for (String tableName : tableNames) { - exportTableData(connection, databaseName, schemaName, tableName, asyncContext); - } + exportTable(connection, databaseName,schemaName, tableName, asyncContext); } + } } - private void exportTable(Connection connection, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection,String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("select pg_get_tabledef('%s','%s',true,'COMMENTS') as ddl;", schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { @@ -63,6 +59,9 @@ private void exportTable(Connection connection, String schemaName, String tableN sqlBuilder.append("\n").append("DROP TABLE IF EXISTS ").append(tableName).append(";").append("\n") .append(resultSet.getString("ddl")).append("\n"); asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); + } } } } @@ -84,7 +83,7 @@ private void exportViews(Connection connection, String schemaName, AsyncContext private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT proname, pg_get_functiondef(oid) AS function_definition FROM pg_proc " + - "WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')", schemaName); + "WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')", schemaName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java index 6b959ad37..c284da489 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/SqliteDBManage.java @@ -26,7 +26,7 @@ private void exportTables(Connection connection, String databaseName, String sch } - private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String sql = String.format("SELECT sql FROM sqlite_master WHERE type='table' AND name='%s'", tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 9ee14ad37..5b3901ee5 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -52,10 +52,6 @@ public class SqlServerDBManage extends DefaultDBManage implements DBManage { + "triggerDefinition, CASE WHEN status & 1 = 1 THEN 'Enabled' ELSE 'Disabled' END AS Status FROM sysobjects " + "WHERE xtype = 'TR' "; - @Override - public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - exportTableData(connection,databaseName,schemaName, tableName, asyncContext); - } @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { @@ -77,7 +73,7 @@ private void exportTables(Connection connection, String databaseName, String sch } - private void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { try { SQLExecutor.getInstance().execute(connection, tableDDLFunction.replace("tableSchema", schemaName), resultSet -> null); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java index 7e811be2e..343e4b52c 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java @@ -139,5 +139,5 @@ void dropProcedure(Connection connection, @NotEmpty String databaseName, String * @param tableName * @return */ - void exportDatabaseData(Connection connection, String databaseName, String schemaName,String tableName,AsyncContext asyncContext) throws SQLException; + void exportTable(Connection connection, String databaseName, String schemaName,String tableName,AsyncContext asyncContext) throws SQLException; } \ No newline at end of file diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 7ab7ad71c..d5a75c84d 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -30,6 +30,24 @@ */ public class DefaultDBManage implements DBManage { + protected static final String DIVIDING_LINE = "-- ----------------------------"; + + protected static final String NEW_LINE = "\n"; + + protected static final String EXPORT_TITLE = DIVIDING_LINE + NEW_LINE + "-- Chat2DB export data , export time: %s" + NEW_LINE + DIVIDING_LINE; + + protected static final String TABLE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Table structure for table %s" + NEW_LINE + DIVIDING_LINE; + + protected static final String VIEW_TITLE = DIVIDING_LINE + NEW_LINE +"-- View structure for view %s"+ NEW_LINE + DIVIDING_LINE; + + protected static final String FUNCTION_TITLE = DIVIDING_LINE + NEW_LINE +"-- Function structure for function %s"+ NEW_LINE + DIVIDING_LINE; + + protected static final String TRIGGER_TITLE = DIVIDING_LINE + NEW_LINE + "-- Trigger structure for trigger %s"+ NEW_LINE + DIVIDING_LINE; + + protected static final String PROCEDURE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Procedure structure for procedure %s"+ NEW_LINE + DIVIDING_LINE; + + private static final String RECORD_TITLE = DIVIDING_LINE + NEW_LINE +"-- Records of "+ NEW_LINE + DIVIDING_LINE; + @Override public Connection getConnection(ConnectInfo connectInfo) { @@ -55,7 +73,7 @@ public Connection getConnection(ConnectInfo connectInfo) { } try { connection = IDriverManager.getConnection(url, connectInfo.getUser(), connectInfo.getPassword(), - connectInfo.getDriverConfig(), connectInfo.getExtendMap()); + connectInfo.getDriverConfig(), connectInfo.getExtendMap()); } catch (Exception e1) { close(connection, session, ssh); @@ -156,10 +174,15 @@ public void exportDatabase(Connection connection, String databaseName, String sc } - public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - exportTableData(connection, databaseName, schemaName, tableName, asyncContext); + @Override + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + } +// public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { +// exportTableData(connection, databaseName, schemaName, tableName, asyncContext); +// } + @Override public void dropTable(Connection connection, String databaseName, String schemaName, String tableName) { String sql = "DROP TABLE " + tableName; @@ -173,6 +196,7 @@ public void exportTableData(Connection connection, String databaseName, String s ResultSetMetaData metaData = resultSet.getMetaData(); List columnList = ResultSetUtils.getRsHeader(resultSet); List valueList = new ArrayList<>(); + asyncContext.write(String.format(RECORD_TITLE, tableName)); while (resultSet.next()) { for (int i = 1; i <= metaData.getColumnCount(); i++) { ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 33df0264f..24f7b13c7 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -218,7 +218,7 @@ protected String buildBaseInsertSql(String databaseName, String schemaName, Stri StringBuilder script = new StringBuilder(); script.append("INSERT INTO "); - buildTableName(databaseName, schemaName, tableName, script); + buildTableName(null, null, tableName, script); if (CollectionUtils.isNotEmpty(columnList)) { script.append(" (") From 8691c1955f68f16493eb179d01790bcaa7b2df6b Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Tue, 25 Jun 2024 21:29:14 +0800 Subject: [PATCH 22/73] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../builder/ClickHouseSqlBuilder.java | 13 ++++++-- .../plugin/db2/builder/DB2SqlBuilder.java | 12 ++++++++ .../plugin/dm/builder/DMSqlBuilder.java | 12 ++++++++ .../plugin/hive/builder/HiveSqlBuilder.java | 12 ++++++++ .../kingbase/builder/KingBaseSqlBuilder.java | 30 +++++++++++++++++++ .../plugin/mysql/builder/MysqlSqlBuilder.java | 12 ++++++++ .../oracle/builder/OracleSqlBuilder.java | 16 ++++++++-- .../builder/PostgreSQLSqlBuilder.java | 30 +++++++++++++++++++ .../plugin/sqlite/builder/SqliteBuilder.java | 15 ++++++++++ .../builder/SqlServerSqlBuilder.java | 12 ++++++++ 10 files changed, 160 insertions(+), 4 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java index f093c198a..1decd44bf 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java @@ -27,6 +27,9 @@ public String buildCreateTableSql(Table table) { continue; } ClickHouseColumnTypeEnum typeEnum = ClickHouseColumnTypeEnum.getByType(column.getColumnType()); + if (typeEnum != null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -85,6 +88,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableColumn tableColumn : newTable.getColumnList()) { if (StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) { ClickHouseColumnTypeEnum typeEnum = ClickHouseColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(",\n"); } } @@ -92,9 +98,12 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { // append modify index for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { - ClickHouseIndexTypeEnum mysqlIndexTypeEnum = ClickHouseIndexTypeEnum + ClickHouseIndexTypeEnum clickHouseIndexTypeEnum = ClickHouseIndexTypeEnum .getByType(tableIndex.getType()); - script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); + if(clickHouseIndexTypeEnum == null){ + continue; + } + script.append("\t").append(clickHouseIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/builder/DB2SqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/builder/DB2SqlBuilder.java index 20dc5cfbe..8b68dcab9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/builder/DB2SqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/builder/DB2SqlBuilder.java @@ -22,6 +22,9 @@ public String buildCreateTableSql(Table table) { continue; } DB2ColumnTypeEnum typeEnum = DB2ColumnTypeEnum.getByType(column.getColumnType()); + if (typeEnum == null) { + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -33,6 +36,9 @@ public String buildCreateTableSql(Table table) { continue; } DB2IndexTypeEnum indexTypeEnum = DB2IndexTypeEnum.getByType(tableIndex.getType()); + if (indexTypeEnum == null) { + continue; + } script.append("\n").append("").append(indexTypeEnum.buildIndexScript(tableIndex)).append(";"); if(StringUtils.isNotBlank(tableIndex.getComment())){ script.append("\n").append(indexTypeEnum.buildIndexComment(tableIndex)).append(";"); @@ -84,6 +90,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableColumn tableColumn : newTable.getColumnList()) { if (StringUtils.isNotBlank(tableColumn.getEditStatus())) { DB2ColumnTypeEnum typeEnum = DB2ColumnTypeEnum.getByType(tableColumn.getColumnType()); + if (typeEnum == null) { + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(";\n"); if (StringUtils.isNotBlank(tableColumn.getComment())) { script.append("\n").append(buildComment(tableColumn)).append(";\n"); @@ -95,6 +104,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { DB2IndexTypeEnum mysqlIndexTypeEnum = DB2IndexTypeEnum.getByType(tableIndex.getType()); + if (mysqlIndexTypeEnum == null) { + continue; + } script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); if(StringUtils.isNotBlank(tableIndex.getComment())) { script.append("\n").append(mysqlIndexTypeEnum.buildIndexComment(tableIndex)).append(";\n"); diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/builder/DMSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/builder/DMSqlBuilder.java index 8ea2ef08b..e5ce7e869 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/builder/DMSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/builder/DMSqlBuilder.java @@ -26,6 +26,9 @@ public String buildCreateTableSql(Table table) { continue; } DMColumnTypeEnum typeEnum = DMColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -37,6 +40,9 @@ public String buildCreateTableSql(Table table) { continue; } DMIndexTypeEnum indexTypeEnum = DMIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } script.append("\n").append("").append(indexTypeEnum.buildIndexScript(tableIndex)).append(";"); } @@ -85,6 +91,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { String editStatus = tableColumn.getEditStatus(); if (StringUtils.isNotBlank(editStatus)) { DMColumnTypeEnum typeEnum = DMColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(";\n"); if (StringUtils.isNotBlank(tableColumn.getComment())&&!Objects.equals(EditStatus.DELETE.toString(),editStatus)) { script.append("\n").append(buildComment(tableColumn)).append(";\n"); @@ -96,6 +105,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { DMIndexTypeEnum mysqlIndexTypeEnum = DMIndexTypeEnum.getByType(tableIndex.getType()); + if(mysqlIndexTypeEnum == null){ + continue; + } script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java index fc690bcf8..4ec08031e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java @@ -26,6 +26,9 @@ public String buildCreateTableSql(Table table) { continue; } HiveColumnTypeEnum typeEnum = HiveColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -35,6 +38,9 @@ public String buildCreateTableSql(Table table) { continue; } HiveIndexTypeEnum hiveIndexTypeEnum = HiveIndexTypeEnum.getByType(tableIndex.getType()); + if(hiveIndexTypeEnum == null){ + continue; + } script.append("\t").append("").append(hiveIndexTypeEnum.buildIndexScript(tableIndex)).append(",\n"); } @@ -98,6 +104,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableColumn tableColumn : newTable.getColumnList()) { if (StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) { HiveColumnTypeEnum typeEnum = HiveColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(",\n"); } } @@ -106,6 +115,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { HiveIndexTypeEnum hiveIndexTypeEnum = HiveIndexTypeEnum.getByType(tableIndex.getType()); + if(hiveIndexTypeEnum == null){ + continue; + } script.append("\t").append(hiveIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/builder/KingBaseSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/builder/KingBaseSqlBuilder.java index 9dcbf920d..2d6942ffa 100644 --- a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/builder/KingBaseSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/builder/KingBaseSqlBuilder.java @@ -26,6 +26,9 @@ public String buildCreateTableSql(Table table) { continue; } KingBaseColumnTypeEnum typeEnum = KingBaseColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum ==null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } Map> tableIndexMap = table.getIndexList().stream() @@ -38,6 +41,9 @@ public String buildCreateTableSql(Table table) { continue; } KingBaseIndexTypeEnum indexTypeEnum = KingBaseIndexTypeEnum.getByType(index.getType()); + if(indexTypeEnum == null){ + continue; + } script.append("\t").append("").append(indexTypeEnum.buildIndexScript(index)); script.append(",\n"); } @@ -58,6 +64,9 @@ public String buildCreateTableSql(Table table) { } script.append("\n"); KingBaseIndexTypeEnum indexTypeEnum = KingBaseIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } script.append("").append(indexTypeEnum.buildIndexScript(tableIndex)).append(";"); } @@ -70,12 +79,18 @@ public String buildCreateTableSql(Table table) { List tableColumnList = table.getColumnList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableColumn tableColumn : tableColumnList) { KingBaseColumnTypeEnum typeEnum = KingBaseColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append(typeEnum.buildComment(tableColumn, typeEnum)).append("\n"); ; } List indexList = table.getIndexList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableIndex index : indexList) { KingBaseIndexTypeEnum indexEnum = KingBaseIndexTypeEnum.getByType(index.getType()); + if(indexEnum == null){ + continue; + } script.append(indexEnum.buildIndexComment(index)).append("\n"); } @@ -109,6 +124,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { // append modify column for (TableColumn tableColumn : newTable.getColumnList()) { KingBaseColumnTypeEnum typeEnum = KingBaseColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } scriptModify.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(",\n"); modify = true; @@ -118,6 +136,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : tableIndexMap.get(Boolean.FALSE)) { if (StringUtils.isNotBlank(tableIndex.getType())) { KingBaseIndexTypeEnum indexTypeEnum = KingBaseIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } scriptModify.append("\t").append(indexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); modify = true; } @@ -133,6 +154,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : tableIndexMap.get(Boolean.TRUE)) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { KingBaseIndexTypeEnum indexTypeEnum = KingBaseIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } script.append(indexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); } } @@ -145,12 +169,18 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { } for (TableColumn tableColumn : newTable.getColumnList()) { KingBaseColumnTypeEnum typeEnum = KingBaseColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append(typeEnum.buildComment(tableColumn, typeEnum)).append("\n"); ; } List indexList = newTable.getIndexList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableIndex index : indexList) { KingBaseIndexTypeEnum indexEnum = KingBaseIndexTypeEnum.getByType(index.getType()); + if(indexEnum == null){ + continue; + } script.append(indexEnum.buildIndexComment(index)).append("\n"); } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index 9a2ad6c7c..168a05b93 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -30,6 +30,9 @@ public String buildCreateTableSql(Table table) { continue; } MysqlColumnTypeEnum typeEnum = MysqlColumnTypeEnum.getByType(column.getColumnType()); + if (typeEnum == null) { + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -39,6 +42,9 @@ public String buildCreateTableSql(Table table) { continue; } MysqlIndexTypeEnum mysqlIndexTypeEnum = MysqlIndexTypeEnum.getByType(tableIndex.getType()); + if (mysqlIndexTypeEnum == null) { + continue; + } script.append("\t").append("").append(mysqlIndexTypeEnum.buildIndexScript(tableIndex)).append(",\n"); } @@ -110,6 +116,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { if ((StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) || moveColumnList.contains(tableColumn) || addColumnList.contains(tableColumn)) { MysqlColumnTypeEnum typeEnum = MysqlColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } if (moveColumnList.contains(tableColumn) || addColumnList.contains(tableColumn)) { script.append("\t").append(typeEnum.buildModifyColumn(tableColumn, true, findPrevious(tableColumn, newTable))).append(",\n"); } else { @@ -122,6 +131,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { MysqlIndexTypeEnum mysqlIndexTypeEnum = MysqlIndexTypeEnum.getByType(tableIndex.getType()); + if(mysqlIndexTypeEnum == null){ + continue; + } script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java index 57d9b4a3f..ef93beffd 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java @@ -20,6 +20,9 @@ public String buildCreateTableSql(Table table) { continue; } OracleColumnTypeEnum typeEnum = OracleColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -31,6 +34,9 @@ public String buildCreateTableSql(Table table) { continue; } OracleIndexTypeEnum oracleColumnTypeEnum = OracleIndexTypeEnum.getByType(tableIndex.getType()); + if(oracleColumnTypeEnum == null){ + continue; + } script.append("\n").append("").append(oracleColumnTypeEnum.buildIndexScript(tableIndex)).append(";"); } @@ -78,6 +84,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableColumn tableColumn : newTable.getColumnList()) { if (StringUtils.isNotBlank(tableColumn.getEditStatus())) { OracleColumnTypeEnum typeEnum = OracleColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(";\n"); if (StringUtils.isNotBlank(tableColumn.getComment())) { script.append("\n").append(buildComment(tableColumn)).append(";\n"); @@ -88,8 +97,11 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { // append modify index for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { - OracleIndexTypeEnum mysqlIndexTypeEnum = OracleIndexTypeEnum.getByType(tableIndex.getType()); - script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); + OracleIndexTypeEnum oracleIndexTypeEnum = OracleIndexTypeEnum.getByType(tableIndex.getType()); + if(oracleIndexTypeEnum == null){ + continue; + } + script.append("\t").append(oracleIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); } } if (script.length() > 2) { diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/builder/PostgreSQLSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/builder/PostgreSQLSqlBuilder.java index 34a856cb7..faf8ed0dd 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/builder/PostgreSQLSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/builder/PostgreSQLSqlBuilder.java @@ -25,6 +25,9 @@ public String buildCreateTableSql(Table table) { continue; } PostgreSQLColumnTypeEnum typeEnum = PostgreSQLColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } Map> tableIndexMap = table.getIndexList().stream() @@ -37,6 +40,9 @@ public String buildCreateTableSql(Table table) { continue; } PostgreSQLIndexTypeEnum indexTypeEnum = PostgreSQLIndexTypeEnum.getByType(index.getType()); + if(indexTypeEnum == null){ + continue; + } script.append("\t").append("").append(indexTypeEnum.buildIndexScript(index)); script.append(",\n"); } @@ -53,6 +59,9 @@ public String buildCreateTableSql(Table table) { } script.append("\n"); PostgreSQLIndexTypeEnum indexTypeEnum = PostgreSQLIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } script.append("").append(indexTypeEnum.buildIndexScript(tableIndex)).append(";"); } @@ -65,12 +74,18 @@ public String buildCreateTableSql(Table table) { List tableColumnList = table.getColumnList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableColumn tableColumn : tableColumnList) { PostgreSQLColumnTypeEnum typeEnum = PostgreSQLColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append(typeEnum.buildComment(tableColumn, typeEnum)).append("\n"); ; } List indexList = table.getIndexList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableIndex index : indexList) { PostgreSQLIndexTypeEnum indexEnum = PostgreSQLIndexTypeEnum.getByType(index.getType()); + if(indexEnum == null){ + continue; + } script.append(indexEnum.buildIndexComment(index)).append("\n"); ; } @@ -105,6 +120,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { // append modify column for (TableColumn tableColumn : newTable.getColumnList()) { PostgreSQLColumnTypeEnum typeEnum = PostgreSQLColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } scriptModify.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(",\n"); modify = true; @@ -114,6 +132,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : tableIndexMap.get(Boolean.FALSE)) { if (StringUtils.isNotBlank(tableIndex.getType())) { PostgreSQLIndexTypeEnum indexTypeEnum = PostgreSQLIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } scriptModify.append("\t").append(indexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); modify = true; } @@ -129,6 +150,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : tableIndexMap.get(Boolean.TRUE)) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { PostgreSQLIndexTypeEnum indexTypeEnum = PostgreSQLIndexTypeEnum.getByType(tableIndex.getType()); + if(indexTypeEnum == null){ + continue; + } script.append(indexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); } } @@ -141,12 +165,18 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { } for (TableColumn tableColumn : newTable.getColumnList()) { PostgreSQLColumnTypeEnum typeEnum = PostgreSQLColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum ==null){ + continue; + } script.append(typeEnum.buildComment(tableColumn, typeEnum)).append("\n"); ; } List indexList = newTable.getIndexList().stream().filter(v -> StringUtils.isNotBlank(v.getComment())).toList(); for (TableIndex index : indexList) { PostgreSQLIndexTypeEnum indexEnum = PostgreSQLIndexTypeEnum.getByType(index.getType()); + if(indexEnum == null){ + continue; + } script.append(indexEnum.buildIndexComment(index)).append("\n"); } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/builder/SqliteBuilder.java b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/builder/SqliteBuilder.java index 5878d6675..68637449d 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/builder/SqliteBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/builder/SqliteBuilder.java @@ -23,11 +23,17 @@ public String buildCreateTableSql(Table table) { continue; } SqliteColumnTypeEnum typeEnum = SqliteColumnTypeEnum.getByType(column.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } for (TableIndex tableIndex : table.getIndexList()) { if(SqliteIndexTypeEnum.PRIMARY_KEY.getName().equals( tableIndex.getType())) { SqliteIndexTypeEnum sqliteIndexTypeEnum = SqliteIndexTypeEnum.getByType(tableIndex.getType()); + if(sqliteIndexTypeEnum == null){ + continue; + } script.append("\t").append(sqliteIndexTypeEnum.buildIndexScript(tableIndex)).append(",\n"); } } @@ -41,6 +47,9 @@ public String buildCreateTableSql(Table table) { } if(!SqliteIndexTypeEnum.PRIMARY_KEY.getName().equals( tableIndex.getType())) { SqliteIndexTypeEnum sqliteIndexTypeEnum = SqliteIndexTypeEnum.getByType(tableIndex.getType()); + if(sqliteIndexTypeEnum == null){ + continue; + } script.append("\n").append("CREATE ").append(sqliteIndexTypeEnum.buildIndexScript(tableIndex)).append(";\n"); } } @@ -61,6 +70,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { if (StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) { script.append("ALTER TABLE ").append("\"").append(newTable.getDatabaseName()).append("\".\"").append(newTable.getName()).append("\"").append("\n"); SqliteColumnTypeEnum typeEnum = SqliteColumnTypeEnum.getByType(tableColumn.getColumnType()); + if(typeEnum == null){ + continue; + } script.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append(";\n"); } } @@ -70,6 +82,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { // script.append("ALTER TABLE ").append("\"").append(newTable.getDatabaseName()).append("\".\"").append(newTable.getName()).append("\"").append("\n"); SqliteIndexTypeEnum sqliteIndexTypeEnum = SqliteIndexTypeEnum.getByType(tableIndex.getType()); + if(sqliteIndexTypeEnum == null){ + continue; + } script.append("\t").append(sqliteIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/builder/SqlServerSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/builder/SqlServerSqlBuilder.java index 85a0baee8..c0f661766 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/builder/SqlServerSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/builder/SqlServerSqlBuilder.java @@ -20,6 +20,9 @@ public String buildCreateTableSql(Table table) { continue; } SqlServerColumnTypeEnum typeEnum = SqlServerColumnTypeEnum.getByType(column.getColumnType()); + if (typeEnum == null) { + continue; + } script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); } @@ -31,6 +34,9 @@ public String buildCreateTableSql(Table table) { continue; } SqlServerIndexTypeEnum sqlServerIndexTypeEnum = SqlServerIndexTypeEnum.getByType(tableIndex.getType()); + if (sqlServerIndexTypeEnum == null) { + continue; + } script.append("\n").append(sqlServerIndexTypeEnum.buildIndexScript(tableIndex)); if (StringUtils.isNotBlank(tableIndex.getComment())) { script.append("\n").append(buildIndexComment(tableIndex)); @@ -92,6 +98,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableColumn tableColumn : newTable.getColumnList()) { if (StringUtils.isNotBlank(tableColumn.getEditStatus())) { SqlServerColumnTypeEnum typeEnum = SqlServerColumnTypeEnum.getByType(tableColumn.getColumnType()); + if (typeEnum == null) { + continue; + } script.append(typeEnum.buildModifyColumn(tableColumn)).append("\n"); } } @@ -100,6 +109,9 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { SqlServerIndexTypeEnum mysqlIndexTypeEnum = SqlServerIndexTypeEnum.getByType(tableIndex.getType()); + if (mysqlIndexTypeEnum == null) { + continue; + } script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append("\n"); if (StringUtils.isNotBlank(tableIndex.getComment())) { script.append("\n").append(buildIndexComment(tableIndex)).append("\ngo"); From 028c3ec4ac0ce503d553d34ece5857f6e0a65cc0 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 26 Jun 2024 16:37:41 +0800 Subject: [PATCH 23/73] export info --- .../main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java | 8 +++++++- .../server/domain/core/impl/DatabaseServiceImpl.java | 6 ++++++ .../main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java | 1 + .../src/main/java/ai/chat2db/spi/model/AsyncCall.java | 5 +++++ .../src/main/java/ai/chat2db/spi/model/AsyncContext.java | 6 +++--- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index d7d183dd3..ab4bc2ad6 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -6,6 +6,7 @@ import ai.chat2db.spi.model.Procedure; import ai.chat2db.spi.sql.SQLExecutor; import cn.hutool.core.date.DateUtil; +import lombok.extern.slf4j.Slf4j; import org.springframework.util.StringUtils; import java.sql.Connection; @@ -13,6 +14,7 @@ import java.sql.SQLException; import java.util.Date; +@Slf4j public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { @@ -52,13 +54,14 @@ private void exportFunction(Connection connection, String functionName, AsyncCon } private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - asyncContext.write("SET FOREIGN_KEY_CHECKS=0;"); + asyncContext.write("SET FOREIGN_KEY_CHECKS=0;\n"); try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } + asyncContext.write("SET FOREIGN_KEY_CHECKS=1;"); } @@ -76,6 +79,9 @@ public void exportTable(Connection connection, String databaseName, String schem exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } + }catch (Exception e){ + log.error("export table error", e); + asyncContext.getCall().error(String.format("export table %s error:%s", tableName,e.getMessage())); } } diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java index d62c1178f..11ba165cc 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.function.Consumer; @@ -195,6 +196,11 @@ public void error(String message) { } + @Override + public void update(Map map) { + + } + @Override public void finish() { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index d5a75c84d..c90a0ab27 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -192,6 +192,7 @@ public void dropTable(Connection connection, String databaseName, String schemaN public void exportTableData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) { SqlBuilder sqlBuilder = Chat2DBContext.getSqlBuilder(); String tableQuerySql = sqlBuilder.buildTableQuerySql(databaseName, schemaName, tableName); + asyncContext.info("export table data sql: " + tableQuerySql); SQLExecutor.getInstance().execute(connection, tableQuerySql, 1000, resultSet -> { ResultSetMetaData metaData = resultSet.getMetaData(); List columnList = ResultSetUtils.getRsHeader(resultSet); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java index 18ea9889e..de5e09c11 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java @@ -1,5 +1,7 @@ package ai.chat2db.spi.model; +import java.util.Map; + public interface AsyncCall { @@ -12,6 +14,9 @@ public interface AsyncCall { void error(String message); + void update(Map map); + + void finish(); } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 19bcf9101..4222fca0d 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -15,11 +15,11 @@ @SuperBuilder public class AsyncContext { - private PrintWriter writer; + protected PrintWriter writer; - private boolean containsData; + protected boolean containsData; - private AsyncCall call; + protected AsyncCall call; public void setProgress(Integer progress) { if (call != null) { From 9a6e89dfb45e929d7c0638773c634f1caae40c1d Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 26 Jun 2024 22:41:48 +0800 Subject: [PATCH 24/73] export info --- .../chat2db/plugin/mysql/MysqlDBManage.java | 5 +- .../domain/core/impl/DatabaseServiceImpl.java | 26 +--- .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 3 +- .../java/ai/chat2db/spi/model/AsyncCall.java | 13 -- .../ai/chat2db/spi/model/AsyncContext.java | 117 ++++++++++++++---- 5 files changed, 102 insertions(+), 62 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index ab4bc2ad6..c1a026108 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -61,7 +61,7 @@ private void exportTables(Connection connection, String databaseName, String sch exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } - asyncContext.write("SET FOREIGN_KEY_CHECKS=1;"); + asyncContext.write("SET FOREIGN_KEY_CHECKS=1;\n"); } @@ -74,14 +74,13 @@ public void exportTable(Connection connection, String databaseName, String schem sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") .append(resultSet.getString("Create Table")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); - asyncContext.write("\n"); if (asyncContext.isContainsData()) { exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } }catch (Exception e){ log.error("export table error", e); - asyncContext.getCall().error(String.format("export table %s error:%s", tableName,e.getMessage())); + asyncContext.error(String.format("export table %s error:%s", tableName,e.getMessage())); } } diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java index 11ba165cc..eb6da6693 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java @@ -19,6 +19,7 @@ import ai.chat2db.server.tools.base.wrapper.result.ActionResult; import ai.chat2db.server.tools.base.wrapper.result.DataResult; import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.common.util.ContextUtils; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.Chat2DBContext; @@ -178,34 +179,15 @@ public ActionResult modifySchema(SchemaOperationParam param) { @Override public String exportDatabase(DatabaseExportParam param) throws SQLException { - AsyncContext asyncContext = new AsyncContext(); - asyncContext.setContainsData(param.getContainData()); - asyncContext.setCall(new AsyncCall() { - @Override - public void setProgress(int progress) { - - } - - @Override - public void info(String message) { - - } - - @Override - public void error(String message) { - - } + AsyncCall call = new AsyncCall() { @Override public void update(Map map) { } + }; - @Override - public void finish() { - - } - }); + AsyncContext asyncContext = new AsyncContext(call, ContextUtils.queryContext(), null, param.getContainData()); Chat2DBContext.getDBManage().exportDatabase(Chat2DBContext.getConnection(), param.getDatabaseName(), param.getSchemaName(), asyncContext); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index c90a0ab27..3c41ee03c 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -46,7 +46,7 @@ public class DefaultDBManage implements DBManage { protected static final String PROCEDURE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Procedure structure for procedure %s"+ NEW_LINE + DIVIDING_LINE; - private static final String RECORD_TITLE = DIVIDING_LINE + NEW_LINE +"-- Records of "+ NEW_LINE + DIVIDING_LINE; + private static final String RECORD_TITLE = DIVIDING_LINE + NEW_LINE +"-- Records of %s"+ NEW_LINE + DIVIDING_LINE; @Override @@ -209,7 +209,6 @@ public void exportTableData(Connection connection, String databaseName, String s asyncContext.write(insertSql); valueList.clear(); } - asyncContext.write("\n"); }); } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java index de5e09c11..f6fb5d15f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncCall.java @@ -4,19 +4,6 @@ public interface AsyncCall { - - void setProgress(int progress); - - - void info(String message); - - - void error(String message); - - void update(Map map); - - void finish(); - } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 4222fca0d..db368995f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -1,59 +1,132 @@ package ai.chat2db.spi.model; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.util.ContextUtils; +import cn.hutool.core.io.FileUtil; +import lombok.extern.slf4j.Slf4j; +import java.io.File; import java.io.PrintWriter; -import java.util.function.Consumer; +import java.util.HashMap; +import java.util.Map; -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuperBuilder +@Slf4j public class AsyncContext { + private File writeFile; + protected PrintWriter writer; protected boolean containsData; protected AsyncCall call; + protected boolean finish; + + public File getWriteFile() { + return writeFile; + } + + + public AsyncContext(AsyncCall call, Context context, File writeFile, boolean containsData) { + this.call = call; + this.writeFile = writeFile; + this.progress = 5; + this.containsData = containsData; + createWriter(); + asyncCallBack(context); + } + + private void createWriter() { + if (writeFile != null) { + this.writer = FileUtil.getPrintWriter(writeFile, "UTF-8", false); + } + } + + + private void asyncCallBack(Context context) { + if (call != null && context != null) { + new Thread(() -> { + try { + ContextUtils.setContext(context); + int n = 1; + while (!finish) { + // 更新时间逐渐变长避免频繁更新 + Thread.sleep(2000 * n); + callUpdate(); + if (n < 300) { + n++; + } + } + } catch (Exception e) { + log.error("AsyncContext call error", e); + } finally { + ContextUtils.removeContext(); + } + }).start(); + } + } + + private void callUpdate() { + if (call == null) { + return; + } + Map map = new HashMap<>(); + map.put("progress", progress); + map.put("info", info.toString()); + map.put("error", error.toString()); + map.put("status", finish ? "FINISHED" : "RUNNING"); + if (progress == 100 && writeFile != null) { + map.put("downloadUrl", writeFile.getAbsolutePath()); + } + info = new StringBuffer(); + error = new StringBuffer(); + call.update(map); + } + + public boolean isContainsData() { + return containsData; + } + + public void setProgress(Integer progress) { - if (call != null) { - call.setProgress(progress); + if (progress == null) { + return; } + this.progress = progress; } public void info(String message) { - if (call != null) { - call.info(message); - } + info.append(message + "\n"); } public void error(String message) { - if (call != null) { - call.error(message); - } + error.append(message + "\n"); + info.append(message + "\n"); } public void finish() { + finish = true; + this.progress = 100; if (writer != null) { writer.flush(); writer.close(); } - if (call != null) { - call.finish(); - } + callUpdate(); } public void write(String message) { if (writer != null) { - writer.write(message); + writer.write(message + "\n"); } } + protected Integer progress; + + private StringBuffer info = new StringBuffer(); + + private StringBuffer error = new StringBuffer(); + + } From 0152ec4ead069d43c184b6e87845ad4f332bd74e Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 26 Jun 2024 22:54:02 +0800 Subject: [PATCH 25/73] export info --- .../java/ai/chat2db/plugin/mysql/MysqlDBManage.java | 10 ++++++---- .../main/java/ai/chat2db/spi/model/AsyncContext.java | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index c1a026108..abdf15833 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -14,11 +14,13 @@ import java.sql.SQLException; import java.util.Date; +import static cn.hutool.core.date.DatePattern.NORM_DATETIME_PATTERN; + @Slf4j public class MysqlDBManage extends DefaultDBManage implements DBManage { @Override public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - asyncContext.write(String.format(EXPORT_TITLE, DateUtil.formatDate(new Date()))); + asyncContext.write(String.format(EXPORT_TITLE, DateUtil.format(new Date(),NORM_DATETIME_PATTERN))); exportTables(connection, databaseName, schemaName, asyncContext); asyncContext.setProgress(50); exportViews(connection, databaseName, asyncContext); @@ -54,14 +56,14 @@ private void exportFunction(Connection connection, String functionName, AsyncCon } private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { - asyncContext.write("SET FOREIGN_KEY_CHECKS=0;\n"); + asyncContext.write("SET FOREIGN_KEY_CHECKS=0;"); try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } - asyncContext.write("SET FOREIGN_KEY_CHECKS=1;\n"); + asyncContext.write("SET FOREIGN_KEY_CHECKS=1;"); } @@ -70,7 +72,7 @@ public void exportTable(Connection connection, String databaseName, String schem try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { if (resultSet.next()) { StringBuilder sqlBuilder = new StringBuilder(); - asyncContext.write(String.format(EXPORT_TITLE, tableName)); + asyncContext.write(String.format(TABLE_TITLE, tableName)); sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") .append(resultSet.getString("Create Table")).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index db368995f..9c68430ec 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -2,14 +2,18 @@ import ai.chat2db.server.tools.common.model.Context; import ai.chat2db.server.tools.common.util.ContextUtils; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.PrintWriter; +import java.util.Date; import java.util.HashMap; import java.util.Map; +import static cn.hutool.core.date.DatePattern.NORM_DATETIME_PATTERN; + @Slf4j public class AsyncContext { @@ -35,6 +39,7 @@ public AsyncContext(AsyncCall call, Context context, File writeFile, boolean con this.containsData = containsData; createWriter(); asyncCallBack(context); + info("start:"+ DateUtil.format(new Date(),NORM_DATETIME_PATTERN)); } private void createWriter() { @@ -108,6 +113,7 @@ public void error(String message) { public void finish() { finish = true; this.progress = 100; + info("finish:"+ DateUtil.format(new Date(),NORM_DATETIME_PATTERN)); if (writer != null) { writer.flush(); writer.close(); From 57f0cfa2db13ce913b8dc76e3b60509649929298 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Thu, 27 Jun 2024 11:50:55 +0800 Subject: [PATCH 26/73] copy and delete Table --- .../plugin/clickhouse/ClickHouseDBManage.java | 9 +++++++ .../plugin/clickhouse/ClickHouseMetaData.java | 15 +++++------ .../ai/chat2db/plugin/db2/DB2DBManage.java | 10 +++++++ .../ai/chat2db/plugin/hive/HiveDBManage.java | 10 +++++++ .../plugin/kingbase/KingBaseDBManage.java | 12 +++++++++ .../chat2db/plugin/oracle/OracleDBManage.java | 11 ++++++++ .../plugin/postgresql/PostgreSQLDBManage.java | 11 ++++++++ .../plugin/sqlserver/SqlServerDBManage.java | 11 ++++++++ .../main/java/ai/chat2db/spi/DBManage.java | 23 ++++++++++++++++ .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 27 +++++++++++++++---- 10 files changed, 125 insertions(+), 14 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java index d05b760b2..1af8b6390 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseDBManage.java @@ -96,4 +96,13 @@ public void dropTable(Connection connection, String databaseName, String schemaN } + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = "CREATE TABLE " + newTableName + " AS " + tableName + ""; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + if(copyData){ + sql = "INSERT INTO " + newTableName + " SELECT * FROM " + tableName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index b13ce6f9f..6ebdd82b4 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -81,7 +81,7 @@ public List databases(Connection connection) { @Override public String tableDDL(Connection connection, @NotEmpty String databaseName, String schemaName, @NotEmpty String tableName) { - String sql = "SHOW CREATE TABLE " + format(databaseName) + "." + String sql = "SHOW CREATE TABLE " + format(schemaName) + "." + format(tableName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { if (resultSet.next()) { @@ -112,7 +112,7 @@ public Function function(Connection connection, @NotEmpty String databaseName, S @Override public List triggers(Connection connection, String databaseName, String schemaName) { List triggers = new ArrayList<>(); - String sql = String.format(TRIGGER_SQL_LIST, databaseName); + String sql = String.format(TRIGGER_SQL_LIST, schemaName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { while (resultSet.next()) { Trigger trigger = new Trigger(); @@ -145,7 +145,7 @@ public Trigger trigger(Connection connection, @NotEmpty String databaseName, Str @Override public Procedure procedure(Connection connection, @NotEmpty String databaseName, String schemaName, String procedureName) { - String sql = String.format(ROUTINES_SQL, "PROCEDURE", databaseName, procedureName); + String sql = String.format(ROUTINES_SQL, "PROCEDURE", schemaName, procedureName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { Procedure procedure = new Procedure(); procedure.setDatabaseName(databaseName); @@ -235,7 +235,7 @@ public Table view(Connection connection, String databaseName, String schemaName, @Override public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { StringBuilder queryBuf = new StringBuilder("SHOW INDEX FROM "); - queryBuf.append("`").append(tableName).append("`"); + queryBuf.append("`").append(schemaName).append("`"); queryBuf.append(" FROM "); queryBuf.append("`").append(databaseName).append("`"); return SQLExecutor.getInstance().execute(connection, queryBuf.toString(), resultSet -> { @@ -298,11 +298,8 @@ public TableMeta getTableMeta(String databaseName, String schemaName, String tab @Override public String getMetaDataName(String... names) { - return Arrays.stream(names) - .skip(1) // 跳过第一个名称 - .filter(StringUtils::isNotBlank) - .map(name -> "`" + name + "`") - .collect(Collectors.joining(".")); + return Arrays.stream(names).filter(name -> StringUtils.isNotBlank(name)).map(name -> "`" + name + "`").collect(Collectors.joining(".")); + } diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java index 39114b593..5f0791701 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/DB2DBManage.java @@ -111,4 +111,14 @@ public void dropTable(Connection connection, String databaseName, String schemaN String sql = "DROP TABLE " + tableName; SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } + + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = "CREATE TABLE " + newTableName + " LIKE " + tableName + " INCLUDING INDEXES"; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + if(copyData){ + sql = "INSERT INTO " + newTableName + " SELECT * FROM " + tableName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveDBManage.java b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveDBManage.java index cb3d7a3f3..92e1bbb69 100644 --- a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/HiveDBManage.java @@ -28,4 +28,14 @@ public void dropTable(Connection connection, String databaseName, String schemaN String sql = "drop table if exists " +tableName; SQLExecutor.getInstance().execute(connection,sql, resultSet -> null); } + + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = "CREATE TABLE " + newTableName + "LIKE " + tableName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + if(copyData){ + sql = "INSERT INTO " + newTableName + " SELECT * FROM " + tableName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/KingBaseDBManage.java b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/KingBaseDBManage.java index 076d7214c..42db21141 100644 --- a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/KingBaseDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/KingBaseDBManage.java @@ -1,6 +1,7 @@ package ai.chat2db.plugin.kingbase; import java.sql.Connection; +import java.sql.SQLException; import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; @@ -64,4 +65,15 @@ public void dropTable(Connection connection, String databaseName, String schemaN String sql = "drop table if exists " +tableName; SQLExecutor.getInstance().execute(connection,sql, resultSet -> null); } + + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = ""; + if(copyData){ + sql = "CREATE TABLE " + newTableName + " AS TABLE " + tableName + " WITH DATA"; + }else { + sql = "CREATE TABLE " + newTableName + " AS TABLE " + tableName + " WITH NO DATA"; + } + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index 3554eb7ad..c2d9af998 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -184,4 +184,15 @@ public void connectDatabase(Connection connection, String database) { } } + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = ""; + if(copyData){ + sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName; + }else { + sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName + " WHERE 1=0"; + } + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java index d09c5c03b..d8411b91a 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLDBManage.java @@ -159,4 +159,15 @@ public void dropTable(Connection connection, String databaseName, String schemaN SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = ""; + if(copyData){ + sql = "CREATE TABLE " + newTableName + " AS TABLE " + tableName + " WITH DATA"; + }else { + sql = "CREATE TABLE " + newTableName + " AS TABLE " + tableName + " WITH NO DATA"; + } + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 5b3901ee5..36331e708 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -203,4 +203,15 @@ public void connectDatabase(Connection connection, String database) { throw new RuntimeException(e); } } + + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = ""; + if(copyData){ + sql = "SELECT * INTO " + newTableName + " FROM " + tableName; + }else { + sql = "SELECT * INTO " + newTableName + " FROM " + tableName + " WHERE 1=0"; + } + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java index 343e4b52c..e2410c46a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/DBManage.java @@ -140,4 +140,27 @@ void dropProcedure(Connection connection, @NotEmpty String databaseName, String * @return */ void exportTable(Connection connection, String databaseName, String schemaName,String tableName,AsyncContext asyncContext) throws SQLException; + + + /** + * truncate table + * @param connection + * @param databaseName + * @param schemaName + * @param tableName + * @throws SQLException + */ + void truncateTable(Connection connection, String databaseName, String schemaName, String tableName)throws SQLException; + + + /** + * copy table + * + * @param databaseName + * @param schemaName + * @param tableName + * @param newTableName + * @return + */ + void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException; } \ No newline at end of file diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 3c41ee03c..da2bd198f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -38,15 +38,15 @@ public class DefaultDBManage implements DBManage { protected static final String TABLE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Table structure for table %s" + NEW_LINE + DIVIDING_LINE; - protected static final String VIEW_TITLE = DIVIDING_LINE + NEW_LINE +"-- View structure for view %s"+ NEW_LINE + DIVIDING_LINE; + protected static final String VIEW_TITLE = DIVIDING_LINE + NEW_LINE + "-- View structure for view %s" + NEW_LINE + DIVIDING_LINE; - protected static final String FUNCTION_TITLE = DIVIDING_LINE + NEW_LINE +"-- Function structure for function %s"+ NEW_LINE + DIVIDING_LINE; + protected static final String FUNCTION_TITLE = DIVIDING_LINE + NEW_LINE + "-- Function structure for function %s" + NEW_LINE + DIVIDING_LINE; - protected static final String TRIGGER_TITLE = DIVIDING_LINE + NEW_LINE + "-- Trigger structure for trigger %s"+ NEW_LINE + DIVIDING_LINE; + protected static final String TRIGGER_TITLE = DIVIDING_LINE + NEW_LINE + "-- Trigger structure for trigger %s" + NEW_LINE + DIVIDING_LINE; - protected static final String PROCEDURE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Procedure structure for procedure %s"+ NEW_LINE + DIVIDING_LINE; + protected static final String PROCEDURE_TITLE = DIVIDING_LINE + NEW_LINE + "-- Procedure structure for procedure %s" + NEW_LINE + DIVIDING_LINE; - private static final String RECORD_TITLE = DIVIDING_LINE + NEW_LINE +"-- Records of %s"+ NEW_LINE + DIVIDING_LINE; + private static final String RECORD_TITLE = DIVIDING_LINE + NEW_LINE + "-- Records of %s" + NEW_LINE + DIVIDING_LINE; @Override @@ -179,6 +179,23 @@ public void exportTable(Connection connection, String databaseName, String schem } + @Override + public void truncateTable(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { + String sql = "TRUNCATE TABLE " + tableName ; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + + @Override + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + String sql = ""; + if(copyData){ + sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName; + }else { + sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName + " WHERE 1=0"; + } + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + // public void exportDatabaseData(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { // exportTableData(connection, databaseName, schemaName, tableName, asyncContext); // } From 3bf8fab69d072f6522aec1dd25bc1cd472732e07 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Thu, 27 Jun 2024 21:35:12 +0800 Subject: [PATCH 27/73] copy and delete Table --- .../java/ai/chat2db/plugin/mysql/MysqlDBManage.java | 6 ++++-- .../java/ai/chat2db/spi/model/AsyncContext.java | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java index abdf15833..56566a0a7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/MysqlDBManage.java @@ -48,8 +48,10 @@ private void exportFunction(Connection connection, String functionName, AsyncCon if (resultSet.next()) { asyncContext.write(String.format(FUNCTION_TITLE, functionName)); StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n") - .append(resultSet.getString("Create Function")).append(";").append("\n"); + sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n"); + + sqlBuilder.append("delimiter ;;").append("\n").append(resultSet.getString("Create Function")).append(";;") + .append("\n").append("delimiter ;").append("\n\n"); asyncContext.write(sqlBuilder.toString()); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 9c68430ec..210a7c34d 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -39,7 +39,7 @@ public AsyncContext(AsyncCall call, Context context, File writeFile, boolean con this.containsData = containsData; createWriter(); asyncCallBack(context); - info("start:"+ DateUtil.format(new Date(),NORM_DATETIME_PATTERN)); + info("start:" + DateUtil.format(new Date(), NORM_DATETIME_PATTERN)); } private void createWriter() { @@ -57,8 +57,8 @@ private void asyncCallBack(Context context) { int n = 1; while (!finish) { // 更新时间逐渐变长避免频繁更新 - Thread.sleep(2000 * n); callUpdate(); + Thread.sleep(2000 * n); if (n < 300) { n++; } @@ -110,10 +110,17 @@ public void error(String message) { info.append(message + "\n"); } + public void stop(){ + this.finish = true; + } + public void finish() { finish = true; this.progress = 100; - info("finish:"+ DateUtil.format(new Date(),NORM_DATETIME_PATTERN)); + info("finish:" + DateUtil.format(new Date(), NORM_DATETIME_PATTERN)); + if (writeFile != null) { + info("file:" + writeFile.getAbsolutePath()); + } if (writer != null) { writer.flush(); writer.close(); From 0d80a36a6fc11c6b360387c7a998f602ed2a0ab4 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 28 Jun 2024 22:23:48 +0800 Subject: [PATCH 28/73] optimize MysqlValueProcessor & OracleValueProcessor --- .../factory/MysqlValueProcessorFactory.java | 1 - .../mysql/value/sub/MysqlBinaryProcessor.java | 17 +- .../mysql/value/sub/MysqlBitProcessor.java | 24 +- .../value/sub/MysqlDecimalProcessor.java | 4 +- .../value/sub/MysqlGeometryProcessor.java | 15 +- .../mysql/value/sub/MysqlTextProcessor.java | 21 +- .../value/sub/MysqlTimestampProcessor.java | 18 +- .../value/sub/MysqlVarBinaryProcessor.java | 21 +- .../mysql/value/sub/MysqlYearProcessor.java | 62 ----- .../value/template/MysqlDmlValueTemplate.java | 15 +- .../java/ai/chat2db/plugin/oracle/oracle.json | 8 +- .../oracle/value/OracleValueProcessor.java | 2 +- .../factory/OracleValueProcessorFactory.java | 14 +- .../oracle/value/sub/OracleBlobProcessor.java | 15 +- .../oracle/value/sub/OracleClobProcessor.java | 8 +- .../oracle/value/sub/OracleDateProcessor.java | 13 +- .../value/sub/OracleIntervalDSProcessor.java | 9 +- .../value/sub/OracleIntervalYMProcessor.java | 10 +- .../value/sub/OracleNumberProcessor.java | 4 +- .../value/sub/OracleRawValueProcessor.java | 31 +++ .../value/sub/OracleTimeStampProcessor.java | 5 +- .../value/sub/OracleTimeStampTZProcessor.java | 10 +- .../value/sub/OracleXmlValueProcessor.java | 30 ++ .../template/OracleDmlValueTemplate.java | 36 ++- .../chat2db/spi/jdbc/BaseValueProcessor.java | 23 +- .../ai/chat2db/spi/model/JDBCDataValue.java | 259 +++++++++++------- .../ai/chat2db/spi/model/SQLDataValue.java | 2 +- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 24 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 11 +- 29 files changed, 390 insertions(+), 322 deletions(-) delete mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index fe9601c47..94367e9a2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -45,7 +45,6 @@ public class MysqlValueProcessorFactory { Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mysqlTimestampProcessor), //others Map.entry(MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor()), - Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor()), Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) ); diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index d35edc708..7e41d21d7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -1,5 +1,6 @@ package ai.chat2db.plugin.mysql.value.sub; +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -12,18 +13,28 @@ public class MysqlBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getBlobHexString(); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + return value; + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + byte[] bytes = dataValue.getBytes(); + if (bytes.length == 1) { + if (bytes[0] >= 32 && bytes[0] <= 126) { + return new String(bytes); + } + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java index 238796e42..568d93df1 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Objects; +import java.util.function.Function; /** * @author: zgq @@ -23,23 +24,16 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - int precision = dataValue.getPrecision(); - byte[] bytes = dataValue.getBytes(); - if (precision == 1) { - //bit(1) [1 -> true] [0 -> false] - if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { - return String.valueOf(dataValue.getBoolean()); - } - // tinyint(1) - return String.valueOf(dataValue.getInt()); - } - //bit(m) m: 1~64 - return EasyStringUtils.getBitString(bytes, precision); + return getValue(dataValue, s -> s); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getValue(dataValue, this::wrap); + } + + private String getValue(JDBCDataValue dataValue, Function function) { int precision = dataValue.getPrecision(); byte[] bytes = dataValue.getBytes(); if (precision == 1) { @@ -51,7 +45,7 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { return String.valueOf(dataValue.getInt()); } //bit(m) m: 2~64 - return wrap(EasyStringUtils.getBitString(bytes, precision)); + return function.apply(EasyStringUtils.getBitString(bytes, precision)); } public String getString(String value) { @@ -65,10 +59,10 @@ public String getString(String value) { if (StringUtils.isBlank(value)) { return "NULL"; } - return wrap(value); + return MysqlDmlValueTemplate.wrapBit(value); } private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.BIT_TEMPLATE, value); + return MysqlDmlValueTemplate.wrapBit(value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java index 781fc8639..2ee2f3d48 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java @@ -12,7 +12,7 @@ public class MysqlDecimalProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @@ -24,6 +24,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return convertJDBCValueByType(dataValue); + return dataValue.getBigDecimalString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java index 76f05f98b..452d09765 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java @@ -6,6 +6,8 @@ import ai.chat2db.spi.model.SQLDataValue; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.WKBReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -17,9 +19,11 @@ public class MysqlGeometryProcessor extends DefaultValueProcessor { + private static final Logger log = LoggerFactory.getLogger(MysqlGeometryProcessor.class); + @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return MysqlDmlValueTemplate.wrapGeometry(dataValue.getValue()); } @Override @@ -75,18 +79,15 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } return dbGeometry != null ? dbGeometry.toString() : null; } catch (Exception e) { - return super.getJdbcValue(dataValue); + log.warn("Error converting database geometry", e); + return dataValue.getStringValue(); } } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue)); - } - - private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE, value); + return MysqlDmlValueTemplate.wrapGeometry(convertJDBCValueByType(dataValue)); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java index 2aabdf2f3..ff1ed2db0 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java @@ -4,11 +4,8 @@ import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; -import java.util.function.Function; - /** * @author: zgq * @date: 2024年06月05日 0:11 @@ -19,31 +16,19 @@ public class MysqlTextProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return getClobString(dataValue, super::convertJDBCValueByType); + return dataValue.getClobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(getClobString(dataValue, super::convertJDBCValueStrByType)); + return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); } - private String getClobString(JDBCDataValue dataValue, Function function) { - try { - return dataValue.getClobString(); - } catch (Exception e) { - log.warn("convertJDBCValue error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return function.apply(dataValue); - } - } - private String wrap(String value) { - return EasyStringUtils.escapeAndQuoteString(value); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 93f0897fe..560068d93 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -1,6 +1,6 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -13,28 +13,18 @@ public class MysqlTimestampProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return isValidTimestamp(dataValue) ? new String(dataValue.getBytes()) : "0000-00-00 00:00:00"; + return new String(dataValue.getBytes()); } - protected boolean isValidTimestamp(JDBCDataValue data) { - byte[] buffer = data.getBytes(); - String stringValue = new String(buffer); - return stringValue.length() <= 0 - || stringValue.charAt(0) != '0' - || !"0000-00-00".equals(stringValue) - && !"0000-00-00 00:00:00".equals(stringValue) - && !"00000000000000".equals(stringValue) - && !"0".equals(stringValue); - } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return String.format(MysqlDmlValueTemplate.COMMON_TEMPLATE, convertJDBCValueByType(dataValue)); + return EasyStringUtils.quoteString(new String(dataValue.getBytes())); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index bff921153..71b689aca 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,10 +1,9 @@ package ai.chat2db.plugin.mysql.value.sub; +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; -import ch.qos.logback.core.model.processor.DefaultProcessor; import lombok.extern.slf4j.Slf4j; /** @@ -16,26 +15,24 @@ public class MysqlVarBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - // TODO: insert file - return super.convertSQLValueByType(dataValue); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + return value; + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - try { - return dataValue.getBlobString(); - } catch (Exception e) { - log.warn("convertJDBCValue error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return super.convertJDBCValueByType(dataValue); - } + return dataValue.getBlobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java deleted file mode 100644 index db9f78311..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java +++ /dev/null @@ -1,62 +0,0 @@ -package ai.chat2db.plugin.mysql.value.sub; - -import ai.chat2db.spi.jdbc.DefaultValueProcessor; -import ai.chat2db.spi.model.JDBCDataValue; -import ai.chat2db.spi.model.SQLDataValue; - -import java.sql.Date; -import java.util.Calendar; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年06月01日 12:57 - */ -public class MysqlYearProcessor extends DefaultValueProcessor { - - @Override - public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); - } - - - @Override - public String convertJDBCValueByType(JDBCDataValue dataValue) { - Date date = dataValue.getDate(); - if (!isValidYear(dataValue)) { - return "0000"; - } - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int year = calendar.get(Calendar.YEAR); - String yStr; - String yZerosPadding = "0000"; - if (year < 1000) { - yStr = "" + year; - yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; - } else { - yStr = "" + year; - } - return yStr; - } - - private boolean isValidYear(JDBCDataValue data) { - byte[] buffer = data.getBytes(); - String stringValue = new String(buffer); - return stringValue.length() <= 0 - || stringValue.charAt(0) != '0' - || !"0000-00-00".equals(stringValue) - && !"0000-00-00 00:00:00".equals(stringValue) - && !"00000000000000".equals(stringValue) - && !"0".equals(stringValue) - && !"00000000".equals(stringValue) - && !"0000".equals(stringValue); - } - - - @Override - public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getJdbcValue(dataValue); - } -} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java index f3547f62b..c43f82be9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -6,7 +6,20 @@ */ public class MysqlDmlValueTemplate { - public static final String COMMON_TEMPLATE = "'%s'"; public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; public static final String BIT_TEMPLATE = "b'%s'"; + public static final String HEX_TEMPLATE = "0x%s"; + + + public static String wrapGeometry(String value) { + return String.format(GEOMETRY_TEMPLATE, value); + } + + public static String wrapBit(String value) { + return String.format(BIT_TEMPLATE, value); + } + + public static String wrapHex(String value) { + return String.format(HEX_TEMPLATE, value); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json index adb455ed2..a16db8c1c 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json @@ -8,10 +8,12 @@ "custom": false, "defaultDriver": true, "downloadJdbcDriverUrls": [ - "https://cdn.chat2db-ai.com/lib/ojdbc8-19.3.0.0.jar", - "https://cdn.chat2db-ai.com/lib/orai18n-19.3.0.0.jar" + "https://cdn.chat2db-ai.com/lib/ojdbc11-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/orai18n-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/xmlparserv2-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/xdb-21.5.0.0.jar" ], - "jdbcDriver": "ojdbc8-19.3.0.0.jar,orai18n-19.3.0.0.jar", + "jdbcDriver": "ojdbc11-21.5.0.0.jar,orai18n-21.5.0.0.jar,xmlparserv2-21.5.0.0.jar,xdb-21.5.0.0.jar", "jdbcDriverClass": "oracle.jdbc.driver.OracleDriver" } ], diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 9f1c2e277..495333b41 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -21,7 +21,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); + return OracleValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 2a545ce00..2cd16da35 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -9,7 +9,7 @@ /** * @author: zgq * @date: 2024年06月03日 23:21 - */ // TODO: 1.空间数据类型 2.XML数据类型 3.动态类型数据 4.ANSI、DB2 和 SQL/DS 数据 + */ // TODO: 1.空间数据类型 2.动态类型数据 public class OracleValueProcessorFactory { private static final Map PROCESSOR_MAP; @@ -17,6 +17,8 @@ public class OracleValueProcessorFactory { static { OracleClobProcessor oracleClobProcessor = new OracleClobProcessor(); OracleTimeStampProcessor oracleTimeStampProcessor = new OracleTimeStampProcessor(); + OracleBlobProcessor oracleBlobProcessor = new OracleBlobProcessor(); + OracleRawValueProcessor oracleRawValueProcessor = new OracleRawValueProcessor(); PROCESSOR_MAP = Map.ofEntries( //clob Map.entry(OracleColumnTypeEnum.CLOB.name(), oracleClobProcessor), @@ -34,14 +36,18 @@ public class OracleValueProcessorFactory { //number Map.entry(OracleColumnTypeEnum.NUMBER.name(), new OracleNumberProcessor()), //blob - Map.entry(OracleColumnTypeEnum.BLOB.name(), new OracleBlobProcessor()) + Map.entry(OracleColumnTypeEnum.BLOB.name(), oracleBlobProcessor), + //raw + Map.entry(OracleColumnTypeEnum.RAW.name(), oracleRawValueProcessor), + Map.entry(OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName(), oracleRawValueProcessor), + //xml + Map.entry("SYS.XMLTYPE", new OracleXmlValueProcessor()) ); } public static DefaultValueProcessor getValueProcessor(String type) { - DefaultValueProcessor processor = PROCESSOR_MAP.get(type); - return processor == null ? new DefaultValueProcessor() : processor; + return PROCESSOR_MAP.getOrDefault(type, new DefaultValueProcessor()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java index e246fdb88..63b3b54e9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -1,9 +1,9 @@ package ai.chat2db.plugin.oracle.value.sub; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; /** @@ -15,24 +15,17 @@ public class OracleBlobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getBlobHexString(); + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - try { - return dataValue.getBlobString(); - } catch (Exception e) { - log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return super.convertJDBCValueByType(dataValue); - } - + return dataValue.getBlobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java index 1409efd57..9773e6b6e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java @@ -13,7 +13,7 @@ public class OracleClobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); } @@ -25,10 +25,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(dataValue.getClobString()); - } - - private String wrap(String value) { - return EasyStringUtils.escapeAndQuoteString(value); + return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java index c807fa670..7507e6bdc 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java @@ -21,7 +21,7 @@ public class OracleDateProcessor extends DefaultValueProcessor { */ @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return OracleDmlValueTemplate.wrapDate(dataValue.getValue()); } /** @@ -41,16 +41,9 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } - /** - * @param dataValue - * @return - */ + @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue)); - } - - private String wrap(String value) { - return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + return OracleDmlValueTemplate.wrapDate(convertJDBCValueByType(dataValue)); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java index 11b619b01..717cea5d8 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java @@ -14,22 +14,19 @@ public class OracleIntervalDSProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); + return OracleDmlValueTemplate.wrapIntervalDayToSecond(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return super.convertJDBCValueByType(dataValue); + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); + return OracleDmlValueTemplate.wrapIntervalDayToSecond(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); } - private String wrap(String value, int precision, int scale) { - return String.format(OracleDmlValueTemplate.INTEGER_DAY_TO_SECOND_TEMPLATE, value, precision, scale); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java index f11ca212f..1ae0d5577 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java @@ -15,22 +15,18 @@ public class OracleIntervalYMProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue(), dataValue.getPrecision()); + return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getValue(), dataValue.getPrecision()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return super.convertJDBCValueByType(dataValue); + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision()); - } - - public String wrap(String value, int precision) { - return String.format(OracleDmlValueTemplate.INTEGER_YEAR_TO_MONTH_TEMPLATE, value, precision); + return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getStringValue(), dataValue.getPrecision()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java index 987bdf6d0..2c9952dd7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java @@ -14,7 +14,7 @@ public class OracleNumberProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @@ -26,6 +26,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return convertJDBCValueByType(dataValue); + return dataValue.getBigDecimalString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java new file mode 100644 index 000000000..db6517808 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java @@ -0,0 +1,31 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月28日 下午1:59 + */ +public class OracleRawValueProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getBinaryDataString(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java index 44a9d8dd7..eb7ec36e9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java @@ -24,7 +24,6 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - // TODO: datagrip对timestampLTZ的处理是不受时区影响的，但其实这个字段就是为了可以协同时区问题的，有待商讨 Timestamp timestamp = dataValue.getTimestamp(); int scale = dataValue.getScale(); LocalDateTime localDateTime = timestamp.toLocalDateTime(); @@ -45,8 +44,8 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { private String wrap(String value, int scale) { if (scale == 0) { - return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + return OracleDmlValueTemplate.wrapDate(value); } - return String.format(OracleDmlValueTemplate.TIMESTAMP_TEMPLATE, value, scale); + return OracleDmlValueTemplate.wrapTimestamp(value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java index 7a2d95744..bd0ceee48 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -21,21 +21,19 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - // TODO: return:2024-06-05 17:32:52.849 +8:00 but it actually is 2024-06-05 17:32:52.849000 +8:00 - return super.convertJDBCValueByType(dataValue); - + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); + return wrap(dataValue.getStringValue(), dataValue.getScale()); } private String wrap(String value, int scale) { if (scale == 0) { - return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, value); + return OracleDmlValueTemplate.wrapTimestampTzWithOutNanos(value); } - return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_TEMPLATE, value, scale); + return OracleDmlValueTemplate.wrapTimestampTz(value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java new file mode 100644 index 000000000..2d8214cbb --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java @@ -0,0 +1,30 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月21日 12:55 + */ +public class OracleXmlValueProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return OracleDmlValueTemplate.wrapXml(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getStringValue(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return OracleDmlValueTemplate.wrapXml(dataValue.getString()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index e9080ae7d..a0cb8f1be 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -13,6 +13,38 @@ public class OracleDmlValueTemplate { public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d TZH:TZM')"; public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')"; - public static final String INTEGER_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; - public static final String INTEGER_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; + public static final String INTERVAL_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; + public static final String INTERVAL_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; + + public static final String XML_TEMPLATE = "XMLType('%s')"; + + + public static String wrapDate(String date) { + return String.format(DATE_TEMPLATE, date); + } + + public static String wrapTimestamp(String timestamp, int scale) { + return String.format(TIMESTAMP_TEMPLATE, timestamp, scale); + } + + public static String wrapTimestampTz(String timestamp, int scale) { + return String.format(TIMESTAMP_TZ_TEMPLATE, timestamp, scale); + } + + public static String wrapTimestampTzWithOutNanos(String timestamp) { + return String.format(TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, timestamp); + } + + public static String wrapIntervalYearToMonth(String year, int precision) { + return String.format(INTERVAL_YEAR_TO_MONTH_TEMPLATE, year, precision); + } + + public static String wrapIntervalDayToSecond(String day, int precision, int scale) { + return String.format(INTERVAL_DAY_TO_SECOND_TEMPLATE, day, precision, scale); + } + + public static String wrapXml(String xml) { + return String.format(XML_TEMPLATE, xml); + } + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index a0ca67500..4c209a3c3 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; import org.apache.commons.lang3.StringUtils; import java.util.Objects; @@ -27,12 +28,19 @@ public String getSqlValueString(SQLDataValue dataValue) { @Override public String getJdbcValue(JDBCDataValue dataValue) { Object value = dataValue.getObject(); - if (Objects.isNull(dataValue.getObject())) { + if (Objects.isNull(value)) { + // mysql -> [date]->0000:00:00 + if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return stringValue; + } + } return null; } - if (value instanceof String emptySry) { - if (StringUtils.isBlank(emptySry)) { - return emptySry; + if (value instanceof String emptyStr) { + if (StringUtils.isBlank(emptyStr)) { + return emptyStr; } } return convertJDBCValueByType(dataValue); @@ -43,6 +51,13 @@ public String getJdbcValue(JDBCDataValue dataValue) { public String getJdbcValueString(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { + // mysql -> [date]->0000:00:00 + if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return EasyStringUtils.escapeAndQuoteString(stringValue); + } + } return "NULL"; } if (value instanceof String stringValue) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 420cf1ba0..4e3d0a330 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -7,14 +7,16 @@ import lombok.Getter; import org.apache.tika.Tika; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.Objects; @@ -25,6 +27,7 @@ @Data @AllArgsConstructor public class JDBCDataValue { + private static final Logger log = LoggerFactory.getLogger(JDBCDataValue.class); private ResultSet resultSet; private ResultSetMetaData metaData; private int columnIndex; @@ -34,6 +37,7 @@ public Object getObject() { try { return resultSet.getObject(columnIndex); } catch (Exception e) { + log.warn("Failed to retrieve object from database", e); try { return resultSet.getString(columnIndex); } catch (SQLException ex) { @@ -71,7 +75,6 @@ public int getScale() { } public int getInt() { - return ResultSetUtils.getInt(resultSet, columnIndex); } @@ -91,154 +94,202 @@ public Blob getBlob() { return ResultSetUtils.getBlob(resultSet, columnIndex); } + public String getBlobHexString() { + return BaseEncoding.base16().encode(getBytes()); + } + + public BigDecimal getBigDecimal() { + return ResultSetUtils.getBigDecimal(resultSet, columnIndex); + } + + public String getBigDecimalString() { + BigDecimal bigDecimal = getBigDecimal(); + return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); + } + + public String getBlobString() { Blob blob = getBlob(); - LOBInfo blobInfo = getBlobInfo(blob); - String unit = blobInfo.getUnit(); - if (blobInfo.getSize() == 0) { - return ""; - } - try (InputStream binaryStream = blob.getBinaryStream()) { - Tika tika = new Tika(); - String contentType = tika.detect(binaryStream); - FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); - if (Objects.isNull(fileTypeEnum)) { - if (limitSize && isBigSize(unit)) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } - return getBlobHexString(); - } - switch (fileTypeEnum) { - case IMAGE: - if (limitSize) { - try (InputStream imageStream = blob.getBinaryStream()) { - BufferedImage bufferedImage = ImageIO.read(imageStream); - return String.format("[%s] %dx%d JPEG image %d %s", - getType(), bufferedImage.getWidth(), - bufferedImage.getHeight(), blobInfo.getSize(), unit); - } - } else { - return getBlobHexString(); - } - case STRING: - if (isBigSize(unit) && limitSize) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } else { - return new String(binaryStream.readAllBytes()); - } - default: - if (isBigSize(unit) && limitSize) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } - return getBlobHexString(); - } + long length = blob.length(); + return converterBinaryData(length, binaryStream); } catch (SQLException | IOException e) { - throw new RuntimeException(e); + log.warn("Error while reading binary stream", e); + return getString(); } } - private LOBInfo getBlobInfo(Blob blob) { - try { - long size = blob.length(); - return getLobInfo(size); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - public String getClobString() { Clob clob = getClob(); - LOBInfo cLobInfo = getCLobInfo(clob); - int size = cLobInfo.getSize(); - if (size == 0) { - return ""; - } - String unit = cLobInfo.getUnit(); - if (limitSize && isBigSize(unit)) { - return String.format("[%s] %d %s", getType(), size, unit); - } - StringBuilder builder = new StringBuilder(size); - String line; try (BufferedReader reader = new BufferedReader(clob.getCharacterStream())) { + long length = clob.length(); + LOBInfo cLobInfo = getLobInfo(length); + double size = cLobInfo.getSize(); + if (size == 0) { + return ""; + } + String unit = cLobInfo.getUnit(); + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %s", getType(), cLobInfo); + } + StringBuilder builder = new StringBuilder((int) (Math.ceil(size))); + String line; + while ((line = reader.readLine()) != null) { builder.append(line).append("\n"); } + return builder.toString(); } catch (IOException | SQLException e) { - throw new RuntimeException(e); + log.warn("Error while reading clob stream", e); + return getStringValue(); } - return builder.toString(); } - private boolean isBigSize(String unit) { - return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); + private String handleImageType(InputStream imageStream, LOBInfo lobInfo) { + if (limitSize) { + try { + BufferedImage bufferedImage = ImageIO.read(imageStream); + return String.format("[%s] %dx%d JPEG image %s", getType(), bufferedImage.getWidth(), bufferedImage.getHeight(), lobInfo); + } catch (IOException e) { + log.warn("Error while reading image stream", e); + return getStringValue(); + } + } else { + return "0x" + getBlobHexString(); + } } - public LOBInfo getCLobInfo(Clob clob) { - try { - long size = clob.length(); - return getLobInfo(size); - } catch (SQLException e) { - throw new RuntimeException(e); + private String handleStringType(InputStream binaryStream, LOBInfo lobInfo) throws IOException { + if (isBigSize(lobInfo.getUnit()) && limitSize) { + return String.format("[%s] %s", getType(), lobInfo); + } else { + return new String(binaryStream.readAllBytes()); } + } + private boolean isBigSize(String unit) { + return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); } + @NotNull private LOBInfo getLobInfo(long size) { if (size == 0) { return new LOBInfo(LobUnit.B.unit, 0); } - return calculateSizeAndUnit(size); + return new LOBInfo(size); } - @NotNull - private LOBInfo calculateSizeAndUnit(long size) { - if (size > LobUnit.G.size) { - return new LOBInfo(LobUnit.G.unit, (int) (size / LobUnit.G.size)); - } else if (size > LobUnit.M.size) { - return new LOBInfo(LobUnit.M.unit, (int) (size / LobUnit.M.size)); - } else if (size > LobUnit.K.size) { - return new LOBInfo(LobUnit.K.unit, (int) (size / LobUnit.K.size)); - } else { - return new LOBInfo(LobUnit.B.unit, (int) size); - } + public String getStringValue() { + return ResultSetUtils.getStringValue(resultSet, columnIndex); } - public String getBlobHexString() { - return "0x" + BaseEncoding.base16().encode(getBytes()); - } + public String getBinaryDataString() { + InputStream binaryStream = null; + try { + binaryStream = getBinaryStream(); + // 检查流是否支持 mark 操作，不支持则用 BufferedInputStream 包装 + if (!binaryStream.markSupported()) { + binaryStream = new BufferedInputStream(binaryStream); + } - public BigDecimal getBigDecimal() { - return ResultSetUtils.getBigDecimal(resultSet, columnIndex); - } + binaryStream.mark(Integer.MAX_VALUE); - public String getBigDecimalString() { - BigDecimal bigDecimal = getBigDecimal(); - return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); + long size = 0; + byte[] buffer = new byte[8192]; // 缓冲区 + int bytesRead; + while ((bytesRead = binaryStream.read(buffer)) != -1) { + size += bytesRead; + } + binaryStream.reset(); // 重置流到标记的位置 + return converterBinaryData(size, binaryStream); + } catch (SQLException | IOException e) { + log.warn("Error while reading binary stream", e); + return getStringValue(); + } finally { + // 关闭流 + if (binaryStream != null) { + try { + binaryStream.close(); + } catch (IOException e) { + log.warn("Error while closing binary stream", e); + } + } + } } - @Data - @AllArgsConstructor - public static class LOBInfo { - private String unit; - private int size; + private String converterBinaryData(long size, InputStream binaryStream) throws IOException, SQLException { + LOBInfo lobInfo = getLobInfo(size); + String unit = lobInfo.unit; + if (size == 0) { + return ""; + } + Tika tika = new Tika(); + String contentType = tika.detect(binaryStream); + FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); + if (Objects.isNull(fileTypeEnum)) { + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %s", getType(), lobInfo); + } + return "0x" + getBlobHexString(); + } + + return switch (fileTypeEnum) { + case IMAGE -> handleImageType(binaryStream, lobInfo); + case STRING -> handleStringType(binaryStream, lobInfo); + default -> ""; + }; } + @Getter public enum LobUnit { - B("B", 1), - K("KB", 1024), - M("MB", 1024 * 1024), - G("GB", 1024 * 1024 * 1024); + B("B", 1L), + K("KB", 1024L), + M("MB", 1024L * 1024L), + G("GB", 1024L * 1024L * 1024L); + private final String unit; - private final int size; + private final long size; - LobUnit(String unit, int size) { + LobUnit(String unit, long size) { this.unit = unit; this.size = size; } + + } + + @Getter + public static class LOBInfo { + private final String unit; + private final double size; + + public LOBInfo(String unit, double size) { + this.unit = unit; + this.size = size; + } + + public LOBInfo(long size) { + if (size >= LobUnit.G.size) { + this.unit = LobUnit.G.unit; + this.size = (double) size / LobUnit.G.size; + } else if (size >= LobUnit.M.size) { + this.unit = LobUnit.M.unit; + this.size = (double) size / LobUnit.M.size; + } else if (size >= LobUnit.K.size) { + this.unit = LobUnit.K.unit; + this.size = (double) size / LobUnit.K.size; + } else { + this.unit = LobUnit.B.unit; + this.size = (double) size; + } + } + + @Override + public String toString() { + return String.format("%.2f %s", size, unit); + } } @Getter diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java index f285bc759..05e95583a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java @@ -25,6 +25,6 @@ public int getScale() { } public String getBlobHexString() { - return "0x" + BaseEncoding.base16().encode(value.getBytes()); + return BaseEncoding.base16().encode(value.getBytes()); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index b947eb079..0503cfa7b 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -1,10 +1,5 @@ package ai.chat2db.spi.sql; -import java.sql.*; -import java.util.*; -import java.util.function.Consumer; -import java.util.stream.Collectors; - import ai.chat2db.server.tools.base.constant.EasyToolsConstant; import ai.chat2db.server.tools.base.enums.DataSourceTypeEnum; import ai.chat2db.server.tools.base.excption.BusinessException; @@ -15,25 +10,26 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DataTypeEnum; import ai.chat2db.spi.enums.SqlTypeEnum; -import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.*; import ai.chat2db.spi.util.JdbcUtils; import ai.chat2db.spi.util.ResultSetUtils; import ai.chat2db.spi.util.SqlUtils; import cn.hutool.core.date.TimeInterval; - import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.google.common.collect.Lists; - import lombok.extern.slf4j.Slf4j; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; +import java.sql.*; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + /** * Dbhub unified database connection management * @@ -248,7 +244,7 @@ private List> generateDataList(ResultSet rs, int col, int chat2dbAu continue; } ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); - row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, false))); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, limitRowSize))); } if (count != null && count > 0 && rowCount++ >= count) { break; @@ -386,7 +382,7 @@ public List columns(Connection connection, String databaseName, Str tableName, String columnName) { try (ResultSet resultSet = connection.getMetaData().getColumns(databaseName, schemaName, tableName, - columnName)) { + columnName)) { return ResultSetUtils.toObjectList(resultSet, TableColumn.class); } catch (Exception e) { throw new RuntimeException(e); @@ -405,8 +401,8 @@ public List columns(Connection connection, String databaseName, Str public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { List tableIndices = Lists.newArrayList(); try (ResultSet resultSet = connection.getMetaData().getIndexInfo(databaseName, schemaName, tableName, - false, - false)) { + false, + false)) { List tableIndexColumns = ResultSetUtils.toObjectList(resultSet, TableIndexColumn.class); tableIndexColumns.stream().filter(c -> c.getIndexName() != null).collect( Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() @@ -582,7 +578,7 @@ private void addRowNumber(ExecuteResult executeResult, int pageNo, int pageSize) Header rowNumberHeader = Header.builder() .name(I18nUtils.getMessage("sqlResult.rowNumber")) .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER - .getCode()).build(); + .getCode()).build(); executeResult.setHeaderList(EasyCollectionUtils.union(Arrays.asList(rowNumberHeader), headers)); // Add row number diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index e26d496ea..9c872dc04 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -1,8 +1,6 @@ package ai.chat2db.spi.util; -import ai.chat2db.server.tools.common.util.I18nUtils; -import cn.hutool.core.io.unit.DataSizeUtil; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -29,7 +27,6 @@ public class ResultSetUtils { - public static List getRsHeader(ResultSet rs) { try { ResultSetMetaData resultSetMetaData = rs.getMetaData(); @@ -262,4 +259,12 @@ public static BigDecimal getBigDecimal(ResultSet resultSet, int columnIndex) { throw new RuntimeException(e); } } + + public static String getStringValue(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getString(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file From c4955bc72bcafc2fb8069b121c413cdf78c8ba64 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 28 Jun 2024 22:27:05 +0800 Subject: [PATCH 29/73] Revert "optimize MysqlValueProcessor & OracleValueProcessor" This reverts commit 0d80a36a6fc11c6b360387c7a998f602ed2a0ab4. --- .../factory/MysqlValueProcessorFactory.java | 1 + .../mysql/value/sub/MysqlBinaryProcessor.java | 17 +- .../mysql/value/sub/MysqlBitProcessor.java | 24 +- .../value/sub/MysqlDecimalProcessor.java | 4 +- .../value/sub/MysqlGeometryProcessor.java | 15 +- .../mysql/value/sub/MysqlTextProcessor.java | 21 +- .../value/sub/MysqlTimestampProcessor.java | 18 +- .../value/sub/MysqlVarBinaryProcessor.java | 21 +- .../mysql/value/sub/MysqlYearProcessor.java | 62 +++++ .../value/template/MysqlDmlValueTemplate.java | 15 +- .../java/ai/chat2db/plugin/oracle/oracle.json | 8 +- .../oracle/value/OracleValueProcessor.java | 2 +- .../factory/OracleValueProcessorFactory.java | 14 +- .../oracle/value/sub/OracleBlobProcessor.java | 15 +- .../oracle/value/sub/OracleClobProcessor.java | 8 +- .../oracle/value/sub/OracleDateProcessor.java | 13 +- .../value/sub/OracleIntervalDSProcessor.java | 9 +- .../value/sub/OracleIntervalYMProcessor.java | 10 +- .../value/sub/OracleNumberProcessor.java | 4 +- .../value/sub/OracleRawValueProcessor.java | 31 --- .../value/sub/OracleTimeStampProcessor.java | 5 +- .../value/sub/OracleTimeStampTZProcessor.java | 10 +- .../value/sub/OracleXmlValueProcessor.java | 30 -- .../template/OracleDmlValueTemplate.java | 36 +-- .../chat2db/spi/jdbc/BaseValueProcessor.java | 23 +- .../ai/chat2db/spi/model/JDBCDataValue.java | 259 +++++++----------- .../ai/chat2db/spi/model/SQLDataValue.java | 2 +- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 24 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 11 +- 29 files changed, 322 insertions(+), 390 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java delete mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java delete mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index 94367e9a2..fe9601c47 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -45,6 +45,7 @@ public class MysqlValueProcessorFactory { Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mysqlTimestampProcessor), //others Map.entry(MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor()), + Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor()), Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) ); diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index 7e41d21d7..d35edc708 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -1,6 +1,5 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -13,28 +12,18 @@ public class MysqlBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - String value = dataValue.getValue(); - if (value.startsWith("0x")) { - return value; - } - return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - byte[] bytes = dataValue.getBytes(); - if (bytes.length == 1) { - if (bytes[0] >= 32 && bytes[0] <= 126) { - return new String(bytes); - } - } - return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java index 568d93df1..238796e42 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java @@ -8,7 +8,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.Objects; -import java.util.function.Function; /** * @author: zgq @@ -24,16 +23,23 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return getValue(dataValue, s -> s); + int precision = dataValue.getPrecision(); + byte[] bytes = dataValue.getBytes(); + if (precision == 1) { + //bit(1) [1 -> true] [0 -> false] + if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { + return String.valueOf(dataValue.getBoolean()); + } + // tinyint(1) + return String.valueOf(dataValue.getInt()); + } + //bit(m) m: 1~64 + return EasyStringUtils.getBitString(bytes, precision); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getValue(dataValue, this::wrap); - } - - private String getValue(JDBCDataValue dataValue, Function function) { int precision = dataValue.getPrecision(); byte[] bytes = dataValue.getBytes(); if (precision == 1) { @@ -45,7 +51,7 @@ private String getValue(JDBCDataValue dataValue, Function functi return String.valueOf(dataValue.getInt()); } //bit(m) m: 2~64 - return function.apply(EasyStringUtils.getBitString(bytes, precision)); + return wrap(EasyStringUtils.getBitString(bytes, precision)); } public String getString(String value) { @@ -59,10 +65,10 @@ public String getString(String value) { if (StringUtils.isBlank(value)) { return "NULL"; } - return MysqlDmlValueTemplate.wrapBit(value); + return wrap(value); } private String wrap(String value) { - return MysqlDmlValueTemplate.wrapBit(value); + return String.format(MysqlDmlValueTemplate.BIT_TEMPLATE, value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java index 2ee2f3d48..781fc8639 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java @@ -12,7 +12,7 @@ public class MysqlDecimalProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); + return super.convertSQLValueByType(dataValue); } @@ -24,6 +24,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBigDecimalString(); + return convertJDBCValueByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java index 452d09765..76f05f98b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java @@ -6,8 +6,6 @@ import ai.chat2db.spi.model.SQLDataValue; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.WKBReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -19,11 +17,9 @@ public class MysqlGeometryProcessor extends DefaultValueProcessor { - private static final Logger log = LoggerFactory.getLogger(MysqlGeometryProcessor.class); - @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return MysqlDmlValueTemplate.wrapGeometry(dataValue.getValue()); + return wrap(dataValue.getValue()); } @Override @@ -79,15 +75,18 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } return dbGeometry != null ? dbGeometry.toString() : null; } catch (Exception e) { - log.warn("Error converting database geometry", e); - return dataValue.getStringValue(); + return super.getJdbcValue(dataValue); } } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return MysqlDmlValueTemplate.wrapGeometry(convertJDBCValueByType(dataValue)); + return wrap(convertJDBCValueByType(dataValue)); + } + + private String wrap(String value) { + return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE, value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java index ff1ed2db0..2aabdf2f3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java @@ -4,8 +4,11 @@ import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; +import java.util.function.Function; + /** * @author: zgq * @date: 2024年06月05日 0:11 @@ -16,19 +19,31 @@ public class MysqlTextProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); + return wrap(dataValue.getValue()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getClobString(); + return getClobString(dataValue, super::convertJDBCValueByType); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); + return wrap(getClobString(dataValue, super::convertJDBCValueStrByType)); } + private String getClobString(JDBCDataValue dataValue, Function function) { + try { + return dataValue.getClobString(); + } catch (Exception e) { + log.warn("convertJDBCValue error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); + return function.apply(dataValue); + } + } + private String wrap(String value) { + return EasyStringUtils.escapeAndQuoteString(value); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 560068d93..93f0897fe 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -1,6 +1,6 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -13,18 +13,28 @@ public class MysqlTimestampProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); + return super.convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return new String(dataValue.getBytes()); + return isValidTimestamp(dataValue) ? new String(dataValue.getBytes()) : "0000-00-00 00:00:00"; } + protected boolean isValidTimestamp(JDBCDataValue data) { + byte[] buffer = data.getBytes(); + String stringValue = new String(buffer); + return stringValue.length() <= 0 + || stringValue.charAt(0) != '0' + || !"0000-00-00".equals(stringValue) + && !"0000-00-00 00:00:00".equals(stringValue) + && !"00000000000000".equals(stringValue) + && !"0".equals(stringValue); + } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.quoteString(new String(dataValue.getBytes())); + return String.format(MysqlDmlValueTemplate.COMMON_TEMPLATE, convertJDBCValueByType(dataValue)); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index 71b689aca..bff921153 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,9 +1,10 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import ch.qos.logback.core.model.processor.DefaultProcessor; import lombok.extern.slf4j.Slf4j; /** @@ -15,24 +16,26 @@ public class MysqlVarBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - String value = dataValue.getValue(); - if (value.startsWith("0x")) { - return value; - } - return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); + // TODO: insert file + return super.convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getBlobString(); + try { + return dataValue.getBlobString(); + } catch (Exception e) { + log.warn("convertJDBCValue error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); + return super.convertJDBCValueByType(dataValue); + } } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } - } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java new file mode 100644 index 000000000..db9f78311 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java @@ -0,0 +1,62 @@ +package ai.chat2db.plugin.mysql.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Date; +import java.util.Calendar; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月01日 12:57 + */ +public class MysqlYearProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + Date date = dataValue.getDate(); + if (!isValidYear(dataValue)) { + return "0000"; + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int year = calendar.get(Calendar.YEAR); + String yStr; + String yZerosPadding = "0000"; + if (year < 1000) { + yStr = "" + year; + yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; + } else { + yStr = "" + year; + } + return yStr; + } + + private boolean isValidYear(JDBCDataValue data) { + byte[] buffer = data.getBytes(); + String stringValue = new String(buffer); + return stringValue.length() <= 0 + || stringValue.charAt(0) != '0' + || !"0000-00-00".equals(stringValue) + && !"0000-00-00 00:00:00".equals(stringValue) + && !"00000000000000".equals(stringValue) + && !"0".equals(stringValue) + && !"00000000".equals(stringValue) + && !"0000".equals(stringValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getJdbcValue(dataValue); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java index c43f82be9..f3547f62b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -6,20 +6,7 @@ */ public class MysqlDmlValueTemplate { + public static final String COMMON_TEMPLATE = "'%s'"; public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; public static final String BIT_TEMPLATE = "b'%s'"; - public static final String HEX_TEMPLATE = "0x%s"; - - - public static String wrapGeometry(String value) { - return String.format(GEOMETRY_TEMPLATE, value); - } - - public static String wrapBit(String value) { - return String.format(BIT_TEMPLATE, value); - } - - public static String wrapHex(String value) { - return String.format(HEX_TEMPLATE, value); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json index a16db8c1c..adb455ed2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json @@ -8,12 +8,10 @@ "custom": false, "defaultDriver": true, "downloadJdbcDriverUrls": [ - "https://cdn.chat2db-ai.com/lib/ojdbc11-21.5.0.0.jar", - "https://cdn.chat2db-ai.com/lib/orai18n-21.5.0.0.jar", - "https://cdn.chat2db-ai.com/lib/xmlparserv2-21.5.0.0.jar", - "https://cdn.chat2db-ai.com/lib/xdb-21.5.0.0.jar" + "https://cdn.chat2db-ai.com/lib/ojdbc8-19.3.0.0.jar", + "https://cdn.chat2db-ai.com/lib/orai18n-19.3.0.0.jar" ], - "jdbcDriver": "ojdbc11-21.5.0.0.jar,orai18n-21.5.0.0.jar,xmlparserv2-21.5.0.0.jar,xdb-21.5.0.0.jar", + "jdbcDriver": "ojdbc8-19.3.0.0.jar,orai18n-19.3.0.0.jar", "jdbcDriverClass": "oracle.jdbc.driver.OracleDriver" } ], diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 495333b41..9f1c2e277 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -21,7 +21,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return OracleValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); + return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 2cd16da35..2a545ce00 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -9,7 +9,7 @@ /** * @author: zgq * @date: 2024年06月03日 23:21 - */ // TODO: 1.空间数据类型 2.动态类型数据 + */ // TODO: 1.空间数据类型 2.XML数据类型 3.动态类型数据 4.ANSI、DB2 和 SQL/DS 数据 public class OracleValueProcessorFactory { private static final Map PROCESSOR_MAP; @@ -17,8 +17,6 @@ public class OracleValueProcessorFactory { static { OracleClobProcessor oracleClobProcessor = new OracleClobProcessor(); OracleTimeStampProcessor oracleTimeStampProcessor = new OracleTimeStampProcessor(); - OracleBlobProcessor oracleBlobProcessor = new OracleBlobProcessor(); - OracleRawValueProcessor oracleRawValueProcessor = new OracleRawValueProcessor(); PROCESSOR_MAP = Map.ofEntries( //clob Map.entry(OracleColumnTypeEnum.CLOB.name(), oracleClobProcessor), @@ -36,18 +34,14 @@ public class OracleValueProcessorFactory { //number Map.entry(OracleColumnTypeEnum.NUMBER.name(), new OracleNumberProcessor()), //blob - Map.entry(OracleColumnTypeEnum.BLOB.name(), oracleBlobProcessor), - //raw - Map.entry(OracleColumnTypeEnum.RAW.name(), oracleRawValueProcessor), - Map.entry(OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName(), oracleRawValueProcessor), - //xml - Map.entry("SYS.XMLTYPE", new OracleXmlValueProcessor()) + Map.entry(OracleColumnTypeEnum.BLOB.name(), new OracleBlobProcessor()) ); } public static DefaultValueProcessor getValueProcessor(String type) { - return PROCESSOR_MAP.getOrDefault(type, new DefaultValueProcessor()); + DefaultValueProcessor processor = PROCESSOR_MAP.get(type); + return processor == null ? new DefaultValueProcessor() : processor; } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java index 63b3b54e9..e246fdb88 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -1,9 +1,9 @@ package ai.chat2db.plugin.oracle.value.sub; -import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; /** @@ -15,17 +15,24 @@ public class OracleBlobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getBlobString(); + try { + return dataValue.getBlobString(); + } catch (Exception e) { + log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", + Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); + return super.convertJDBCValueByType(dataValue); + } + } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + return dataValue.getBlobHexString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java index 9773e6b6e..1409efd57 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java @@ -13,7 +13,7 @@ public class OracleClobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); + return wrap(dataValue.getValue()); } @@ -25,6 +25,10 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); + return wrap(dataValue.getClobString()); + } + + private String wrap(String value) { + return EasyStringUtils.escapeAndQuoteString(value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java index 7507e6bdc..c807fa670 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java @@ -21,7 +21,7 @@ public class OracleDateProcessor extends DefaultValueProcessor { */ @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return OracleDmlValueTemplate.wrapDate(dataValue.getValue()); + return wrap(dataValue.getValue()); } /** @@ -41,9 +41,16 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } - + /** + * @param dataValue + * @return + */ @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleDmlValueTemplate.wrapDate(convertJDBCValueByType(dataValue)); + return wrap(convertJDBCValueByType(dataValue)); + } + + private String wrap(String value) { + return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java index 717cea5d8..11b619b01 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java @@ -14,19 +14,22 @@ public class OracleIntervalDSProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return OracleDmlValueTemplate.wrapIntervalDayToSecond(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); + return wrap(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getStringValue(); + return super.convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleDmlValueTemplate.wrapIntervalDayToSecond(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); + return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); } + private String wrap(String value, int precision, int scale) { + return String.format(OracleDmlValueTemplate.INTEGER_DAY_TO_SECOND_TEMPLATE, value, precision, scale); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java index 1ae0d5577..f11ca212f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java @@ -15,18 +15,22 @@ public class OracleIntervalYMProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getValue(), dataValue.getPrecision()); + return wrap(dataValue.getValue(), dataValue.getPrecision()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getStringValue(); + return super.convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getStringValue(), dataValue.getPrecision()); + return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision()); + } + + public String wrap(String value, int precision) { + return String.format(OracleDmlValueTemplate.INTEGER_YEAR_TO_MONTH_TEMPLATE, value, precision); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java index 2c9952dd7..987bdf6d0 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java @@ -14,7 +14,7 @@ public class OracleNumberProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); + return super.convertSQLValueByType(dataValue); } @@ -26,6 +26,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBigDecimalString(); + return convertJDBCValueByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java deleted file mode 100644 index db6517808..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java +++ /dev/null @@ -1,31 +0,0 @@ -package ai.chat2db.plugin.oracle.value.sub; - -import ai.chat2db.server.tools.common.util.EasyStringUtils; -import ai.chat2db.spi.jdbc.DefaultValueProcessor; -import ai.chat2db.spi.model.JDBCDataValue; -import ai.chat2db.spi.model.SQLDataValue; - -/** - * @author: zgq - * @date: 2024年06月28日 下午1:59 - */ -public class OracleRawValueProcessor extends DefaultValueProcessor { - - - @Override - public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getValue()); - } - - - @Override - public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getBinaryDataString(); - } - - - @Override - public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getBlobHexString()); - } -} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java index eb7ec36e9..44a9d8dd7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java @@ -24,6 +24,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { + // TODO: datagrip对timestampLTZ的处理是不受时区影响的，但其实这个字段就是为了可以协同时区问题的，有待商讨 Timestamp timestamp = dataValue.getTimestamp(); int scale = dataValue.getScale(); LocalDateTime localDateTime = timestamp.toLocalDateTime(); @@ -44,8 +45,8 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { private String wrap(String value, int scale) { if (scale == 0) { - return OracleDmlValueTemplate.wrapDate(value); + return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); } - return OracleDmlValueTemplate.wrapTimestamp(value, scale); + return String.format(OracleDmlValueTemplate.TIMESTAMP_TEMPLATE, value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java index bd0ceee48..7a2d95744 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -21,19 +21,21 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getStringValue(); + // TODO: return:2024-06-05 17:32:52.849 +8:00 but it actually is 2024-06-05 17:32:52.849000 +8:00 + return super.convertJDBCValueByType(dataValue); + } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(dataValue.getStringValue(), dataValue.getScale()); + return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); } private String wrap(String value, int scale) { if (scale == 0) { - return OracleDmlValueTemplate.wrapTimestampTzWithOutNanos(value); + return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, value); } - return OracleDmlValueTemplate.wrapTimestampTz(value, scale); + return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_TEMPLATE, value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java deleted file mode 100644 index 2d8214cbb..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java +++ /dev/null @@ -1,30 +0,0 @@ -package ai.chat2db.plugin.oracle.value.sub; - -import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; -import ai.chat2db.spi.jdbc.DefaultValueProcessor; -import ai.chat2db.spi.model.JDBCDataValue; -import ai.chat2db.spi.model.SQLDataValue; - -/** - * @author: zgq - * @date: 2024年06月21日 12:55 - */ -public class OracleXmlValueProcessor extends DefaultValueProcessor { - - @Override - public String convertSQLValueByType(SQLDataValue dataValue) { - return OracleDmlValueTemplate.wrapXml(dataValue.getValue()); - } - - - @Override - public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getStringValue(); - } - - - @Override - public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleDmlValueTemplate.wrapXml(dataValue.getString()); - } -} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index a0cb8f1be..e9080ae7d 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -13,38 +13,6 @@ public class OracleDmlValueTemplate { public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d TZH:TZM')"; public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')"; - public static final String INTERVAL_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; - public static final String INTERVAL_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; - - public static final String XML_TEMPLATE = "XMLType('%s')"; - - - public static String wrapDate(String date) { - return String.format(DATE_TEMPLATE, date); - } - - public static String wrapTimestamp(String timestamp, int scale) { - return String.format(TIMESTAMP_TEMPLATE, timestamp, scale); - } - - public static String wrapTimestampTz(String timestamp, int scale) { - return String.format(TIMESTAMP_TZ_TEMPLATE, timestamp, scale); - } - - public static String wrapTimestampTzWithOutNanos(String timestamp) { - return String.format(TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, timestamp); - } - - public static String wrapIntervalYearToMonth(String year, int precision) { - return String.format(INTERVAL_YEAR_TO_MONTH_TEMPLATE, year, precision); - } - - public static String wrapIntervalDayToSecond(String day, int precision, int scale) { - return String.format(INTERVAL_DAY_TO_SECOND_TEMPLATE, day, precision, scale); - } - - public static String wrapXml(String xml) { - return String.format(XML_TEMPLATE, xml); - } - + public static final String INTEGER_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; + public static final String INTEGER_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index 4c209a3c3..a0ca67500 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -4,7 +4,6 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import org.apache.commons.lang3.StringUtils; import java.util.Objects; @@ -28,19 +27,12 @@ public String getSqlValueString(SQLDataValue dataValue) { @Override public String getJdbcValue(JDBCDataValue dataValue) { Object value = dataValue.getObject(); - if (Objects.isNull(value)) { - // mysql -> [date]->0000:00:00 - if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { - String stringValue = dataValue.getStringValue(); - if (Objects.nonNull(stringValue)) { - return stringValue; - } - } + if (Objects.isNull(dataValue.getObject())) { return null; } - if (value instanceof String emptyStr) { - if (StringUtils.isBlank(emptyStr)) { - return emptyStr; + if (value instanceof String emptySry) { + if (StringUtils.isBlank(emptySry)) { + return emptySry; } } return convertJDBCValueByType(dataValue); @@ -51,13 +43,6 @@ public String getJdbcValue(JDBCDataValue dataValue) { public String getJdbcValueString(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { - // mysql -> [date]->0000:00:00 - if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { - String stringValue = dataValue.getStringValue(); - if (Objects.nonNull(stringValue)) { - return EasyStringUtils.escapeAndQuoteString(stringValue); - } - } return "NULL"; } if (value instanceof String stringValue) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 4e3d0a330..420cf1ba0 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -7,16 +7,14 @@ import lombok.Getter; import org.apache.tika.Tika; import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.Objects; @@ -27,7 +25,6 @@ @Data @AllArgsConstructor public class JDBCDataValue { - private static final Logger log = LoggerFactory.getLogger(JDBCDataValue.class); private ResultSet resultSet; private ResultSetMetaData metaData; private int columnIndex; @@ -37,7 +34,6 @@ public Object getObject() { try { return resultSet.getObject(columnIndex); } catch (Exception e) { - log.warn("Failed to retrieve object from database", e); try { return resultSet.getString(columnIndex); } catch (SQLException ex) { @@ -75,6 +71,7 @@ public int getScale() { } public int getInt() { + return ResultSetUtils.getInt(resultSet, columnIndex); } @@ -94,202 +91,154 @@ public Blob getBlob() { return ResultSetUtils.getBlob(resultSet, columnIndex); } - public String getBlobHexString() { - return BaseEncoding.base16().encode(getBytes()); - } - - public BigDecimal getBigDecimal() { - return ResultSetUtils.getBigDecimal(resultSet, columnIndex); - } - - public String getBigDecimalString() { - BigDecimal bigDecimal = getBigDecimal(); - return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); - } - - public String getBlobString() { Blob blob = getBlob(); + LOBInfo blobInfo = getBlobInfo(blob); + String unit = blobInfo.getUnit(); + if (blobInfo.getSize() == 0) { + return ""; + } + try (InputStream binaryStream = blob.getBinaryStream()) { - long length = blob.length(); - return converterBinaryData(length, binaryStream); + Tika tika = new Tika(); + String contentType = tika.detect(binaryStream); + FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); + if (Objects.isNull(fileTypeEnum)) { + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); + } + return getBlobHexString(); + } + switch (fileTypeEnum) { + case IMAGE: + if (limitSize) { + try (InputStream imageStream = blob.getBinaryStream()) { + BufferedImage bufferedImage = ImageIO.read(imageStream); + return String.format("[%s] %dx%d JPEG image %d %s", + getType(), bufferedImage.getWidth(), + bufferedImage.getHeight(), blobInfo.getSize(), unit); + } + } else { + return getBlobHexString(); + } + case STRING: + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); + } else { + return new String(binaryStream.readAllBytes()); + } + default: + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); + } + return getBlobHexString(); + } } catch (SQLException | IOException e) { - log.warn("Error while reading binary stream", e); - return getString(); + throw new RuntimeException(e); } } + private LOBInfo getBlobInfo(Blob blob) { + try { + long size = blob.length(); + return getLobInfo(size); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + public String getClobString() { Clob clob = getClob(); + LOBInfo cLobInfo = getCLobInfo(clob); + int size = cLobInfo.getSize(); + if (size == 0) { + return ""; + } + String unit = cLobInfo.getUnit(); + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %d %s", getType(), size, unit); + } + StringBuilder builder = new StringBuilder(size); + String line; try (BufferedReader reader = new BufferedReader(clob.getCharacterStream())) { - long length = clob.length(); - LOBInfo cLobInfo = getLobInfo(length); - double size = cLobInfo.getSize(); - if (size == 0) { - return ""; - } - String unit = cLobInfo.getUnit(); - if (limitSize && isBigSize(unit)) { - return String.format("[%s] %s", getType(), cLobInfo); - } - StringBuilder builder = new StringBuilder((int) (Math.ceil(size))); - String line; - while ((line = reader.readLine()) != null) { builder.append(line).append("\n"); } - return builder.toString(); } catch (IOException | SQLException e) { - log.warn("Error while reading clob stream", e); - return getStringValue(); + throw new RuntimeException(e); } + return builder.toString(); } - private String handleImageType(InputStream imageStream, LOBInfo lobInfo) { - if (limitSize) { - try { - BufferedImage bufferedImage = ImageIO.read(imageStream); - return String.format("[%s] %dx%d JPEG image %s", getType(), bufferedImage.getWidth(), bufferedImage.getHeight(), lobInfo); - } catch (IOException e) { - log.warn("Error while reading image stream", e); - return getStringValue(); - } - } else { - return "0x" + getBlobHexString(); - } + private boolean isBigSize(String unit) { + return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); } - private String handleStringType(InputStream binaryStream, LOBInfo lobInfo) throws IOException { - if (isBigSize(lobInfo.getUnit()) && limitSize) { - return String.format("[%s] %s", getType(), lobInfo); - } else { - return new String(binaryStream.readAllBytes()); + public LOBInfo getCLobInfo(Clob clob) { + try { + long size = clob.length(); + return getLobInfo(size); + } catch (SQLException e) { + throw new RuntimeException(e); } - } - private boolean isBigSize(String unit) { - return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); } - @NotNull private LOBInfo getLobInfo(long size) { if (size == 0) { return new LOBInfo(LobUnit.B.unit, 0); } - return new LOBInfo(size); + return calculateSizeAndUnit(size); } - public String getStringValue() { - return ResultSetUtils.getStringValue(resultSet, columnIndex); - } - - public String getBinaryDataString() { - InputStream binaryStream = null; - try { - binaryStream = getBinaryStream(); - // 检查流是否支持 mark 操作，不支持则用 BufferedInputStream 包装 - if (!binaryStream.markSupported()) { - binaryStream = new BufferedInputStream(binaryStream); - } - - binaryStream.mark(Integer.MAX_VALUE); - - long size = 0; - byte[] buffer = new byte[8192]; // 缓冲区 - int bytesRead; - while ((bytesRead = binaryStream.read(buffer)) != -1) { - size += bytesRead; - } - binaryStream.reset(); // 重置流到标记的位置 - return converterBinaryData(size, binaryStream); - } catch (SQLException | IOException e) { - log.warn("Error while reading binary stream", e); - return getStringValue(); - } finally { - // 关闭流 - if (binaryStream != null) { - try { - binaryStream.close(); - } catch (IOException e) { - log.warn("Error while closing binary stream", e); - } - } + @NotNull + private LOBInfo calculateSizeAndUnit(long size) { + if (size > LobUnit.G.size) { + return new LOBInfo(LobUnit.G.unit, (int) (size / LobUnit.G.size)); + } else if (size > LobUnit.M.size) { + return new LOBInfo(LobUnit.M.unit, (int) (size / LobUnit.M.size)); + } else if (size > LobUnit.K.size) { + return new LOBInfo(LobUnit.K.unit, (int) (size / LobUnit.K.size)); + } else { + return new LOBInfo(LobUnit.B.unit, (int) size); } } - private String converterBinaryData(long size, InputStream binaryStream) throws IOException, SQLException { - LOBInfo lobInfo = getLobInfo(size); - String unit = lobInfo.unit; - if (size == 0) { - return ""; - } - Tika tika = new Tika(); - String contentType = tika.detect(binaryStream); - FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); - if (Objects.isNull(fileTypeEnum)) { - if (isBigSize(unit) && limitSize) { - return String.format("[%s] %s", getType(), lobInfo); - } - return "0x" + getBlobHexString(); - } - - return switch (fileTypeEnum) { - case IMAGE -> handleImageType(binaryStream, lobInfo); - case STRING -> handleStringType(binaryStream, lobInfo); - default -> ""; - }; + public String getBlobHexString() { + return "0x" + BaseEncoding.base16().encode(getBytes()); } + public BigDecimal getBigDecimal() { + return ResultSetUtils.getBigDecimal(resultSet, columnIndex); + } - @Getter - public enum LobUnit { - B("B", 1L), - K("KB", 1024L), - M("MB", 1024L * 1024L), - G("GB", 1024L * 1024L * 1024L); - - private final String unit; - private final long size; - - LobUnit(String unit, long size) { - this.unit = unit; - this.size = size; - } + public String getBigDecimalString() { + BigDecimal bigDecimal = getBigDecimal(); + return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); + } + @Data + @AllArgsConstructor + public static class LOBInfo { + private String unit; + private int size; } @Getter - public static class LOBInfo { + public enum LobUnit { + B("B", 1), + K("KB", 1024), + M("MB", 1024 * 1024), + G("GB", 1024 * 1024 * 1024); private final String unit; - private final double size; + private final int size; - public LOBInfo(String unit, double size) { + LobUnit(String unit, int size) { this.unit = unit; this.size = size; } - - public LOBInfo(long size) { - if (size >= LobUnit.G.size) { - this.unit = LobUnit.G.unit; - this.size = (double) size / LobUnit.G.size; - } else if (size >= LobUnit.M.size) { - this.unit = LobUnit.M.unit; - this.size = (double) size / LobUnit.M.size; - } else if (size >= LobUnit.K.size) { - this.unit = LobUnit.K.unit; - this.size = (double) size / LobUnit.K.size; - } else { - this.unit = LobUnit.B.unit; - this.size = (double) size; - } - } - - @Override - public String toString() { - return String.format("%.2f %s", size, unit); - } } @Getter diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java index 05e95583a..f285bc759 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java @@ -25,6 +25,6 @@ public int getScale() { } public String getBlobHexString() { - return BaseEncoding.base16().encode(value.getBytes()); + return "0x" + BaseEncoding.base16().encode(value.getBytes()); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 0503cfa7b..b947eb079 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -1,5 +1,10 @@ package ai.chat2db.spi.sql; +import java.sql.*; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + import ai.chat2db.server.tools.base.constant.EasyToolsConstant; import ai.chat2db.server.tools.base.enums.DataSourceTypeEnum; import ai.chat2db.server.tools.base.excption.BusinessException; @@ -10,26 +15,25 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DataTypeEnum; import ai.chat2db.spi.enums.SqlTypeEnum; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.*; import ai.chat2db.spi.util.JdbcUtils; import ai.chat2db.spi.util.ResultSetUtils; import ai.chat2db.spi.util.SqlUtils; import cn.hutool.core.date.TimeInterval; + import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.google.common.collect.Lists; + import lombok.extern.slf4j.Slf4j; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; -import java.sql.*; -import java.util.*; -import java.util.function.Consumer; -import java.util.stream.Collectors; - /** * Dbhub unified database connection management * @@ -244,7 +248,7 @@ private List> generateDataList(ResultSet rs, int col, int chat2dbAu continue; } ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); - row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, limitRowSize))); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, false))); } if (count != null && count > 0 && rowCount++ >= count) { break; @@ -382,7 +386,7 @@ public List columns(Connection connection, String databaseName, Str tableName, String columnName) { try (ResultSet resultSet = connection.getMetaData().getColumns(databaseName, schemaName, tableName, - columnName)) { + columnName)) { return ResultSetUtils.toObjectList(resultSet, TableColumn.class); } catch (Exception e) { throw new RuntimeException(e); @@ -401,8 +405,8 @@ public List columns(Connection connection, String databaseName, Str public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { List tableIndices = Lists.newArrayList(); try (ResultSet resultSet = connection.getMetaData().getIndexInfo(databaseName, schemaName, tableName, - false, - false)) { + false, + false)) { List tableIndexColumns = ResultSetUtils.toObjectList(resultSet, TableIndexColumn.class); tableIndexColumns.stream().filter(c -> c.getIndexName() != null).collect( Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() @@ -578,7 +582,7 @@ private void addRowNumber(ExecuteResult executeResult, int pageNo, int pageSize) Header rowNumberHeader = Header.builder() .name(I18nUtils.getMessage("sqlResult.rowNumber")) .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER - .getCode()).build(); + .getCode()).build(); executeResult.setHeaderList(EasyCollectionUtils.union(Arrays.asList(rowNumberHeader), headers)); // Add row number diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index 9c872dc04..e26d496ea 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -1,6 +1,8 @@ package ai.chat2db.spi.util; +import ai.chat2db.server.tools.common.util.I18nUtils; +import cn.hutool.core.io.unit.DataSizeUtil; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,6 +29,7 @@ public class ResultSetUtils { + public static List getRsHeader(ResultSet rs) { try { ResultSetMetaData resultSetMetaData = rs.getMetaData(); @@ -259,12 +262,4 @@ public static BigDecimal getBigDecimal(ResultSet resultSet, int columnIndex) { throw new RuntimeException(e); } } - - public static String getStringValue(ResultSet resultSet, int columnIndex) { - try { - return resultSet.getString(columnIndex); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } } \ No newline at end of file From 786dcd701d774b5b36158a57cd088c34f7f9de17 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 28 Jun 2024 22:28:02 +0800 Subject: [PATCH 30/73] Revert "Revert "optimize MysqlValueProcessor & OracleValueProcessor"" This reverts commit c4955bc72bcafc2fb8069b121c413cdf78c8ba64. --- .../factory/MysqlValueProcessorFactory.java | 1 - .../mysql/value/sub/MysqlBinaryProcessor.java | 17 +- .../mysql/value/sub/MysqlBitProcessor.java | 24 +- .../value/sub/MysqlDecimalProcessor.java | 4 +- .../value/sub/MysqlGeometryProcessor.java | 15 +- .../mysql/value/sub/MysqlTextProcessor.java | 21 +- .../value/sub/MysqlTimestampProcessor.java | 18 +- .../value/sub/MysqlVarBinaryProcessor.java | 21 +- .../mysql/value/sub/MysqlYearProcessor.java | 62 ----- .../value/template/MysqlDmlValueTemplate.java | 15 +- .../java/ai/chat2db/plugin/oracle/oracle.json | 8 +- .../oracle/value/OracleValueProcessor.java | 2 +- .../factory/OracleValueProcessorFactory.java | 14 +- .../oracle/value/sub/OracleBlobProcessor.java | 15 +- .../oracle/value/sub/OracleClobProcessor.java | 8 +- .../oracle/value/sub/OracleDateProcessor.java | 13 +- .../value/sub/OracleIntervalDSProcessor.java | 9 +- .../value/sub/OracleIntervalYMProcessor.java | 10 +- .../value/sub/OracleNumberProcessor.java | 4 +- .../value/sub/OracleRawValueProcessor.java | 31 +++ .../value/sub/OracleTimeStampProcessor.java | 5 +- .../value/sub/OracleTimeStampTZProcessor.java | 10 +- .../value/sub/OracleXmlValueProcessor.java | 30 ++ .../template/OracleDmlValueTemplate.java | 36 ++- .../chat2db/spi/jdbc/BaseValueProcessor.java | 23 +- .../ai/chat2db/spi/model/JDBCDataValue.java | 259 +++++++++++------- .../ai/chat2db/spi/model/SQLDataValue.java | 2 +- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 24 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 11 +- 29 files changed, 390 insertions(+), 322 deletions(-) delete mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index fe9601c47..94367e9a2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -45,7 +45,6 @@ public class MysqlValueProcessorFactory { Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mysqlTimestampProcessor), //others Map.entry(MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor()), - Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor()), Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) ); diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java index d35edc708..7e41d21d7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBinaryProcessor.java @@ -1,5 +1,6 @@ package ai.chat2db.plugin.mysql.value.sub; +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -12,18 +13,28 @@ public class MysqlBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getBlobHexString(); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + return value; + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + byte[] bytes = dataValue.getBytes(); + if (bytes.length == 1) { + if (bytes[0] >= 32 && bytes[0] <= 126) { + return new String(bytes); + } + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java index 238796e42..568d93df1 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlBitProcessor.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Objects; +import java.util.function.Function; /** * @author: zgq @@ -23,23 +24,16 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - int precision = dataValue.getPrecision(); - byte[] bytes = dataValue.getBytes(); - if (precision == 1) { - //bit(1) [1 -> true] [0 -> false] - if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { - return String.valueOf(dataValue.getBoolean()); - } - // tinyint(1) - return String.valueOf(dataValue.getInt()); - } - //bit(m) m: 1~64 - return EasyStringUtils.getBitString(bytes, precision); + return getValue(dataValue, s -> s); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getValue(dataValue, this::wrap); + } + + private String getValue(JDBCDataValue dataValue, Function function) { int precision = dataValue.getPrecision(); byte[] bytes = dataValue.getBytes(); if (precision == 1) { @@ -51,7 +45,7 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { return String.valueOf(dataValue.getInt()); } //bit(m) m: 2~64 - return wrap(EasyStringUtils.getBitString(bytes, precision)); + return function.apply(EasyStringUtils.getBitString(bytes, precision)); } public String getString(String value) { @@ -65,10 +59,10 @@ public String getString(String value) { if (StringUtils.isBlank(value)) { return "NULL"; } - return wrap(value); + return MysqlDmlValueTemplate.wrapBit(value); } private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.BIT_TEMPLATE, value); + return MysqlDmlValueTemplate.wrapBit(value); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java index 781fc8639..2ee2f3d48 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlDecimalProcessor.java @@ -12,7 +12,7 @@ public class MysqlDecimalProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @@ -24,6 +24,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return convertJDBCValueByType(dataValue); + return dataValue.getBigDecimalString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java index 76f05f98b..452d09765 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlGeometryProcessor.java @@ -6,6 +6,8 @@ import ai.chat2db.spi.model.SQLDataValue; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.WKBReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -17,9 +19,11 @@ public class MysqlGeometryProcessor extends DefaultValueProcessor { + private static final Logger log = LoggerFactory.getLogger(MysqlGeometryProcessor.class); + @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return MysqlDmlValueTemplate.wrapGeometry(dataValue.getValue()); } @Override @@ -75,18 +79,15 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } return dbGeometry != null ? dbGeometry.toString() : null; } catch (Exception e) { - return super.getJdbcValue(dataValue); + log.warn("Error converting database geometry", e); + return dataValue.getStringValue(); } } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue)); - } - - private String wrap(String value) { - return String.format(MysqlDmlValueTemplate.GEOMETRY_TEMPLATE, value); + return MysqlDmlValueTemplate.wrapGeometry(convertJDBCValueByType(dataValue)); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java index 2aabdf2f3..ff1ed2db0 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTextProcessor.java @@ -4,11 +4,8 @@ import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; -import java.util.function.Function; - /** * @author: zgq * @date: 2024年06月05日 0:11 @@ -19,31 +16,19 @@ public class MysqlTextProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return getClobString(dataValue, super::convertJDBCValueByType); + return dataValue.getClobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(getClobString(dataValue, super::convertJDBCValueStrByType)); + return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); } - private String getClobString(JDBCDataValue dataValue, Function function) { - try { - return dataValue.getClobString(); - } catch (Exception e) { - log.warn("convertJDBCValue error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return function.apply(dataValue); - } - } - private String wrap(String value) { - return EasyStringUtils.escapeAndQuoteString(value); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 93f0897fe..560068d93 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -1,6 +1,6 @@ package ai.chat2db.plugin.mysql.value.sub; -import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; @@ -13,28 +13,18 @@ public class MysqlTimestampProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return isValidTimestamp(dataValue) ? new String(dataValue.getBytes()) : "0000-00-00 00:00:00"; + return new String(dataValue.getBytes()); } - protected boolean isValidTimestamp(JDBCDataValue data) { - byte[] buffer = data.getBytes(); - String stringValue = new String(buffer); - return stringValue.length() <= 0 - || stringValue.charAt(0) != '0' - || !"0000-00-00".equals(stringValue) - && !"0000-00-00 00:00:00".equals(stringValue) - && !"00000000000000".equals(stringValue) - && !"0".equals(stringValue); - } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return String.format(MysqlDmlValueTemplate.COMMON_TEMPLATE, convertJDBCValueByType(dataValue)); + return EasyStringUtils.quoteString(new String(dataValue.getBytes())); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java index bff921153..71b689aca 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlVarBinaryProcessor.java @@ -1,10 +1,9 @@ package ai.chat2db.plugin.mysql.value.sub; +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; -import ch.qos.logback.core.model.processor.DefaultProcessor; import lombok.extern.slf4j.Slf4j; /** @@ -16,26 +15,24 @@ public class MysqlVarBinaryProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - // TODO: insert file - return super.convertSQLValueByType(dataValue); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + return value; + } + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - try { - return dataValue.getBlobString(); - } catch (Exception e) { - log.warn("convertJDBCValue error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return super.convertJDBCValueByType(dataValue); - } + return dataValue.getBlobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return MysqlDmlValueTemplate.wrapHex(dataValue.getBlobHexString()); } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java deleted file mode 100644 index db9f78311..000000000 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java +++ /dev/null @@ -1,62 +0,0 @@ -package ai.chat2db.plugin.mysql.value.sub; - -import ai.chat2db.spi.jdbc.DefaultValueProcessor; -import ai.chat2db.spi.model.JDBCDataValue; -import ai.chat2db.spi.model.SQLDataValue; - -import java.sql.Date; -import java.util.Calendar; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年06月01日 12:57 - */ -public class MysqlYearProcessor extends DefaultValueProcessor { - - @Override - public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); - } - - - @Override - public String convertJDBCValueByType(JDBCDataValue dataValue) { - Date date = dataValue.getDate(); - if (!isValidYear(dataValue)) { - return "0000"; - } - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int year = calendar.get(Calendar.YEAR); - String yStr; - String yZerosPadding = "0000"; - if (year < 1000) { - yStr = "" + year; - yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; - } else { - yStr = "" + year; - } - return yStr; - } - - private boolean isValidYear(JDBCDataValue data) { - byte[] buffer = data.getBytes(); - String stringValue = new String(buffer); - return stringValue.length() <= 0 - || stringValue.charAt(0) != '0' - || !"0000-00-00".equals(stringValue) - && !"0000-00-00 00:00:00".equals(stringValue) - && !"00000000000000".equals(stringValue) - && !"0".equals(stringValue) - && !"00000000".equals(stringValue) - && !"0000".equals(stringValue); - } - - - @Override - public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getJdbcValue(dataValue); - } -} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java index f3547f62b..c43f82be9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/template/MysqlDmlValueTemplate.java @@ -6,7 +6,20 @@ */ public class MysqlDmlValueTemplate { - public static final String COMMON_TEMPLATE = "'%s'"; public static final String GEOMETRY_TEMPLATE = "ST_GeomFromText('%s')"; public static final String BIT_TEMPLATE = "b'%s'"; + public static final String HEX_TEMPLATE = "0x%s"; + + + public static String wrapGeometry(String value) { + return String.format(GEOMETRY_TEMPLATE, value); + } + + public static String wrapBit(String value) { + return String.format(BIT_TEMPLATE, value); + } + + public static String wrapHex(String value) { + return String.format(HEX_TEMPLATE, value); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json index adb455ed2..a16db8c1c 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/oracle.json @@ -8,10 +8,12 @@ "custom": false, "defaultDriver": true, "downloadJdbcDriverUrls": [ - "https://cdn.chat2db-ai.com/lib/ojdbc8-19.3.0.0.jar", - "https://cdn.chat2db-ai.com/lib/orai18n-19.3.0.0.jar" + "https://cdn.chat2db-ai.com/lib/ojdbc11-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/orai18n-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/xmlparserv2-21.5.0.0.jar", + "https://cdn.chat2db-ai.com/lib/xdb-21.5.0.0.jar" ], - "jdbcDriver": "ojdbc8-19.3.0.0.jar,orai18n-19.3.0.0.jar", + "jdbcDriver": "ojdbc11-21.5.0.0.jar,orai18n-21.5.0.0.jar,xmlparserv2-21.5.0.0.jar,xdb-21.5.0.0.jar", "jdbcDriverClass": "oracle.jdbc.driver.OracleDriver" } ], diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 9f1c2e277..495333b41 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -21,7 +21,7 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueByType(dataValue); + return OracleValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 2a545ce00..2cd16da35 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -9,7 +9,7 @@ /** * @author: zgq * @date: 2024年06月03日 23:21 - */ // TODO: 1.空间数据类型 2.XML数据类型 3.动态类型数据 4.ANSI、DB2 和 SQL/DS 数据 + */ // TODO: 1.空间数据类型 2.动态类型数据 public class OracleValueProcessorFactory { private static final Map PROCESSOR_MAP; @@ -17,6 +17,8 @@ public class OracleValueProcessorFactory { static { OracleClobProcessor oracleClobProcessor = new OracleClobProcessor(); OracleTimeStampProcessor oracleTimeStampProcessor = new OracleTimeStampProcessor(); + OracleBlobProcessor oracleBlobProcessor = new OracleBlobProcessor(); + OracleRawValueProcessor oracleRawValueProcessor = new OracleRawValueProcessor(); PROCESSOR_MAP = Map.ofEntries( //clob Map.entry(OracleColumnTypeEnum.CLOB.name(), oracleClobProcessor), @@ -34,14 +36,18 @@ public class OracleValueProcessorFactory { //number Map.entry(OracleColumnTypeEnum.NUMBER.name(), new OracleNumberProcessor()), //blob - Map.entry(OracleColumnTypeEnum.BLOB.name(), new OracleBlobProcessor()) + Map.entry(OracleColumnTypeEnum.BLOB.name(), oracleBlobProcessor), + //raw + Map.entry(OracleColumnTypeEnum.RAW.name(), oracleRawValueProcessor), + Map.entry(OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName(), oracleRawValueProcessor), + //xml + Map.entry("SYS.XMLTYPE", new OracleXmlValueProcessor()) ); } public static DefaultValueProcessor getValueProcessor(String type) { - DefaultValueProcessor processor = PROCESSOR_MAP.get(type); - return processor == null ? new DefaultValueProcessor() : processor; + return PROCESSOR_MAP.getOrDefault(type, new DefaultValueProcessor()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java index e246fdb88..63b3b54e9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -1,9 +1,9 @@ package ai.chat2db.plugin.oracle.value.sub; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import lombok.extern.slf4j.Slf4j; /** @@ -15,24 +15,17 @@ public class OracleBlobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getBlobHexString(); + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - try { - return dataValue.getBlobString(); - } catch (Exception e) { - log.warn("convertJDBCValueByType error database: {} , error dataType: {} ", - Chat2DBContext.getDBConfig().getDbType(), dataValue.getType(), e); - return super.convertJDBCValueByType(dataValue); - } - + return dataValue.getBlobString(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return dataValue.getBlobHexString(); + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java index 1409efd57..9773e6b6e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleClobProcessor.java @@ -13,7 +13,7 @@ public class OracleClobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return EasyStringUtils.escapeAndQuoteString(dataValue.getValue()); } @@ -25,10 +25,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(dataValue.getClobString()); - } - - private String wrap(String value) { - return EasyStringUtils.escapeAndQuoteString(value); + return EasyStringUtils.escapeAndQuoteString(dataValue.getClobString()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java index c807fa670..7507e6bdc 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java @@ -21,7 +21,7 @@ public class OracleDateProcessor extends DefaultValueProcessor { */ @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue()); + return OracleDmlValueTemplate.wrapDate(dataValue.getValue()); } /** @@ -41,16 +41,9 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { } - /** - * @param dataValue - * @return - */ + @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue)); - } - - private String wrap(String value) { - return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + return OracleDmlValueTemplate.wrapDate(convertJDBCValueByType(dataValue)); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java index 11b619b01..717cea5d8 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalDSProcessor.java @@ -14,22 +14,19 @@ public class OracleIntervalDSProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); + return OracleDmlValueTemplate.wrapIntervalDayToSecond(dataValue.getValue(), dataValue.getPrecision(), dataValue.getScale()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return super.convertJDBCValueByType(dataValue); + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); + return OracleDmlValueTemplate.wrapIntervalDayToSecond(convertJDBCValueByType(dataValue), dataValue.getPrecision(), dataValue.getScale()); } - private String wrap(String value, int precision, int scale) { - return String.format(OracleDmlValueTemplate.INTEGER_DAY_TO_SECOND_TEMPLATE, value, precision, scale); - } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java index f11ca212f..1ae0d5577 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleIntervalYMProcessor.java @@ -15,22 +15,18 @@ public class OracleIntervalYMProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return wrap(dataValue.getValue(), dataValue.getPrecision()); + return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getValue(), dataValue.getPrecision()); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return super.convertJDBCValueByType(dataValue); + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getPrecision()); - } - - public String wrap(String value, int precision) { - return String.format(OracleDmlValueTemplate.INTEGER_YEAR_TO_MONTH_TEMPLATE, value, precision); + return OracleDmlValueTemplate.wrapIntervalYearToMonth(dataValue.getStringValue(), dataValue.getPrecision()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java index 987bdf6d0..2c9952dd7 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleNumberProcessor.java @@ -14,7 +14,7 @@ public class OracleNumberProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return super.convertSQLValueByType(dataValue); + return dataValue.getValue(); } @@ -26,6 +26,6 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return convertJDBCValueByType(dataValue); + return dataValue.getBigDecimalString(); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java new file mode 100644 index 000000000..db6517808 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java @@ -0,0 +1,31 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月28日 下午1:59 + */ +public class OracleRawValueProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getBinaryDataString(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java index 44a9d8dd7..eb7ec36e9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampProcessor.java @@ -24,7 +24,6 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - // TODO: datagrip对timestampLTZ的处理是不受时区影响的，但其实这个字段就是为了可以协同时区问题的，有待商讨 Timestamp timestamp = dataValue.getTimestamp(); int scale = dataValue.getScale(); LocalDateTime localDateTime = timestamp.toLocalDateTime(); @@ -45,8 +44,8 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { private String wrap(String value, int scale) { if (scale == 0) { - return String.format(OracleDmlValueTemplate.DATE_TEMPLATE, value); + return OracleDmlValueTemplate.wrapDate(value); } - return String.format(OracleDmlValueTemplate.TIMESTAMP_TEMPLATE, value, scale); + return OracleDmlValueTemplate.wrapTimestamp(value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java index 7a2d95744..bd0ceee48 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -21,21 +21,19 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - // TODO: return:2024-06-05 17:32:52.849 +8:00 but it actually is 2024-06-05 17:32:52.849000 +8:00 - return super.convertJDBCValueByType(dataValue); - + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); + return wrap(dataValue.getStringValue(), dataValue.getScale()); } private String wrap(String value, int scale) { if (scale == 0) { - return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, value); + return OracleDmlValueTemplate.wrapTimestampTzWithOutNanos(value); } - return String.format(OracleDmlValueTemplate.TIMESTAMP_TZ_TEMPLATE, value, scale); + return OracleDmlValueTemplate.wrapTimestampTz(value, scale); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java new file mode 100644 index 000000000..2d8214cbb --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleXmlValueProcessor.java @@ -0,0 +1,30 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月21日 12:55 + */ +public class OracleXmlValueProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return OracleDmlValueTemplate.wrapXml(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getStringValue(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return OracleDmlValueTemplate.wrapXml(dataValue.getString()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index e9080ae7d..a0cb8f1be 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -13,6 +13,38 @@ public class OracleDmlValueTemplate { public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d TZH:TZM')"; public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')"; - public static final String INTEGER_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; - public static final String INTEGER_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; + public static final String INTERVAL_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; + public static final String INTERVAL_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; + + public static final String XML_TEMPLATE = "XMLType('%s')"; + + + public static String wrapDate(String date) { + return String.format(DATE_TEMPLATE, date); + } + + public static String wrapTimestamp(String timestamp, int scale) { + return String.format(TIMESTAMP_TEMPLATE, timestamp, scale); + } + + public static String wrapTimestampTz(String timestamp, int scale) { + return String.format(TIMESTAMP_TZ_TEMPLATE, timestamp, scale); + } + + public static String wrapTimestampTzWithOutNanos(String timestamp) { + return String.format(TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE, timestamp); + } + + public static String wrapIntervalYearToMonth(String year, int precision) { + return String.format(INTERVAL_YEAR_TO_MONTH_TEMPLATE, year, precision); + } + + public static String wrapIntervalDayToSecond(String day, int precision, int scale) { + return String.format(INTERVAL_DAY_TO_SECOND_TEMPLATE, day, precision, scale); + } + + public static String wrapXml(String xml) { + return String.format(XML_TEMPLATE, xml); + } + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index a0ca67500..4c209a3c3 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; import org.apache.commons.lang3.StringUtils; import java.util.Objects; @@ -27,12 +28,19 @@ public String getSqlValueString(SQLDataValue dataValue) { @Override public String getJdbcValue(JDBCDataValue dataValue) { Object value = dataValue.getObject(); - if (Objects.isNull(dataValue.getObject())) { + if (Objects.isNull(value)) { + // mysql -> [date]->0000:00:00 + if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return stringValue; + } + } return null; } - if (value instanceof String emptySry) { - if (StringUtils.isBlank(emptySry)) { - return emptySry; + if (value instanceof String emptyStr) { + if (StringUtils.isBlank(emptyStr)) { + return emptyStr; } } return convertJDBCValueByType(dataValue); @@ -43,6 +51,13 @@ public String getJdbcValue(JDBCDataValue dataValue) { public String getJdbcValueString(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { + // mysql -> [date]->0000:00:00 + if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return EasyStringUtils.escapeAndQuoteString(stringValue); + } + } return "NULL"; } if (value instanceof String stringValue) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 420cf1ba0..4e3d0a330 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -7,14 +7,16 @@ import lombok.Getter; import org.apache.tika.Tika; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.Objects; @@ -25,6 +27,7 @@ @Data @AllArgsConstructor public class JDBCDataValue { + private static final Logger log = LoggerFactory.getLogger(JDBCDataValue.class); private ResultSet resultSet; private ResultSetMetaData metaData; private int columnIndex; @@ -34,6 +37,7 @@ public Object getObject() { try { return resultSet.getObject(columnIndex); } catch (Exception e) { + log.warn("Failed to retrieve object from database", e); try { return resultSet.getString(columnIndex); } catch (SQLException ex) { @@ -71,7 +75,6 @@ public int getScale() { } public int getInt() { - return ResultSetUtils.getInt(resultSet, columnIndex); } @@ -91,154 +94,202 @@ public Blob getBlob() { return ResultSetUtils.getBlob(resultSet, columnIndex); } + public String getBlobHexString() { + return BaseEncoding.base16().encode(getBytes()); + } + + public BigDecimal getBigDecimal() { + return ResultSetUtils.getBigDecimal(resultSet, columnIndex); + } + + public String getBigDecimalString() { + BigDecimal bigDecimal = getBigDecimal(); + return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); + } + + public String getBlobString() { Blob blob = getBlob(); - LOBInfo blobInfo = getBlobInfo(blob); - String unit = blobInfo.getUnit(); - if (blobInfo.getSize() == 0) { - return ""; - } - try (InputStream binaryStream = blob.getBinaryStream()) { - Tika tika = new Tika(); - String contentType = tika.detect(binaryStream); - FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); - if (Objects.isNull(fileTypeEnum)) { - if (limitSize && isBigSize(unit)) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } - return getBlobHexString(); - } - switch (fileTypeEnum) { - case IMAGE: - if (limitSize) { - try (InputStream imageStream = blob.getBinaryStream()) { - BufferedImage bufferedImage = ImageIO.read(imageStream); - return String.format("[%s] %dx%d JPEG image %d %s", - getType(), bufferedImage.getWidth(), - bufferedImage.getHeight(), blobInfo.getSize(), unit); - } - } else { - return getBlobHexString(); - } - case STRING: - if (isBigSize(unit) && limitSize) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } else { - return new String(binaryStream.readAllBytes()); - } - default: - if (isBigSize(unit) && limitSize) { - return String.format("[%s] %d %s", getType(), blobInfo.getSize(), unit); - } - return getBlobHexString(); - } + long length = blob.length(); + return converterBinaryData(length, binaryStream); } catch (SQLException | IOException e) { - throw new RuntimeException(e); + log.warn("Error while reading binary stream", e); + return getString(); } } - private LOBInfo getBlobInfo(Blob blob) { - try { - long size = blob.length(); - return getLobInfo(size); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - public String getClobString() { Clob clob = getClob(); - LOBInfo cLobInfo = getCLobInfo(clob); - int size = cLobInfo.getSize(); - if (size == 0) { - return ""; - } - String unit = cLobInfo.getUnit(); - if (limitSize && isBigSize(unit)) { - return String.format("[%s] %d %s", getType(), size, unit); - } - StringBuilder builder = new StringBuilder(size); - String line; try (BufferedReader reader = new BufferedReader(clob.getCharacterStream())) { + long length = clob.length(); + LOBInfo cLobInfo = getLobInfo(length); + double size = cLobInfo.getSize(); + if (size == 0) { + return ""; + } + String unit = cLobInfo.getUnit(); + if (limitSize && isBigSize(unit)) { + return String.format("[%s] %s", getType(), cLobInfo); + } + StringBuilder builder = new StringBuilder((int) (Math.ceil(size))); + String line; + while ((line = reader.readLine()) != null) { builder.append(line).append("\n"); } + return builder.toString(); } catch (IOException | SQLException e) { - throw new RuntimeException(e); + log.warn("Error while reading clob stream", e); + return getStringValue(); } - return builder.toString(); } - private boolean isBigSize(String unit) { - return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); + private String handleImageType(InputStream imageStream, LOBInfo lobInfo) { + if (limitSize) { + try { + BufferedImage bufferedImage = ImageIO.read(imageStream); + return String.format("[%s] %dx%d JPEG image %s", getType(), bufferedImage.getWidth(), bufferedImage.getHeight(), lobInfo); + } catch (IOException e) { + log.warn("Error while reading image stream", e); + return getStringValue(); + } + } else { + return "0x" + getBlobHexString(); + } } - public LOBInfo getCLobInfo(Clob clob) { - try { - long size = clob.length(); - return getLobInfo(size); - } catch (SQLException e) { - throw new RuntimeException(e); + private String handleStringType(InputStream binaryStream, LOBInfo lobInfo) throws IOException { + if (isBigSize(lobInfo.getUnit()) && limitSize) { + return String.format("[%s] %s", getType(), lobInfo); + } else { + return new String(binaryStream.readAllBytes()); } + } + private boolean isBigSize(String unit) { + return LobUnit.G.unit.equals(unit) || LobUnit.M.unit.equals(unit); } + @NotNull private LOBInfo getLobInfo(long size) { if (size == 0) { return new LOBInfo(LobUnit.B.unit, 0); } - return calculateSizeAndUnit(size); + return new LOBInfo(size); } - @NotNull - private LOBInfo calculateSizeAndUnit(long size) { - if (size > LobUnit.G.size) { - return new LOBInfo(LobUnit.G.unit, (int) (size / LobUnit.G.size)); - } else if (size > LobUnit.M.size) { - return new LOBInfo(LobUnit.M.unit, (int) (size / LobUnit.M.size)); - } else if (size > LobUnit.K.size) { - return new LOBInfo(LobUnit.K.unit, (int) (size / LobUnit.K.size)); - } else { - return new LOBInfo(LobUnit.B.unit, (int) size); - } + public String getStringValue() { + return ResultSetUtils.getStringValue(resultSet, columnIndex); } - public String getBlobHexString() { - return "0x" + BaseEncoding.base16().encode(getBytes()); - } + public String getBinaryDataString() { + InputStream binaryStream = null; + try { + binaryStream = getBinaryStream(); + // 检查流是否支持 mark 操作，不支持则用 BufferedInputStream 包装 + if (!binaryStream.markSupported()) { + binaryStream = new BufferedInputStream(binaryStream); + } - public BigDecimal getBigDecimal() { - return ResultSetUtils.getBigDecimal(resultSet, columnIndex); - } + binaryStream.mark(Integer.MAX_VALUE); - public String getBigDecimalString() { - BigDecimal bigDecimal = getBigDecimal(); - return bigDecimal == null ? new String(getBytes()) : bigDecimal.toPlainString(); + long size = 0; + byte[] buffer = new byte[8192]; // 缓冲区 + int bytesRead; + while ((bytesRead = binaryStream.read(buffer)) != -1) { + size += bytesRead; + } + binaryStream.reset(); // 重置流到标记的位置 + return converterBinaryData(size, binaryStream); + } catch (SQLException | IOException e) { + log.warn("Error while reading binary stream", e); + return getStringValue(); + } finally { + // 关闭流 + if (binaryStream != null) { + try { + binaryStream.close(); + } catch (IOException e) { + log.warn("Error while closing binary stream", e); + } + } + } } - @Data - @AllArgsConstructor - public static class LOBInfo { - private String unit; - private int size; + private String converterBinaryData(long size, InputStream binaryStream) throws IOException, SQLException { + LOBInfo lobInfo = getLobInfo(size); + String unit = lobInfo.unit; + if (size == 0) { + return ""; + } + Tika tika = new Tika(); + String contentType = tika.detect(binaryStream); + FileTypeEnum fileTypeEnum = FileTypeEnum.fromDescription(contentType); + if (Objects.isNull(fileTypeEnum)) { + if (isBigSize(unit) && limitSize) { + return String.format("[%s] %s", getType(), lobInfo); + } + return "0x" + getBlobHexString(); + } + + return switch (fileTypeEnum) { + case IMAGE -> handleImageType(binaryStream, lobInfo); + case STRING -> handleStringType(binaryStream, lobInfo); + default -> ""; + }; } + @Getter public enum LobUnit { - B("B", 1), - K("KB", 1024), - M("MB", 1024 * 1024), - G("GB", 1024 * 1024 * 1024); + B("B", 1L), + K("KB", 1024L), + M("MB", 1024L * 1024L), + G("GB", 1024L * 1024L * 1024L); + private final String unit; - private final int size; + private final long size; - LobUnit(String unit, int size) { + LobUnit(String unit, long size) { this.unit = unit; this.size = size; } + + } + + @Getter + public static class LOBInfo { + private final String unit; + private final double size; + + public LOBInfo(String unit, double size) { + this.unit = unit; + this.size = size; + } + + public LOBInfo(long size) { + if (size >= LobUnit.G.size) { + this.unit = LobUnit.G.unit; + this.size = (double) size / LobUnit.G.size; + } else if (size >= LobUnit.M.size) { + this.unit = LobUnit.M.unit; + this.size = (double) size / LobUnit.M.size; + } else if (size >= LobUnit.K.size) { + this.unit = LobUnit.K.unit; + this.size = (double) size / LobUnit.K.size; + } else { + this.unit = LobUnit.B.unit; + this.size = (double) size; + } + } + + @Override + public String toString() { + return String.format("%.2f %s", size, unit); + } } @Getter diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java index f285bc759..05e95583a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/SQLDataValue.java @@ -25,6 +25,6 @@ public int getScale() { } public String getBlobHexString() { - return "0x" + BaseEncoding.base16().encode(value.getBytes()); + return BaseEncoding.base16().encode(value.getBytes()); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index b947eb079..0503cfa7b 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -1,10 +1,5 @@ package ai.chat2db.spi.sql; -import java.sql.*; -import java.util.*; -import java.util.function.Consumer; -import java.util.stream.Collectors; - import ai.chat2db.server.tools.base.constant.EasyToolsConstant; import ai.chat2db.server.tools.base.enums.DataSourceTypeEnum; import ai.chat2db.server.tools.base.excption.BusinessException; @@ -15,25 +10,26 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DataTypeEnum; import ai.chat2db.spi.enums.SqlTypeEnum; -import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.*; import ai.chat2db.spi.util.JdbcUtils; import ai.chat2db.spi.util.ResultSetUtils; import ai.chat2db.spi.util.SqlUtils; import cn.hutool.core.date.TimeInterval; - import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.google.common.collect.Lists; - import lombok.extern.slf4j.Slf4j; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; +import java.sql.*; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + /** * Dbhub unified database connection management * @@ -248,7 +244,7 @@ private List> generateDataList(ResultSet rs, int col, int chat2dbAu continue; } ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); - row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, false))); + row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, rs.getMetaData(), i, limitRowSize))); } if (count != null && count > 0 && rowCount++ >= count) { break; @@ -386,7 +382,7 @@ public List columns(Connection connection, String databaseName, Str tableName, String columnName) { try (ResultSet resultSet = connection.getMetaData().getColumns(databaseName, schemaName, tableName, - columnName)) { + columnName)) { return ResultSetUtils.toObjectList(resultSet, TableColumn.class); } catch (Exception e) { throw new RuntimeException(e); @@ -405,8 +401,8 @@ public List columns(Connection connection, String databaseName, Str public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { List tableIndices = Lists.newArrayList(); try (ResultSet resultSet = connection.getMetaData().getIndexInfo(databaseName, schemaName, tableName, - false, - false)) { + false, + false)) { List tableIndexColumns = ResultSetUtils.toObjectList(resultSet, TableIndexColumn.class); tableIndexColumns.stream().filter(c -> c.getIndexName() != null).collect( Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() @@ -582,7 +578,7 @@ private void addRowNumber(ExecuteResult executeResult, int pageNo, int pageSize) Header rowNumberHeader = Header.builder() .name(I18nUtils.getMessage("sqlResult.rowNumber")) .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER - .getCode()).build(); + .getCode()).build(); executeResult.setHeaderList(EasyCollectionUtils.union(Arrays.asList(rowNumberHeader), headers)); // Add row number diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index e26d496ea..9c872dc04 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -1,8 +1,6 @@ package ai.chat2db.spi.util; -import ai.chat2db.server.tools.common.util.I18nUtils; -import cn.hutool.core.io.unit.DataSizeUtil; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -29,7 +27,6 @@ public class ResultSetUtils { - public static List getRsHeader(ResultSet rs) { try { ResultSetMetaData resultSetMetaData = rs.getMetaData(); @@ -262,4 +259,12 @@ public static BigDecimal getBigDecimal(ResultSet resultSet, int columnIndex) { throw new RuntimeException(e); } } + + public static String getStringValue(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getString(columnIndex); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file From b5addde7f8960c2a50c26c803a442c855d0bf8d7 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 28 Jun 2024 22:30:14 +0800 Subject: [PATCH 31/73] MysqlYearProcessor --- .../factory/MysqlValueProcessorFactory.java | 1 + .../mysql/value/sub/MysqlYearProcessor.java | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index 94367e9a2..4e8ae2512 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -44,6 +44,7 @@ public class MysqlValueProcessorFactory { Map.entry(MysqlColumnTypeEnum.TIMESTAMP.name(), mysqlTimestampProcessor), Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mysqlTimestampProcessor), //others + Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MysqlYearProcessor()), Map.entry(MysqlColumnTypeEnum.BIT.name(), new MysqlBitProcessor()), Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java new file mode 100644 index 000000000..db9f78311 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java @@ -0,0 +1,62 @@ +package ai.chat2db.plugin.mysql.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Date; +import java.util.Calendar; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年06月01日 12:57 + */ +public class MysqlYearProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + Date date = dataValue.getDate(); + if (!isValidYear(dataValue)) { + return "0000"; + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + int year = calendar.get(Calendar.YEAR); + String yStr; + String yZerosPadding = "0000"; + if (year < 1000) { + yStr = "" + year; + yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; + } else { + yStr = "" + year; + } + return yStr; + } + + private boolean isValidYear(JDBCDataValue data) { + byte[] buffer = data.getBytes(); + String stringValue = new String(buffer); + return stringValue.length() <= 0 + || stringValue.charAt(0) != '0' + || !"0000-00-00".equals(stringValue) + && !"0000-00-00 00:00:00".equals(stringValue) + && !"00000000000000".equals(stringValue) + && !"0".equals(stringValue) + && !"00000000".equals(stringValue) + && !"0000".equals(stringValue); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getJdbcValue(dataValue); + } +} From debbf3ab829c45f5a6e97ce2afd433238040984d Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Sun, 30 Jun 2024 00:07:05 +0800 Subject: [PATCH 32/73] copy and delete Table --- .../plugin/mysql/builder/MysqlSqlBuilder.java | 2 +- .../server/domain/core/util/H2Triggers.java | 2 +- .../java/ai/chat2db/spi/model/AsyncContext.java | 2 +- .../main/java/ai/chat2db/spi/sql/SQLExecutor.java | 11 +++++------ .../ai/chat2db/spi/util/SqlSplitProcessor.java | 2 ++ .../main/java/ai/chat2db/spi/util/SqlUtils.java | 15 +++++++++++++-- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index 168a05b93..6cec12e5b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -220,7 +220,7 @@ public static List movedElements(List original, List= dp[i][j - 1]) { moved.add(original.get(i - 1)); // modified List中找到original.get(i-1)的位置 - System.out.println("Moved elements:"+ original.get(i-1).getName() + " after " + modified.indexOf(original.get(i-1)) ); +// System.out.println("Moved elements:"+ original.get(i-1).getName() + " after " + modified.indexOf(original.get(i-1)) ); i--; } else { j--; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/H2Triggers.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/H2Triggers.java index d2c5dd3d4..2df0c9d2a 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/H2Triggers.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/H2Triggers.java @@ -20,7 +20,7 @@ public void fire(Connection conn, Object[] oldRow, Object[] newRow) // This method is called when the trigger is executed. // In this example, let's simply print the new values when a row is inserted. if (newRow != null) { - System.out.println("New Row Inserted: " + Arrays.toString(newRow)); +// System.out.println("New Row Inserted: " + Arrays.toString(newRow)); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 210a7c34d..8f30b3100 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -59,7 +59,7 @@ private void asyncCallBack(Context context) { // 更新时间逐渐变长避免频繁更新 callUpdate(); Thread.sleep(2000 * n); - if (n < 300) { + if (n < 15) { n++; } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index b947eb079..f9f3e0c02 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -56,7 +56,6 @@ public static SQLExecutor getInstance() { public R execute(Connection connection, String sql, ResultSetFunction function) { - log.info("execute:{}", sql); try (Statement stmt = connection.createStatement();) { boolean query = stmt.execute(sql); // Represents the query @@ -66,13 +65,13 @@ public R execute(Connection connection, String sql, ResultSetFunction fun } } } catch (Exception e) { + log.error("execute:{}", sql,e); throw new RuntimeException(e); } return null; } public void execute(Connection connection, String sql, ResultSetConsumer consumer) { - log.info("execute:{}", sql); try (Statement stmt = connection.createStatement()) { boolean query = stmt.execute(sql); // Represents the query @@ -82,6 +81,7 @@ public void execute(Connection connection, String sql, ResultSetConsumer consume } } } catch (Exception e) { + log.error("execute:{}", sql,e); throw new RuntimeException(e); } } @@ -94,7 +94,7 @@ public void execute(Connection connection, String sql, ResultSetConsumer consume public void execute(Connection connection, String sql, Consumer> headerConsumer, Consumer> rowConsumer, boolean limitSize) { Assert.notNull(sql, "SQL must not be null"); - log.info("execute:{}", sql); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); try (Statement stmt = connection.createStatement();) { boolean query = stmt.execute(sql); @@ -123,6 +123,7 @@ public void execute(Connection connection, String sql, Consumer> he } } } catch (SQLException e) { + log.error("execute:{}", sql,e); throw new RuntimeException(e); } } @@ -143,7 +144,6 @@ public void execute(Connection connection, String sql, Consumer> he public ExecuteResult executeUpdate(String sql, Connection connection, int n) throws SQLException { Assert.notNull(sql, "SQL must not be null"); - log.info("execute:{}", sql); ExecuteResult executeResult = ExecuteResult.builder().sql(sql).success(Boolean.TRUE).build(); try (Statement stmt = connection.createStatement()) { int affectedRows = stmt.executeUpdate(sql); @@ -179,7 +179,6 @@ public List executeSelectTable(Command command) { public ExecuteResult execute(final String sql, Connection connection, boolean limitRowSize, Integer offset, Integer count) throws SQLException { Assert.notNull(sql, "SQL must not be null"); - log.info("execute:{}", sql); ExecuteResult executeResult = ExecuteResult.builder().sql(sql).success(Boolean.TRUE).build(); try (Statement stmt = connection.createStatement()) { stmt.setFetchSize(EasyToolsConstant.MAX_PAGE_SIZE); @@ -627,7 +626,6 @@ private ExecuteResult execute(String sql, Integer offset, Integer count) { } public void execute(Connection connection, String sql, int batchSize, ResultSetConsumer consumer) { - log.info("execute:{}", sql); try (Statement stmt = connection.createStatement()) { stmt.setFetchSize(batchSize); boolean query = stmt.execute(sql); @@ -638,6 +636,7 @@ public void execute(Connection connection, String sql, int batchSize, ResultSetC } } } catch (Exception e) { + log.error("execute error:{}", sql,e); throw new RuntimeException(e); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlSplitProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlSplitProcessor.java index 0ae63cefe..e5a0e7024 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlSplitProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlSplitProcessor.java @@ -146,6 +146,7 @@ public synchronized List split(StringBuffer buffer, String sqlSc List> lines = splitLine(sqlScript); Holder bufferOrder = new Holder<>(0); + int i = 0; for (List item : lines) { if (Objects.nonNull(this.dialectType) && DbType.mysql.equals(this.dialectType)) { addLineMysql(offsetStrings, buffer, bufferOrder, item); @@ -156,6 +157,7 @@ public synchronized List split(StringBuffer buffer, String sqlSc } else { throw new IllegalArgumentException("dialect type is illegal"); } + i++; } return offsetStrings; } finally { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index 43f245e71..2b6fcb010 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -288,8 +288,19 @@ private static List split(SqlSplitProcessor processor, String sql, DbTyp if (bufferStr.trim().length() != 0) { // if buffer is not empty, there will be some errors in syntax log.info("sql processor's buffer is not empty, there may be some errors. buffer={}", bufferStr); - String sqlstr = SQLParserUtils.removeComment(sql, dbType); - return Lists.newArrayList(sqlstr); + int lastSqlOffset; + if (sqls.size() == 0) { + int index = sql.indexOf(bufferStr.trim(), 0); + lastSqlOffset = index == -1 ? 0 : index; + } else { + int from = sqls.get(sqls.size() - 1).getOffset() + sqls.get(sqls.size() - 1).getStr().length(); + int index = sql.indexOf(bufferStr.trim(), from); + lastSqlOffset = index == -1 ? from : index; + } + sqls.add(new SplitSqlString(lastSqlOffset, bufferStr)); + +// String sqlstr = SQLParserUtils.removeComment(sql, dbType); +// return Lists.newArrayList(sqlstr); } return sqls.stream().map(splitSqlString -> SQLParserUtils.removeComment(splitSqlString.getStr(), dbType)).collect(Collectors.toList()); } From 820e2d51861a02b8460e2f0cac357e0d609c5714 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Tue, 2 Jul 2024 16:00:21 +0800 Subject: [PATCH 33/73] Some typos and unit tests. Unit tests for two classes: ChartServiceImpl and ConfigServiceImpl --- .../domain/api/service/ChartService.java | 2 +- .../domain/core/util/PermissionUtils.java | 2 +- .../domain/repository/entity/ChartDO.java | 4 +- .../start/test/core/ChartServiceTest.java | 180 ++++++++++++++++++ .../start/test/core/ConfigServiceTest.java | 135 +++++++++++++ .../java/ai/chat2db/spi/model/Function.java | 2 +- .../java/ai/chat2db/spi/model/Procedure.java | 2 +- 7 files changed, 321 insertions(+), 6 deletions(-) create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/ChartService.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/ChartService.java index dd579e0c7..d8d0956ea 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/ChartService.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/ChartService.java @@ -20,7 +20,7 @@ */ public interface ChartService { /** - * Save report + * Create report * * @param param * @return diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/PermissionUtils.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/PermissionUtils.java index cdef5918f..4925d7346 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/PermissionUtils.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/util/PermissionUtils.java @@ -13,7 +13,7 @@ public class PermissionUtils { /** - * Verify whether the currently logged in user has permission to operate on the current content + * Verify whether the currently logged-in user has permission to operate on the current content * * @param createUserId The creator of the current content */ diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/java/ai/chat2db/server/domain/repository/entity/ChartDO.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/java/ai/chat2db/server/domain/repository/entity/ChartDO.java index 10cf78ef7..8e07907e0 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/java/ai/chat2db/server/domain/repository/entity/ChartDO.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/java/ai/chat2db/server/domain/repository/entity/ChartDO.java @@ -73,12 +73,12 @@ public class ChartDO implements Serializable { private String schemaName; /** - * ddl content + * DDL content */ private String ddl; /** - * Whether it has been deleted, y means deleted, n means not deleted + * Whether it has been deleted, 'Y' means deleted, 'N' means not deleted */ private String deleted; diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java new file mode 100644 index 000000000..eb179c20b --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java @@ -0,0 +1,180 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.chart.ChartCreateParam; +import ai.chat2db.server.domain.api.chart.ChartListQueryParam; +import ai.chat2db.server.domain.api.chart.ChartQueryParam; +import ai.chat2db.server.domain.api.chart.ChartUpdateParam; +import ai.chat2db.server.domain.api.model.Chart; +import ai.chat2db.server.domain.api.service.ChartService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Arrays; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +public class ChartServiceTest extends TestApplication { + + @Autowired + private ChartService chartService; + + + @Test + public void testCreateWithPermission() { + try { + userLoginIdentity(false, 4L); + userLoginIdentity(true, 3L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ChartCreateParam createParam = new ChartCreateParam(); + Optional.of(createParam).ifPresent(param -> { + param.setName("chat2db"); + param.setSchema("test"); + param.setDataSourceId(1L); + param.setType("MYSQL"); + param.setDatabaseName("chat2db"); + param.setSchemaName("ali_dbhub"); + param.setDdl("test"); + }); + + DataResult withPermission = chartService.createWithPermission(createParam); + assertNotNull(withPermission); + + Long id = withPermission.getData(); + chartService.find(id); + + } + + + @Test + public void testUpdateWithPermission() { + try { + userLoginIdentity(false, 1L); + userLoginIdentity(true, 4L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ChartUpdateParam chartUpdateParam = new ChartUpdateParam(); + Optional.of(chartUpdateParam).ifPresent(param -> { + param.setId(1L); + param.setName("chat2db"); + param.setSchema("test"); + param.setDataSourceId(1L); + param.setType("DM"); + param.setDatabaseName("chat2db"); + param.setSchemaName("ali_dbhub"); + param.setDdl("test"); + }); + + ActionResult actionResult = chartService.updateWithPermission(chartUpdateParam); + assertNotNull(actionResult); + } + + + @Test + public void testFind() { + try { + userLoginIdentity(false, 6L); + userLoginIdentity(true, 8L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + DataResult result = chartService.find(2L); + assertNotNull(result.getData()); + } + + + @Test + public void testQueryExistent() { + try { + userLoginIdentity(false, 7L); + userLoginIdentity(true, 9L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ChartQueryParam chartQueryParam = new ChartQueryParam(); + chartQueryParam.setId(1L); + chartQueryParam.setUserId(1L); + + DataResult chartDataResult = chartService.queryExistent(chartQueryParam); + DataResult queryExistent = chartService.queryExistent(chartDataResult.getData().getId()); + assertNotNull(chartDataResult); + assertEquals(chartDataResult, queryExistent); + } + + + @Test + public void testListQuery() { + try { + userLoginIdentity(false, 8L); + userLoginIdentity(true, 10L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ChartListQueryParam param = new ChartListQueryParam(); + param.setIdList(Arrays.asList(4L, 5L, 6L)); + param.setUserId(1L); + + ListResult listQuery = chartService.listQuery(param); + assertNotNull(listQuery); + + } + + + @Test + public void testQueryByIds() { + try { + userLoginIdentity(false, 9L); + userLoginIdentity(true, 11L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ListResult chartListResult = chartService.queryByIds(Arrays.asList(1L, 2L, 3L)); + assertNotNull(chartListResult); + } + + @Test + public void testDeleteWithPermission() { + try { + userLoginIdentity(false, 10L); + userLoginIdentity(true, 12L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ActionResult actionResult = chartService.deleteWithPermission(3L); + assertNotNull(actionResult); + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java new file mode 100644 index 000000000..c94ff163c --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java @@ -0,0 +1,135 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.Config; +import ai.chat2db.server.domain.api.param.SystemConfigParam; +import ai.chat2db.server.domain.api.service.ConfigService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.security.SecureRandom; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConfigServiceTest extends TestApplication { + + @Autowired + private ConfigService configService; + + @Test + public void testCreate() { + try { + userLoginIdentity(true, 1L); + userLoginIdentity(false, 2L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + SystemConfigParam systemConfigParam = new SystemConfigParam(); + Optional.ofNullable(systemConfigParam).ifPresent(param -> { + param.setCode(RandomCodeGenerator.generateRandomCode(6)); + param.setContent(RandomCodeGenerator.generateRandomCode(6)); + param.setSummary(RandomCodeGenerator.generateRandomCode(6)); + }); + + ActionResult actionResult = configService.create(systemConfigParam); + assertNotNull(actionResult); + } + + @Test + public void testUpdate() { + try { + userLoginIdentity(true, 4L); + userLoginIdentity(false, 5L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + SystemConfigParam systemConfigParam = new SystemConfigParam(); + systemConfigParam.setCode(RandomCodeGenerator.generateRandomCode(6)); + systemConfigParam.setContent(RandomCodeGenerator.generateRandomCode(6)); + systemConfigParam.setSummary(RandomCodeGenerator.generateRandomCode(6)); + + ActionResult update = configService.update(systemConfigParam); + assertNotNull(update); + + } + + @Test + public void testCreateOrUpdate() { + try { + userLoginIdentity(true, 3L); + userLoginIdentity(false, 6L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + SystemConfigParam systemConfigParam = new SystemConfigParam(); + systemConfigParam.setCode(RandomCodeGenerator.generateRandomCode(6)); + systemConfigParam.setContent(RandomCodeGenerator.generateRandomCode(6)); + systemConfigParam.setSummary(RandomCodeGenerator.generateRandomCode(6)); + ActionResult orUpdate = configService.createOrUpdate(systemConfigParam); + assertNotNull(orUpdate); + + } + + @Test + public void testFind() { + try { + userLoginIdentity(true, 9L); + userLoginIdentity(false, 4L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + DataResult configDataResult = configService.find("4TxfzW"); + assertNotNull(configDataResult.getData()); + } + + @Test + public void testDelete() { + try { + userLoginIdentity(true, 11L); + userLoginIdentity(false, 12L); + } catch (Exception e) { + throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); + } + + ActionResult result = configService.delete("4TxfzW"); + assertNotNull(result); + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } + + public class RandomCodeGenerator { + private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + private static final SecureRandom RANDOM = new SecureRandom(); + + public static String generateRandomCode(int length) { + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append(CHARACTERS.charAt(RANDOM.nextInt(CHARACTERS.length()))); + } + return sb.toString(); + } + } +} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Function.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Function.java index 24894c7dc..2d4f39ed5 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Function.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Function.java @@ -20,7 +20,7 @@ public class Function implements Serializable { private static final long serialVersionUID = 1L; //FUNCTION_CAT String => function catalog (may be null) - //FUNCTION_SCHEM String => function schema (may be null) + //FUNCTION_SCHEME String => function schema (may be null) //FUNCTION_NAME String => function name. This is the name used to invoke the function //REMARKS String => explanatory comment on the function //FUNCTION_TYPE short => kind of function: diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Procedure.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Procedure.java index 93163e3fa..6cb56418d 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Procedure.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/Procedure.java @@ -20,7 +20,7 @@ public class Procedure implements Serializable { private static final long serialVersionUID = 1L; //PROCEDURE_CAT String => procedure catalog (may be null) - //PROCEDURE_SCHEM String => procedure schema (may be null) + //PROCEDURE_SCHEME String => procedure schema (may be null) //PROCEDURE_NAME String => procedure name //REMARKS String => explanatory comment on the procedure //PROCEDURE_TYPE short => kind of procedure: From f152df2ae73039d1e0cee074ff03518719ddd882 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Tue, 2 Jul 2024 16:58:46 +0800 Subject: [PATCH 34/73] Some typos and unit tests. Unit tests for two classes: ChartServiceImpl and ConfigServiceImpl --- .../start/test/core/ChartServiceTest.java | 56 +++++-------------- .../start/test/core/ConfigServiceTest.java | 41 ++++---------- 2 files changed, 25 insertions(+), 72 deletions(-) diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java index eb179c20b..1593d802e 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java @@ -32,12 +32,8 @@ public class ChartServiceTest extends TestApplication { @Test public void testCreateWithPermission() { - try { - userLoginIdentity(false, 4L); - userLoginIdentity(true, 3L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 4L); +// userLoginIdentity(true, 3L); ChartCreateParam createParam = new ChartCreateParam(); Optional.of(createParam).ifPresent(param -> { @@ -61,12 +57,8 @@ public void testCreateWithPermission() { @Test public void testUpdateWithPermission() { - try { - userLoginIdentity(false, 1L); - userLoginIdentity(true, 4L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 1L); +// userLoginIdentity(true, 4L); ChartUpdateParam chartUpdateParam = new ChartUpdateParam(); Optional.of(chartUpdateParam).ifPresent(param -> { @@ -87,12 +79,8 @@ public void testUpdateWithPermission() { @Test public void testFind() { - try { - userLoginIdentity(false, 6L); - userLoginIdentity(true, 8L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 6L); +// userLoginIdentity(true, 8L); DataResult result = chartService.find(2L); assertNotNull(result.getData()); @@ -101,12 +89,8 @@ public void testFind() { @Test public void testQueryExistent() { - try { - userLoginIdentity(false, 7L); - userLoginIdentity(true, 9L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 7L); +// userLoginIdentity(true, 9L); ChartQueryParam chartQueryParam = new ChartQueryParam(); chartQueryParam.setId(1L); @@ -121,12 +105,8 @@ public void testQueryExistent() { @Test public void testListQuery() { - try { - userLoginIdentity(false, 8L); - userLoginIdentity(true, 10L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 8L); +// userLoginIdentity(true, 10L); ChartListQueryParam param = new ChartListQueryParam(); param.setIdList(Arrays.asList(4L, 5L, 6L)); @@ -140,12 +120,8 @@ public void testListQuery() { @Test public void testQueryByIds() { - try { - userLoginIdentity(false, 9L); - userLoginIdentity(true, 11L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 9L); +// userLoginIdentity(true, 11L); ListResult chartListResult = chartService.queryByIds(Arrays.asList(1L, 2L, 3L)); assertNotNull(chartListResult); @@ -153,12 +129,8 @@ public void testQueryByIds() { @Test public void testDeleteWithPermission() { - try { - userLoginIdentity(false, 10L); - userLoginIdentity(true, 12L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(false, 10L); +// userLoginIdentity(true, 12L); ActionResult actionResult = chartService.deleteWithPermission(3L); assertNotNull(actionResult); diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java index c94ff163c..6e1fbf6ad 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConfigServiceTest.java @@ -25,12 +25,8 @@ public class ConfigServiceTest extends TestApplication { @Test public void testCreate() { - try { - userLoginIdentity(true, 1L); - userLoginIdentity(false, 2L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(true, 1L); +// userLoginIdentity(false, 2L); SystemConfigParam systemConfigParam = new SystemConfigParam(); Optional.ofNullable(systemConfigParam).ifPresent(param -> { @@ -45,12 +41,8 @@ public void testCreate() { @Test public void testUpdate() { - try { - userLoginIdentity(true, 4L); - userLoginIdentity(false, 5L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(true, 4L); +// userLoginIdentity(false, 5L); SystemConfigParam systemConfigParam = new SystemConfigParam(); systemConfigParam.setCode(RandomCodeGenerator.generateRandomCode(6)); @@ -64,12 +56,9 @@ public void testUpdate() { @Test public void testCreateOrUpdate() { - try { - userLoginIdentity(true, 3L); - userLoginIdentity(false, 6L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(true, 3L); +// userLoginIdentity(false, 6L); + SystemConfigParam systemConfigParam = new SystemConfigParam(); systemConfigParam.setCode(RandomCodeGenerator.generateRandomCode(6)); @@ -82,12 +71,8 @@ public void testCreateOrUpdate() { @Test public void testFind() { - try { - userLoginIdentity(true, 9L); - userLoginIdentity(false, 4L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(true, 9L); +// userLoginIdentity(false, 4L); DataResult configDataResult = configService.find("4TxfzW"); assertNotNull(configDataResult.getData()); @@ -95,12 +80,8 @@ public void testFind() { @Test public void testDelete() { - try { - userLoginIdentity(true, 11L); - userLoginIdentity(false, 12L); - } catch (Exception e) { - throw new RuntimeException("An unexpected exception occurred during userLoginIdentity: " + e.getMessage()); - } + userLoginIdentity(true, 11L); +// userLoginIdentity(false, 12L); ActionResult result = configService.delete("4TxfzW"); assertNotNull(result); From 50c6d17c72510cb48e9b6cb38584937868462698 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Wed, 3 Jul 2024 20:09:37 +0800 Subject: [PATCH 35/73] =?UTF-8?q?Some=20typos=20and=20unit=20tests.=20Unit?= =?UTF-8?q?=20tests=20for=20three=20classes:=20ConsoleServiceImpl=E3=80=81?= =?UTF-8?q?DashboardServiceImpl=E3=80=81DatabaseServiceImpl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../param/dashboard/DashboardCreateParam.java | 2 +- .../param/datasource/DatabaseCreateParam.java | 2 + .../domain/api/service/DatabaseService.java | 6 + .../domain/core/impl/DatabaseServiceImpl.java | 2 +- .../start/test/core/ChartServiceTest.java | 3 +- .../start/test/core/ConsoleServiceTest.java | 52 +++++ .../start/test/core/DashboardServiceTest.java | 130 ++++++++++++ .../start/test/core/DatabaseServiceTest.java | 200 ++++++++++++++++++ .../start/test/dialect/DialectProperties.java | 114 ++++++++++ .../dialect/MariadbDialectProperties.java | 74 +++++++ .../dialect/MongodbDialectProperties.java | 74 +++++++ .../test/dialect/MysqlDialectProperties.java | 95 +++++++++ .../test/dialect/OracleDialectProperties.java | 89 ++++++++ .../dialect/PostgresqlDialectProperties.java | 108 ++++++++++ .../server/start/test/dialect/TestUtils.java | 63 ++++++ .../data/service/ConfigServiceTest.java | 58 ++--- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 2 +- 17 files changed, 1041 insertions(+), 33 deletions(-) create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConsoleServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DashboardServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/DialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/dashboard/DashboardCreateParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/dashboard/DashboardCreateParam.java index c08d19adb..8a97ddf14 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/dashboard/DashboardCreateParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/dashboard/DashboardCreateParam.java @@ -39,7 +39,7 @@ public class DashboardCreateParam { private String schema; /** - * Whether it has been deleted, y means deleted, n means not deleted + * Whether it has been deleted, 'Y' means deleted, 'N' means not deleted */ private String deleted; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseCreateParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseCreateParam.java index 920a01f31..5842ed739 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseCreateParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseCreateParam.java @@ -20,6 +20,8 @@ public class DatabaseCreateParam { private String name; + private String newName; + private String comment; private String charset; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/DatabaseService.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/DatabaseService.java index c990eec48..ae2e55d3d 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/DatabaseService.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/DatabaseService.java @@ -91,5 +91,11 @@ public interface DatabaseService { */ ActionResult modifySchema( SchemaOperationParam request); + /** + * Export database + * + * @param param + * @return + */ String exportDatabase(DatabaseExportParam param) throws SQLException; } diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java index eb6da6693..6a9808df4 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DatabaseServiceImpl.java @@ -152,7 +152,7 @@ public DataResult createDatabase(Database database) { @Override public ActionResult modifyDatabase(DatabaseCreateParam param) { Chat2DBContext.getDBManage().modifyDatabase(Chat2DBContext.getConnection(), param.getName(), - param.getName()); + param.getNewName()); return ActionResult.isSuccess(); } diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java index 1593d802e..a61fa2f1a 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ChartServiceTest.java @@ -99,6 +99,7 @@ public void testQueryExistent() { DataResult chartDataResult = chartService.queryExistent(chartQueryParam); DataResult queryExistent = chartService.queryExistent(chartDataResult.getData().getId()); assertNotNull(chartDataResult); + assertNotNull(queryExistent); assertEquals(chartDataResult, queryExistent); } @@ -124,7 +125,7 @@ public void testQueryByIds() { // userLoginIdentity(true, 11L); ListResult chartListResult = chartService.queryByIds(Arrays.asList(1L, 2L, 3L)); - assertNotNull(chartListResult); + assertNotNull(chartListResult.getData()); } @Test diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConsoleServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConsoleServiceTest.java new file mode 100644 index 000000000..1fd3bb579 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/ConsoleServiceTest.java @@ -0,0 +1,52 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.param.ConsoleCloseParam; +import ai.chat2db.server.domain.api.param.ConsoleConnectParam; +import ai.chat2db.server.domain.api.service.ConsoleService; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConsoleServiceTest extends TestApplication { + + @Autowired + private ConsoleService consoleService; + + @Autowired + private List dialectPropertiesList; + + @Test + public void testCreateAndCloseConsole() { + // MYSQL ORACLE POSTGRESQL + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + // creat + ConsoleConnectParam consoleCreateParam = new ConsoleConnectParam(); + consoleCreateParam.setDataSourceId(dataSourceId); + consoleCreateParam.setConsoleId(consoleId); + consoleCreateParam.setDatabaseName(dialectProperties.getDatabaseName()); + ActionResult console = consoleService.createConsole(consoleCreateParam); + assertNotNull(console); + + // close + ConsoleCloseParam consoleCloseParam = new ConsoleCloseParam(); + consoleCloseParam.setDataSourceId(dataSourceId); + consoleCloseParam.setConsoleId(consoleId); + consoleService.closeConsole(consoleCloseParam); + Chat2DBContext.removeContext(); + } + } + + +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DashboardServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DashboardServiceTest.java new file mode 100644 index 000000000..0d2d91381 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DashboardServiceTest.java @@ -0,0 +1,130 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.Dashboard; +import ai.chat2db.server.domain.api.param.dashboard.DashboardCreateParam; +import ai.chat2db.server.domain.api.param.dashboard.DashboardPageQueryParam; +import ai.chat2db.server.domain.api.param.dashboard.DashboardQueryParam; +import ai.chat2db.server.domain.api.param.dashboard.DashboardUpdateParam; +import ai.chat2db.server.domain.api.service.DashboardService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.tools.base.enums.YesOrNoEnum; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.PageResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DashboardServiceTest extends TestApplication { + + @Autowired + private DashboardService dashboardService; + + @Test + public void testCreateWithPermission() { + userLoginIdentity(false, 9L); +// userLoginIdentity(true, 11L); + + DashboardCreateParam dashboardCreateParam = new DashboardCreateParam(); + dashboardCreateParam.setName("chat2db"); + dashboardCreateParam.setSchema("test"); + dashboardCreateParam.setDescription("This is a test!"); + dashboardCreateParam.setDeleted(YesOrNoEnum.NO.getLetter()); + dashboardCreateParam.setUserId(5L); + dashboardCreateParam.setChartIds(new ArrayList()); + + DataResult withPermission = dashboardService.createWithPermission(dashboardCreateParam); + assertNotNull(withPermission); + } + + @Test + public void testUpdateWithPermission() { + // Note: Only administrators can edit this. + userLoginIdentity(true, 9L); + + DashboardUpdateParam dashboardUpdateParam = new DashboardUpdateParam(); + dashboardUpdateParam.setId(1L); + dashboardUpdateParam.setName("chat2db"); + dashboardUpdateParam.setSchema("test"); + dashboardUpdateParam.setDescription("This is a test!"); + dashboardUpdateParam.setDeleted(YesOrNoEnum.NO.getLetter()); + dashboardUpdateParam.setUserId(5L); + dashboardUpdateParam.setChartIds(new ArrayList()); + + ActionResult actionResult = dashboardService.updateWithPermission(dashboardUpdateParam); + assertNotNull(actionResult); + + } + + @Test + public void testFind() { + userLoginIdentity(false, 4L); +// userLoginIdentity(true, 2L); + + DataResult find = dashboardService.find(2L); + assertNotNull(find.getData()); + } + + @Test + public void testQueryExistent() { + userLoginIdentity(false, 8L); + + DashboardQueryParam param = new DashboardQueryParam(); + param.setId(5L); + param.setUserId(9L); + + DataResult existent = dashboardService.queryExistent(param); + DataResult dashboardDataResult = dashboardService.queryExistent(5L); + assertNotNull(existent.getData()); + assertNotNull(dashboardDataResult.getData()); + assertEquals(existent, dashboardDataResult); + } + + @Test + public void testDeleteWithPermission() { + userLoginIdentity(false, 7L); +// userLoginIdentity(true, 4L); + + DataResult dashboardDataResult = dashboardService.find(4L); + if (dashboardDataResult.getData() != null) { + ActionResult actionResult = dashboardService.deleteWithPermission(dashboardDataResult.getData().getId()); + assertNotNull(actionResult); + } + + } + + @Test + public void testQueryPage() { + userLoginIdentity(false, 12L); + + DashboardPageQueryParam param = new DashboardPageQueryParam(); + param.setUserId(5L); + param.setSearchKey("chat"); + + PageResult queryPage = dashboardService.queryPage(param); + assertNotNull(queryPage.getData()); + } + + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java new file mode 100644 index 000000000..04676f380 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java @@ -0,0 +1,200 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.param.MetaDataQueryParam; +import ai.chat2db.server.domain.api.param.SchemaOperationParam; +import ai.chat2db.server.domain.api.param.SchemaQueryParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseCreateParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseQueryAllParam; +import ai.chat2db.server.domain.api.service.DatabaseService; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.spi.model.Database; +import ai.chat2db.spi.model.MetaSchema; +import ai.chat2db.spi.model.Schema; +import ai.chat2db.spi.model.Sql; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.sql.SQLException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DatabaseServiceTest extends TestApplication { + + @Autowired + private DatabaseService databaseService; + + @Autowired + private List dialectPropertiesList; + + @Test + public void testQueryAll() { + + // MYSQL ORACLE POSTGRESQL + for (DialectProperties dialectProperties : dialectPropertiesList) { + String dbType = dialectProperties.getDbType(); + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + DatabaseQueryAllParam queryAllParam = new DatabaseQueryAllParam(); + queryAllParam.setDbType(dbType); + queryAllParam.setDataSourceId(dataSourceId); + + ListResult result = databaseService.queryAll(queryAllParam); + assertNotNull(result.getData()); + } + + } + + @Test + public void testQuerySchema() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + SchemaQueryParam schemaQueryParam = new SchemaQueryParam(); + schemaQueryParam.setDataSourceId(dataSourceId); + schemaQueryParam.setDataBaseName(dialectProperties.getDatabaseName()); + + ListResult schemaListResult = databaseService.querySchema(schemaQueryParam); + assertNotNull(schemaListResult.getData()); + } + } + + @Test + public void testQueryDatabaseSchema() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + MetaDataQueryParam param = new MetaDataQueryParam(); + param.setDataSourceId(dataSourceId); + param.setRefresh(true); + + DataResult metaSchemaDataResult = databaseService.queryDatabaseSchema(param); + assertNotNull(metaSchemaDataResult.getData()); + } + } + + @Test + public void testDeleteDatabase() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + DatabaseCreateParam param = new DatabaseCreateParam(); + param.setName("test"); + + ActionResult actionResult = databaseService.deleteDatabase(param); + assertNotNull(actionResult); + } + } + + @Test + public void testCreateDatabase() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + Database database = new Database(); + DataResult database1 = databaseService.createDatabase(database); + assertNotNull(database1); + } + } + + @Test + public void testModifyDatabase() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + DatabaseCreateParam databaseCreateParam = new DatabaseCreateParam(); + databaseCreateParam.setName("test" + TestUtils.nextLong()); + + ActionResult actionResult = databaseService.modifyDatabase(databaseCreateParam); + assertNotNull(actionResult); + + } + } + + @Test + public void testDeleteSchema() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + SchemaOperationParam operationParam = new SchemaOperationParam(); + operationParam.setDatabaseName(dialectProperties.getDatabaseName()); + operationParam.setSchemaName("test" + TestUtils.nextLong()); + + ActionResult actionResult = databaseService.deleteSchema(operationParam); + assertNotNull(actionResult); + } + } + + + @Test + public void testCreateSchema() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + Schema schema = new Schema(); + DataResult result = databaseService.createSchema(schema); + assertNotNull(result); + } + } + + @Test + public void testModifySchema() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + SchemaOperationParam schemaOperationParam = new SchemaOperationParam(); + schemaOperationParam.setDatabaseName(dialectProperties.getDatabaseName()); + schemaOperationParam.setSchemaName("test" + TestUtils.nextLong()); + schemaOperationParam.setNewSchemaName("test" + TestUtils.nextLong()); + + ActionResult actionResult = databaseService.modifySchema(schemaOperationParam); + assertNotNull(actionResult); + } + } + + // TODO：回头专门测试 + @Test + public void testExportDatabase() { + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + DatabaseExportParam exportParam = new DatabaseExportParam(); + exportParam.setDatabaseName(dialectProperties.getDatabaseName()); + exportParam.setContainData(true); + exportParam.setSchemaName("test" + TestUtils.nextLong()); + + try { + String exportDatabase = databaseService.exportDatabase(exportParam); + assertNotNull(exportDatabase); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/DialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/DialectProperties.java new file mode 100644 index 000000000..2b5be81ca --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/DialectProperties.java @@ -0,0 +1,114 @@ +package ai.chat2db.server.start.test.dialect; + +import java.util.Date; + +/** + * Dialect configuration + */ +public interface DialectProperties { + + /** + * Supported database types + * + * @return + */ + String getDbType(); + + /** + * connection + * + * @return + */ + String getUrl(); + + /** + * Abnormal connection + * + * @return + */ + String getErrorUrl(); + + /** + * userName + * + * @return + */ + + String getUsername(); + + /** + * password + * + * @return + */ + String getPassword(); + + /** + * Name database + * + * @return + */ + String getDatabaseName(); + + /** + * The case depends on the specific database: + * Create table structure: test table + * Field: + * id primary key auto-increment + * date date is not empty + * number long integer type + * string string length 100 default value "DATA" + * + * Index (plus $tableName_ because some database indexes are globally unique): + * $tableName_idx_date date index reverse order + * $tableName_uk_number unique index + * $tableName_idx_number_string joint index + * + * @return + */ + String getCrateTableSql(String tableName); + + /** + * Create table structure + * + * @return + */ + String getDropTableSql(String tableName); + + /** + * Create a piece of data + * + * @return + */ + String getInsertSql(String tableName, Date date, Long number, String string); + + /** + * Query a query sql + * + * @return + */ + String getSelectSqlById(String tableName, Long id); + + /** + * Get a sql whose table structure does not exist + * + * @return + */ + String getTableNotFoundSqlById(String tableName); + + /** + * Convert case + * Some database table structures store uppercase letters by default + * Some databases store lowercase by default + * + * @param string + * @return + */ + String toCase(String string); + + /** + * port + * @return + */ + Integer getPort(); +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java new file mode 100644 index 000000000..f4eedd53a --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java @@ -0,0 +1,74 @@ +package ai.chat2db.server.start.test.dialect; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.Date; + +@Component +public class MariadbDialectProperties implements DialectProperties{ + @Override + public String getDbType() { + return "MARIADB"; + } + + @Override + public String getUrl() { + return "jdbc:mariadb://183.247.151.185:13303/"; + } + + @Override + public String getErrorUrl() { + return "jdbc:mariadb://error:13303/"; + } + + @Override + public String getUsername() { + return "root"; + } + + @Override + public String getPassword() { + return "ali_dbhub"; + } + + @Override + public String getDatabaseName() { + return null; + } + + @Override + public String getCrateTableSql(String tableName) { + return null; + } + + @Override + public String getDropTableSql(String tableName) { + return null; + } + + @Override + public String getInsertSql(String tableName, Date date, Long number, String string) { + return null; + } + + @Override + public String getSelectSqlById(String tableName, Long id) { + return null; + } + + @Override + public String getTableNotFoundSqlById(String tableName) { + return null; + } + + @Override + public String toCase(String string) { + return StringUtils.toRootLowerCase(string); + } + + @Override + public Integer getPort() { + return 13303; + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java new file mode 100644 index 000000000..b687e9a23 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java @@ -0,0 +1,74 @@ +package ai.chat2db.server.start.test.dialect; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.Date; + +@Component +public class MongodbDialectProperties implements DialectProperties{ + @Override + public String getDbType() { + return "MONGODB"; + } + + @Override + public String getUrl() { + return "mongodb://183.247.151.185:27017/"; + } + + @Override + public String getErrorUrl() { + return "mongodb://error:27017/"; + } + + @Override + public String getUsername() { + return "test"; + } + + @Override + public String getPassword() { + return "test@123456"; + } + + @Override + public String getDatabaseName() { + return null; + } + + @Override + public String getCrateTableSql(String tableName) { + return null; + } + + @Override + public String getDropTableSql(String tableName) { + return null; + } + + @Override + public String getInsertSql(String tableName, Date date, Long number, String string) { + return null; + } + + @Override + public String getSelectSqlById(String tableName, Long id) { + return null; + } + + @Override + public String getTableNotFoundSqlById(String tableName) { + return null; + } + + @Override + public String toCase(String string) { + return StringUtils.toRootLowerCase(string); + } + + @Override + public Integer getPort() { + return 27017; + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java new file mode 100644 index 000000000..1146f45d6 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java @@ -0,0 +1,95 @@ +package ai.chat2db.server.start.test.dialect; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.Date; + +/** + * mysql + * + * @author Jiaju Zhuang + */ +@Component +public class MysqlDialectProperties implements DialectProperties { + + @Override + public String getDbType() { + return "MYSQL"; + } + + @Override + public String getUrl() { + return "jdbc:mysql://183.247.151.185:13306"; + } + + @Override + public String getErrorUrl() { + return "jdbc:mysql://error.rm-8vb099vo8309mcngk.mysql.zhangbei.rds.aliyuncs.com:13306"; + } + + @Override + public String getUsername() { + return "root"; + } + + @Override + public String getPassword() { + return "ali_dbhub"; + } + + @Override + public String getDatabaseName() { + return "ali_dbhub_test"; + } + + @Override + public String getCrateTableSql(String tableName) { + return "CREATE TABLE `" + tableName + "`\n\t" + + "(\n\t" + + " `id` bigint PRIMARY KEY AUTO_INCREMENT NOT NULL COMMENT 'Primary key auto-increment',\n\t" + + " `date` datetime(3) not null COMMENT 'date',\n\t" + + " `number` bigint COMMENT 'long integer',\n\t" + + " `string` VARCHAR(100) default 'DATA' COMMENT 'name',\n\t" + + " index " + tableName + "_idx_date (date desc) comment 'date index',\n\t" + + " unique " + tableName + "_uk_number (number) comment 'unique index',\n\t" + + " index " + tableName + "_idx_number_string (number, date) comment 'Union index'\n\t" + + ") COMMENT ='Test table';"; + } + + @Override + public String getDropTableSql(String tableName) { + return "drop table " + tableName + ";"; + } + + @Override + public String getInsertSql(String tableName, Date date, Long number, String string) { + return "INSERT INTO `" + tableName + "` (date,number,string) VALUES ('" + DateUtil.format(date, + DatePattern.NORM_DATETIME_MS_FORMAT) + "','" + number + "','" + string + "');"; + } + + @Override + public String getSelectSqlById(String tableName, Long id) { + return "select *\n\t" + + "from " + tableName + "\n\t" + + "where `id` = '" + id + "';"; + } + + @Override + public String getTableNotFoundSqlById(String tableName) { + return "select *\n" + + "from " + tableName + "_notfound;"; + } + + @Override + public String toCase(String string) { + return StringUtils.toRootLowerCase(string); + } + + @Override + public Integer getPort() { + return 3306; + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java new file mode 100644 index 000000000..0eac03a52 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java @@ -0,0 +1,89 @@ +package ai.chat2db.server.start.test.dialect; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.Date; +@Component +public class OracleDialectProperties implements DialectProperties { + + @Override + public String getDbType() { + return "ORACLE"; + } + + @Override + public String getUrl() { + return "jdbc:oracle:thin:@183.247.151.185:11521:XE"; + } + + @Override + public String getErrorUrl() { + return "jdbc:oracle:thin:@183.247.151.185:11521:XE1"; + } + + @Override + public String getUsername() { + return "system"; + } + + @Override + public String getPassword() { + return "ali_dbhub"; + } + + @Override + public String getDatabaseName() { + return "TEST_USER"; + } + + @Override + public String getCrateTableSql(String tableName) { + return "CREATE TABLE `" + tableName + "`\n\t" + + "(\n\t" + + " `id` bigint PRIMARY KEY AUTO_INCREMENT NOT NULL COMMENT 'Primary key auto-increment',\n\t" + + " `date` datetime(3) not null COMMENT 'date',\n\t" + + " `number` bigint COMMENT 'long integer',\n\t" + + " `string` VARCHAR(100) default 'DATA' COMMENT 'name',\n\t" + + " index " + tableName + "_idx_date (date desc) comment 'date index',\n\t" + + " unique " + tableName + "_uk_number (number) comment 'unique index',\n\t" + + " index " + tableName + "_idx_number_string (number, date) comment 'Union index'\n\t" + + ") COMMENT ='Test table';"; + } + + @Override + public String getDropTableSql(String tableName) { + return "drop table " + tableName + ";"; + } + + @Override + public String getInsertSql(String tableName, Date date, Long number, String string) { + return "INSERT INTO `" + tableName + "` (date,number,string) VALUES ('" + DateUtil.format(date, + DatePattern.NORM_DATETIME_MS_FORMAT) + "','" + number + "','" + string + "');"; + } + + @Override + public String getSelectSqlById(String tableName, Long id) { + return "select *\n\t" + + "from " + tableName + "\n\t" + + "where `id` = '" + id + "';"; + } + + @Override + public String getTableNotFoundSqlById(String tableName) { + return "select *\n" + + "from " + tableName + "_notfound;"; + } + + @Override + public String toCase(String string) { + return StringUtils.toRootLowerCase(string); + } + + @Override + public Integer getPort() { + return 11521; + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java new file mode 100644 index 000000000..ac81c7cb1 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java @@ -0,0 +1,108 @@ +/** + * Alipay.com Inc. + * Copyright (c) 2004-2022 All Rights Reserved. + */ +package ai.chat2db.server.start.test.dialect; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.Date; + +/** + * @author jipengfei + * @version : PgDialectProperties.java, v 0.1 December 13, 2022 21:48 jipengfei Exp $ + */ +@Component +public class PostgresqlDialectProperties implements DialectProperties { + + @Override + public String getDbType() { + return "POSTGRESQL"; + } + + @Override + public String getUrl() { + return "jdbc:postgresql://183.247.151.185:15431/ali_dbhub_test"; + } + + @Override + public String getErrorUrl() { + return "jdbc:postgresql://error:15431/ali_dbhub"; + } + + @Override + public String getUsername() { + return "ali_dbhub"; + } + + @Override + public String getPassword() { + return "ali_dbhub"; + } + + @Override + public String getDatabaseName() { + return "ali_dbhub_test"; + } + + @Override + public String getCrateTableSql(String tableName) { + String sql = "CREATE TABLE " + tableName + "\n" + + "(\n" + + " id serial\n" + + " constraint " + tableName + "_pk primary key,\n" + + " date timestamp,\n" + + " number int,\n" + + " string varchar(100) default 'DATA'\n" + + ");\n"; + sql += "comment on table " + tableName + " is 'Test table';\n"; + sql += "comment on column " + tableName + ".id is 'Primary key auto-increment';\n"; + sql += "comment on column " + tableName + ".date is 'date';\n"; + sql += "comment on column " + tableName + ".number is 'long integer';\n"; + sql += "comment on column " + tableName + ".string is 'name';\n"; + sql += "create index " + tableName + "idx_date on " + tableName + " (date desc);"; + sql += "create unique index " + tableName + "_uk_number on " + tableName + " (number);"; + sql += "create index " + tableName + "_idx_number_string on " + tableName + " (number, date);"; + sql += "comment on index " + tableName + "_uk_number is 'date index';"; + sql += "comment on index " + tableName + "_uk_number is 'unique index';"; + sql += "comment on index " + tableName + "_idx_number_string is 'Union index';"; + return sql; + } + + @Override + public String getDropTableSql(String tableName) { + return "drop table " + tableName + ";"; + } + + @Override + public String getInsertSql(String tableName, Date date, Long number, String string) { + return "INSERT INTO " + tableName + " (date,number,string) VALUES ('" + DateUtil.format(date, + DatePattern.NORM_DATETIME_MS_FORMAT) + "','" + number + "','" + string + "');"; + } + + @Override + public String getSelectSqlById(String tableName, Long id) { + return "select *\n" + + "from " + tableName + "\n" + + "where id = '" + id + "';"; + } + + @Override + public String getTableNotFoundSqlById(String tableName) { + return "select *\n" + + "from " + tableName + "_notfound;"; + } + + @Override + public String toCase(String string) { + return StringUtils.toRootLowerCase(string); + } + + @Override + public Integer getPort() { + return 5432; + } +} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java new file mode 100644 index 000000000..6003c8adc --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java @@ -0,0 +1,63 @@ +package ai.chat2db.server.start.test.dialect; + +import ai.chat2db.spi.model.SSHInfo; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectInfo; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Test tool class + */ +public class TestUtils { + + public static final AtomicLong ATOMIC_LONG = new AtomicLong(); + + /** + * a globally unique long + * + * @return + */ + public static long nextLong() { + return ATOMIC_LONG.incrementAndGet(); + } + + /** + * If the default value is something like 'DATA' + * then you need to remove '' + * + * @param defaultValue + * @return + */ + public static String unWrapperDefaultValue(String defaultValue) { + if (defaultValue == null) { + return null; + } + if (defaultValue.startsWith("'") && defaultValue.endsWith("'")) { + if (defaultValue.length() < 2) { + return defaultValue; + } else if (defaultValue.length() == 2) { + return ""; + } else { + return defaultValue.substring(1, defaultValue.length() - 1); + } + } + return defaultValue; + } + + public static void buildContext(DialectProperties dialectProperties,Long dataSourceId,Long consoleId){ + ConnectInfo connectInfo = new ConnectInfo(); + connectInfo.setUser(dialectProperties.getUsername()); + connectInfo.setPort(dialectProperties.getPort()); + connectInfo.setHost("183.247.151.185"); + connectInfo.setSsh(new SSHInfo()); + connectInfo.setConsoleId(consoleId); + connectInfo.setDataSourceId(dataSourceId); + connectInfo.setPassword(dialectProperties.getPassword()); + connectInfo.setDbType(dialectProperties.getDbType()); + connectInfo.setUrl(dialectProperties.getUrl()); + connectInfo.setDatabase(dialectProperties.getDatabaseName()); + connectInfo.setConsoleOwn(false); + Chat2DBContext.putContext(connectInfo); + } +} diff --git a/chat2db-server/chat2db-server-test/src/test/java/ai/chat2db/server/test/domain/data/service/ConfigServiceTest.java b/chat2db-server/chat2db-server-test/src/test/java/ai/chat2db/server/test/domain/data/service/ConfigServiceTest.java index 5ad725e54..bda952457 100644 --- a/chat2db-server/chat2db-server-test/src/test/java/ai/chat2db/server/test/domain/data/service/ConfigServiceTest.java +++ b/chat2db-server/chat2db-server-test/src/test/java/ai/chat2db/server/test/domain/data/service/ConfigServiceTest.java @@ -1,29 +1,29 @@ - -package ai.chat2db.server.test.domain.data.service; - -import ai.chat2db.server.domain.api.param.SystemConfigParam; -import ai.chat2db.server.domain.api.service.ConfigService; -import ai.chat2db.server.test.common.BaseTest; - -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * @author jipengfei - * @version : ConfihServiceTest.java - */ -@Slf4j -public class ConfigServiceTest extends BaseTest { - - @Autowired - private ConfigService configService; - - @Test - public void testCreate() { - SystemConfigParam systemConfigParam = new SystemConfigParam(); - systemConfigParam.setCode("test"); - systemConfigParam.setContent("test1"); - configService.createOrUpdate(systemConfigParam); - } -} \ No newline at end of file +// +//package ai.chat2db.server.test.domain.data.service; +// +//import ai.chat2db.server.domain.api.param.SystemConfigParam; +//import ai.chat2db.server.domain.api.service.ConfigService; +//import ai.chat2db.server.test.common.BaseTest; +// +//import lombok.extern.slf4j.Slf4j; +//import org.junit.jupiter.api.Test; +//import org.springframework.beans.factory.annotation.Autowired; +// +///** +// * @author jipengfei +// * @version : ConfihServiceTest.java +// */ +//@Slf4j +//public class ConfigServiceTest extends BaseTest { +// +// @Autowired +// private ConfigService configService; +// +// @Test +// public void testCreate() { +// SystemConfigParam systemConfigParam = new SystemConfigParam(); +// systemConfigParam.setCode("test"); +// systemConfigParam.setContent("test1"); +// configService.createOrUpdate(systemConfigParam); +// } +//} \ No newline at end of file diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index f9f3e0c02..2080f992d 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -262,7 +262,7 @@ private int getChat2dbAutoRowIdIndex(List
headerList) { Header header = headerList.get(i); if ("CAHT2DB_AUTO_ROW_ID".equals(header.getName())) { headerList.remove(i); - return i; + return i + 1; } } return -1; From 231d692e6c319e58d798ebb995462d5cc7c046e5 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Thu, 4 Jul 2024 16:41:14 +0800 Subject: [PATCH 36/73] =?UTF-8?q?fix=20issue=EF=BC=9A#1390?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/ai/chat2db/server/domain/api/enums/RoleCodeEnum.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/RoleCodeEnum.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/RoleCodeEnum.java index 11cfebd29..37b7c14c3 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/RoleCodeEnum.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/RoleCodeEnum.java @@ -18,7 +18,8 @@ public enum RoleCodeEnum implements BaseEnum { /** * ADMIN */ - ADMIN("ADMIN", 2L, "chat2db", "chat2db"), + ADMIN("ADMIN", 2L, System.getenv().getOrDefault("ADMIN_NAME","chat2db"), + System.getenv().getOrDefault("ADMIN_PASSWORD","chat2db")), /** * USER From 488444e09565d3a5e9095edd1d0088c944c1d358 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 5 Jul 2024 15:22:26 +0800 Subject: [PATCH 37/73] fix rowIdIndex --- .../src/main/java/ai/chat2db/spi/sql/SQLExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 0503cfa7b..c31ab1052 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -259,7 +259,7 @@ private int getChat2dbAutoRowIdIndex(List
headerList) { Header header = headerList.get(i); if ("CAHT2DB_AUTO_ROW_ID".equals(header.getName())) { headerList.remove(i); - return i; + return i+1; } } return -1; From 8bf09448f21f1f196bc01cff80aadee3948049be Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Fri, 5 Jul 2024 15:26:57 +0800 Subject: [PATCH 38/73] . . --- .../src/main/java/ai/chat2db/spi/sql/SQLExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index 0503cfa7b..c31ab1052 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -259,7 +259,7 @@ private int getChat2dbAutoRowIdIndex(List
headerList) { Header header = headerList.get(i); if ("CAHT2DB_AUTO_ROW_ID".equals(header.getName())) { headerList.remove(i); - return i; + return i+1; } } return -1; From 44a0e6feb5c2d0b9799f5f960a76a114d1982244 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sun, 7 Jul 2024 08:40:31 +0800 Subject: [PATCH 39/73] fix(oracledb): correct timestamp processing and add TimeStampLTZ support --- .../factory/OracleValueProcessorFactory.java | 2 +- .../oracle/value/sub/OracleDateProcessor.java | 15 +--- .../sub/OracleTimeStampLTZProcessor.java | 69 +++++++++++++++++++ .../value/sub/OracleTimeStampTZProcessor.java | 19 ++++- .../template/OracleDmlValueTemplate.java | 8 +-- 5 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampLTZProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 2cd16da35..97962177e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -28,7 +28,7 @@ public class OracleValueProcessorFactory { Map.entry(OracleColumnTypeEnum.DATE.name(), new OracleDateProcessor()), //timestamp Map.entry(OracleColumnTypeEnum.TIMESTAMP.name(), oracleTimeStampProcessor), - Map.entry(OracleColumnTypeEnum.TIMESTAMP_WITH_LOCAL_TIME_ZONE.getColumnType().getTypeName(), oracleTimeStampProcessor), + Map.entry(OracleColumnTypeEnum.TIMESTAMP_WITH_LOCAL_TIME_ZONE.getColumnType().getTypeName(), new OracleTimeStampLTZProcessor()), Map.entry(OracleColumnTypeEnum.TIMESTAMP_WITH_TIME_ZONE.getColumnType().getTypeName(), new OracleTimeStampTZProcessor()), //INTERVAL Map.entry("INTERVALDS", new OracleIntervalDSProcessor()), diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java index 7507e6bdc..875d3da49 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleDateProcessor.java @@ -5,10 +5,6 @@ import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import java.sql.Timestamp; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - /** * @author: zgq * @date: 2024年06月04日 16:33 @@ -30,20 +26,13 @@ public String convertSQLValueByType(SQLDataValue dataValue) { */ @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - Timestamp timestamp = dataValue.getTimestamp(); - LocalDateTime localDateTime = timestamp.toLocalDateTime(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - String formattedDateTime = localDateTime.format(formatter); - if (formattedDateTime.endsWith("00:00:00")) { - formattedDateTime = formattedDateTime.substring(0, formattedDateTime.length() - 9); - } - return formattedDateTime; + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleDmlValueTemplate.wrapDate(convertJDBCValueByType(dataValue)); + return OracleDmlValueTemplate.wrapDate(dataValue.getStringValue()); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampLTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampLTZProcessor.java new file mode 100644 index 000000000..be3b289b5 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampLTZProcessor.java @@ -0,0 +1,69 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.plugin.oracle.value.template.OracleDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +/** + * @author: zgq + * @date: 2024年07月05日 下午4:19 + */ +public class OracleTimeStampLTZProcessor extends DefaultValueProcessor { + + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return wrap(dataValue.getValue(), dataValue.getScale()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + Timestamp timestamp = dataValue.getTimestamp(); + int scale = dataValue.getScale(); + LocalDateTime localDateTime = timestamp.toLocalDateTime(); + StringBuilder templateBuilder = new StringBuilder("yyyy-MM-dd HH:mm:ss"); + if (scale != 0) { + templateBuilder.append("."); + templateBuilder.append("S".repeat(scale)); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(templateBuilder.toString()); + return localDateTime.format(formatter); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + Timestamp timestamp = dataValue.getTimestamp(); + int scale = dataValue.getScale(); + // 将 Timestamp 转换为 Instant 对象 + Instant instant = timestamp.toInstant(); + // 将 Instant 转换为 UTC 时区的 ZonedDateTime + ZonedDateTime utcZonedDateTime = instant.atZone(ZoneId.of("UTC")); + StringBuilder templateBuilder = new StringBuilder("yyyy-MM-dd HH:mm:ss"); + if (scale != 0) { + templateBuilder.append("."); + templateBuilder.append("S".repeat(scale)); + } + // 定义日期时间格式化器 + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(templateBuilder.toString()); + // 格式化 UTC 时区的 ZonedDateTime + String formattedUtcTime = utcZonedDateTime.format(formatter); + return wrap(formattedUtcTime, scale); + } + + private String wrap(String value, int scale) { + if (scale == 0) { + return OracleDmlValueTemplate.wrapDate(value); + } + return OracleDmlValueTemplate.wrapTimestamp(value, scale); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java index bd0ceee48..182ea8582 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -21,13 +21,28 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return dataValue.getStringValue(); + //2024-06-05 17:41:27.52 +0:00 -> + String timeStampString = dataValue.getStringValue(); + int scale = dataValue.getScale(); + int lastSpaceIndex = timeStampString.lastIndexOf(" "); + int nanosLength = lastSpaceIndex - timeStampString.indexOf(".") - 1; + if (scale != 0 && nanosLength < scale) { + // 计算需要补充的零的数量 + int zerosToAdd = scale - nanosLength; + StringBuilder sb = new StringBuilder(timeStampString); + for (int i = 0; i < zerosToAdd; i++) { + sb.insert(lastSpaceIndex, '0'); + } + return sb.toString(); + + } + return timeStampString; } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return wrap(dataValue.getStringValue(), dataValue.getScale()); + return wrap(convertJDBCValueByType(dataValue), dataValue.getScale()); } private String wrap(String value, int scale) { diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java index a0cb8f1be..a9b02a2ef 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/template/OracleDmlValueTemplate.java @@ -6,12 +6,12 @@ */ public class OracleDmlValueTemplate { - public static final String DATE_TEMPLATE = "TO_DATE('%s', 'YYYY-MM-DD HH24:MI:SS')"; + public static final String DATE_TEMPLATE = "TO_DATE('%s', 'SYYYY-MM-DD HH24:MI:SS')"; - public static final String TIMESTAMP_TEMPLATE = "TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d')"; + public static final String TIMESTAMP_TEMPLATE = "TO_TIMESTAMP('%s', 'SYYYY-MM-DD HH24:MI:SS.FF%d')"; - public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS.FF%d TZH:TZM')"; - public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')"; + public static final String TIMESTAMP_TZ_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'SYYYY-MM-DD HH24:MI:SS.FF%d TZR')"; + public static final String TIMESTAMP_TZ_WITHOUT_NANOS_TEMPLATE = "TO_TIMESTAMP_TZ('%s', 'SYYYY-MM-DD HH24:MI:SS TZR')"; public static final String INTERVAL_YEAR_TO_MONTH_TEMPLATE = "INTERVAL '%s' YEAR(%d) TO MONTH"; public static final String INTERVAL_DAY_TO_SECOND_TEMPLATE = "INTERVAL '%s' DAY(%d) TO SECOND(%d)"; From 6e3b58a8f194437567e3218da117c02029fbc7f6 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sun, 7 Jul 2024 09:59:51 +0800 Subject: [PATCH 40/73] optimize MysqlYearProcessor --- .../mysql/value/sub/MysqlYearProcessor.java | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java index db9f78311..83d050fe2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlYearProcessor.java @@ -4,9 +4,6 @@ import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import java.sql.Date; -import java.util.Calendar; - /** * 功能描述 * @@ -23,40 +20,12 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - Date date = dataValue.getDate(); - if (!isValidYear(dataValue)) { - return "0000"; - } - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - int year = calendar.get(Calendar.YEAR); - String yStr; - String yZerosPadding = "0000"; - if (year < 1000) { - yStr = "" + year; - yStr = yZerosPadding.substring(0, (4 - yStr.length())) + yStr; - } else { - yStr = "" + year; - } - return yStr; - } - - private boolean isValidYear(JDBCDataValue data) { - byte[] buffer = data.getBytes(); - String stringValue = new String(buffer); - return stringValue.length() <= 0 - || stringValue.charAt(0) != '0' - || !"0000-00-00".equals(stringValue) - && !"0000-00-00 00:00:00".equals(stringValue) - && !"00000000000000".equals(stringValue) - && !"0".equals(stringValue) - && !"00000000".equals(stringValue) - && !"0000".equals(stringValue); + return new String(dataValue.getBytes()); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getJdbcValue(dataValue); + return new String(dataValue.getBytes()); } } From 805e703f69a6e2a0fbdbfd3bee42e3d704d6aece Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Mon, 8 Jul 2024 10:13:42 +0800 Subject: [PATCH 41/73] add if remove comment --- .../core/impl/DlTemplateServiceImpl.java | 2 +- .../java/ai/chat2db/spi/sql/SQLExecutor.java | 2 +- .../java/ai/chat2db/spi/util/SqlUtils.java | 35 ++++++++++++------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java index 2b45ef966..186887410 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/DlTemplateServiceImpl.java @@ -96,7 +96,7 @@ public DataResult executeUpdate(DlExecuteParam param) { //RemoveSpecialGO(param); DbType dbType = JdbcUtils.parse2DruidDbType(Chat2DBContext.getConnectInfo().getDbType()); - List sqlList = SqlUtils.parse(param.getSql(), dbType); + List sqlList = SqlUtils.parse(param.getSql(), dbType,true); Connection connection = Chat2DBContext.getConnection(); try { // connection.setAutoCommit(false); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java index f9f3e0c02..07d311618 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/sql/SQLExecutor.java @@ -497,7 +497,7 @@ public List execute(Command command) { // parse sql String type = Chat2DBContext.getConnectInfo().getDbType(); DbType dbType = JdbcUtils.parse2DruidDbType(type); - List sqlList = SqlUtils.parse(command.getScript(), dbType); + List sqlList = SqlUtils.parse(command.getScript(), dbType,true); if (CollectionUtils.isEmpty(sqlList)) { throw new BusinessException("dataSource.sqlAnalysisError"); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index 2b6fcb010..337d20f96 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -126,10 +126,10 @@ private static SQLTableSource getSQLExprTableSource(SQLTableSource sqlTableSourc private static final String EVENT_REGEX = "(?i)\\bcreate\\s+event\\b.*?\\bend\\b"; - public static List parse(String sql, DbType dbType) { + public static List parse(String sql, DbType dbType, boolean removeComment) { List list = new ArrayList<>(); try { - sql = SQLParserUtils.removeComment(sql, dbType); + if (StringUtils.isBlank(sql)) { return list; } @@ -138,10 +138,10 @@ public static List parse(String sql, DbType dbType) { SqlSplitter sqlSplitter = new SqlSplitter(PlSqlLexer.class, ";", false); sqlSplitter.setRemoveCommentPrefix(true); List sqls = sqlSplitter.split(sql); - return sqls.stream().map(splitSqlString -> SQLParserUtils.removeComment(splitSqlString.getStr(), dbType)).collect(Collectors.toList()); + return sqls.stream().map(splitSqlString -> removeComment ? SQLParserUtils.removeComment(splitSqlString.getStr(), dbType) : splitSqlString.getStr()).collect(Collectors.toList()); } - }catch (Exception e){ - log.error("sqlSplitter error",e); + } catch (Exception e) { + log.error("sqlSplitter error", e); } try { if (DbType.mysql.equals(dbType) || @@ -150,10 +150,13 @@ public static List parse(String sql, DbType dbType) { sql = updateNow(sql, dbType); SqlSplitProcessor sqlSplitProcessor = new SqlSplitProcessor(dbType, true, true); sqlSplitProcessor.setDelimiter(";"); - return split(sqlSplitProcessor, sql, dbType); + return split(sqlSplitProcessor, sql, dbType, removeComment); } - }catch (Exception e){ - log.error("sqlSplitProcessor error",e); + } catch (Exception e) { + log.error("sqlSplitProcessor error", e); + } + if (removeComment) { + sql = SQLParserUtils.removeComment(sql, dbType); } // sql = removeDelimiter(sql); if (StringUtils.isBlank(sql)) { @@ -173,7 +176,11 @@ public static List parse(String sql, DbType dbType) { try { return splitWithCreateEvent(sql, dbType); } catch (Exception e1) { - return SQLParserUtils.splitAndRemoveComment(sql, dbType); + if(removeComment) { + return SQLParserUtils.splitAndRemoveComment(sql, dbType); + }{ + return SQLParserUtils.split(sql, dbType); + } } } return list; @@ -246,7 +253,7 @@ public static String getSqlValue(String value, String dataType) { if (value == null) { return null; } - if("".equals(value)){ + if ("".equals(value)) { return "''"; } if (DEFAULT_VALUE.equals(value)) { @@ -281,7 +288,7 @@ public static boolean hasPageLimit(String sql, DbType dbType) { return false; } - private static List split(SqlSplitProcessor processor, String sql, DbType dbType) { + private static List split(SqlSplitProcessor processor, String sql, DbType dbType, boolean removeComment) { StringBuffer buffer = new StringBuffer(); List sqls = processor.split(buffer, sql); String bufferStr = buffer.toString(); @@ -302,7 +309,11 @@ private static List split(SqlSplitProcessor processor, String sql, DbTyp // String sqlstr = SQLParserUtils.removeComment(sql, dbType); // return Lists.newArrayList(sqlstr); } - return sqls.stream().map(splitSqlString -> SQLParserUtils.removeComment(splitSqlString.getStr(), dbType)).collect(Collectors.toList()); + return sqls.stream().map(splitSqlString -> removeComment ? SQLParserUtils.removeComment(splitSqlString.getStr(), dbType) : splitSqlString.getStr()).collect(Collectors.toList()); + } + + public static void main(String[] args) { + } } \ No newline at end of file From ec9121bf3582252d641e748755f76a1c3a04e844 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 8 Jul 2024 17:01:42 +0800 Subject: [PATCH 42/73] fix(chat2db): ensure proper escaping of string values in SQL queries String values in SQL queries are now properly escaped to prevent potential security issues and incorrect query syntax. This update affects the JDBC value processing logic and the way columns are built into SQL queries, streamlining the escaping mechanism for various data types. The changes include: - Removal of unnecessary null checks that were redundant with Objects.isNull(). - Streamlined string escaping logic using EasyStringUtils.escapeAndQuoteString().- Utilization of the stream API for more concise and readable code. BREAKING CHANGE: If any external code relies on the previous behavior of not escaping string values, it must now handle the escaped values appropriately to avoid syntax errors or potential SQL injection vulnerabilities. --- .../plugin/mysql/builder/MysqlSqlBuilder.java | 31 +++++++++--- .../mysql/value/MysqlValueProcessor.java | 43 ++++++++++++++++ .../value/sub/MysqlTimestampProcessor.java | 2 +- .../oracle/value/OracleValueProcessor.java | 40 +++++++++++++++ .../factory/OracleValueProcessorFactory.java | 6 ++- .../value/sub/OracleAnyDataProcessor.java | 49 +++++++++++++++++++ .../value/sub/OracleLongRawProcessor.java | 31 ++++++++++++ .../tools/common/util/EasyStringUtils.java | 10 ++++ .../chat2db/spi/jdbc/BaseValueProcessor.java | 15 ------ .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 19 ++++--- .../ai/chat2db/spi/model/JDBCDataValue.java | 5 +- .../java/ai/chat2db/spi/util/SqlUtils.java | 38 +++++++++++--- 12 files changed, 251 insertions(+), 38 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleAnyDataProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java index 6cec12e5b..21ef1cd7a 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/builder/MysqlSqlBuilder.java @@ -8,10 +8,13 @@ import ai.chat2db.spi.model.Table; import ai.chat2db.spi.model.TableColumn; import ai.chat2db.spi.model.TableIndex; +import ai.chat2db.spi.util.SqlUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; public class MysqlSqlBuilder extends DefaultSqlBuilder { @@ -103,7 +106,7 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { // 判断新增字段 List addColumnList = new ArrayList<>(); for (TableColumn tableColumn : newTable.getColumnList()) { - if (tableColumn.getEditStatus() != null ? tableColumn.getEditStatus().equals("ADD") : false) { + if (tableColumn.getEditStatus() != null ? tableColumn.getEditStatus().equals("ADD") : false) { addColumnList.add(tableColumn); } } @@ -116,7 +119,7 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { if ((StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) && StringUtils.isNotBlank(tableColumn.getName())) || moveColumnList.contains(tableColumn) || addColumnList.contains(tableColumn)) { MysqlColumnTypeEnum typeEnum = MysqlColumnTypeEnum.getByType(tableColumn.getColumnType()); - if(typeEnum == null){ + if (typeEnum == null) { continue; } if (moveColumnList.contains(tableColumn) || addColumnList.contains(tableColumn)) { @@ -131,7 +134,7 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { for (TableIndex tableIndex : newTable.getIndexList()) { if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { MysqlIndexTypeEnum mysqlIndexTypeEnum = MysqlIndexTypeEnum.getByType(tableIndex.getType()); - if(mysqlIndexTypeEnum == null){ + if (mysqlIndexTypeEnum == null) { continue; } script.append("\t").append(mysqlIndexTypeEnum.buildModifyIndex(tableIndex)).append(",\n"); @@ -139,13 +142,13 @@ public String buildModifyTaleSql(Table oldTable, Table newTable) { } // append reorder column - // script.append(buildGenerateReorderColumnSql(oldTable, newTable)); + // script.append(buildGenerateReorderColumnSql(oldTable, newTable)); if (script.length() > 2) { script = new StringBuilder(script.substring(0, script.length() - 2)); script.append(";"); return tableBuilder.append(script).toString(); - }else { + } else { return StringUtils.EMPTY; } @@ -400,8 +403,22 @@ private static String[] moveElement(String[] originalArray, int from, int to, St @Override protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { if (StringUtils.isNotBlank(databaseName)) { - script.append("`").append(databaseName).append("`").append('.'); + script.append(SqlUtils.quoteObjectName(databaseName, "`")).append('.'); + } + script.append(SqlUtils.quoteObjectName(tableName, "`")); + } + + /** + * @param columnList + * @param script + */ + @Override + protected void buildColumns(List columnList, StringBuilder script) { + if (CollectionUtils.isNotEmpty(columnList)) { + script.append(" (") + .append(columnList.stream().map(s -> SqlUtils.quoteObjectName(s, "`")).collect(Collectors.joining(","))) + .append(") "); } - script.append("`").append(tableName).append("`"); } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index dcb16dca4..0f5aad138 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -1,10 +1,13 @@ package ai.chat2db.plugin.mysql.value; import ai.chat2db.plugin.mysql.value.factory.MysqlValueProcessorFactory; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; +import java.util.Objects; import java.util.Set; /** @@ -17,6 +20,46 @@ public class MysqlValueProcessor extends DefaultValueProcessor { public static final Set FUNCTION_SET = Set.of("now()", "default"); + + @Override + public String getJdbcValue(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + // mysql -> example: [date]->0000-00-00 + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return stringValue; + } + return null; + } + if (value instanceof String emptyStr) { + if (StringUtils.isBlank(emptyStr)) { + return emptyStr; + } + } + return convertJDBCValueByType(dataValue); + } + + + @Override + public String getJdbcValueString(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + // mysql -> example: [date]->0000-00-00 + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return EasyStringUtils.escapeAndQuoteString(stringValue); + } + return "NULL"; + } + if (value instanceof String stringValue) { + if (StringUtils.isBlank(stringValue)) { + return EasyStringUtils.quoteString(stringValue); + } + } + return convertJDBCValueStrByType(dataValue); + } + @Override public String convertSQLValueByType(SQLDataValue dataValue) { if (FUNCTION_SET.contains(dataValue.getValue().toLowerCase())) { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 560068d93..29a6cf60e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -13,7 +13,7 @@ public class MysqlTimestampProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return dataValue.getValue(); + return EasyStringUtils.quoteString(dataValue.getValue()); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 495333b41..3c640e730 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -1,9 +1,14 @@ package ai.chat2db.plugin.oracle.value; +import ai.chat2db.plugin.oracle.type.OracleColumnTypeEnum; import ai.chat2db.plugin.oracle.value.factory.OracleValueProcessorFactory; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.jdbc.DefaultValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; /** * @author: zgq @@ -12,6 +17,41 @@ public class OracleValueProcessor extends DefaultValueProcessor { + @Override + public String getJdbcValue(JDBCDataValue dataValue) { + if (OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName().equalsIgnoreCase(dataValue.getType())) { + return convertJDBCValueByType(dataValue); + } + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + return null; + } + if (value instanceof String emptyStr) { + if (StringUtils.isBlank(emptyStr)) { + return emptyStr; + } + } + return convertJDBCValueByType(dataValue); + } + + + @Override + public String getJdbcValueString(JDBCDataValue dataValue) { + if (OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName().equalsIgnoreCase(dataValue.getType())) { + return convertJDBCValueStrByType(dataValue); + } + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + return "NULL"; + } + if (value instanceof String stringValue) { + if (StringUtils.isBlank(stringValue)) { + return EasyStringUtils.quoteString(stringValue); + } + } + return convertJDBCValueStrByType(dataValue); + } + @Override public String convertSQLValueByType(SQLDataValue dataValue) { return OracleValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()).convertSQLValueByType(dataValue); diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index 97962177e..fbd8d1dc9 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -39,9 +39,11 @@ public class OracleValueProcessorFactory { Map.entry(OracleColumnTypeEnum.BLOB.name(), oracleBlobProcessor), //raw Map.entry(OracleColumnTypeEnum.RAW.name(), oracleRawValueProcessor), - Map.entry(OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName(), oracleRawValueProcessor), + //long raw + Map.entry(OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName(), new OracleLongRawProcessor()), //xml - Map.entry("SYS.XMLTYPE", new OracleXmlValueProcessor()) + Map.entry("SYS.XMLTYPE", new OracleXmlValueProcessor()), + Map.entry("SYS.ANYDATA", new OracleAnyDataProcessor()) ); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleAnyDataProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleAnyDataProcessor.java new file mode 100644 index 000000000..b752ab56b --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleAnyDataProcessor.java @@ -0,0 +1,49 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年07月07日 10:42 + */ +public class OracleAnyDataProcessor extends DefaultValueProcessor { + /** + * @param dataValue + * @return + */ + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { +// byte[] bytes = dataValue.getBytes(); +// int length = bytes.length; +// String rawString = new String(bytes); +// +// // Filter printable characters +// StringBuilder printableString = new StringBuilder(); +// for (char c : rawString.toCharArray()) { +// if (c >= 32 && c <= 126) { // ASCII printable characters range +// printableString.append(c); +// } +// } + + return "SYS.ANYDATA"; + } + + + /** + * @param dataValue + * @return + */ + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return "SYS.ANYDATA"; + } + + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java new file mode 100644 index 000000000..2d030319b --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java @@ -0,0 +1,31 @@ +package ai.chat2db.plugin.oracle.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年07月07日 16:58 + */ +public class OracleLongRawProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getBinaryDataString(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 7ae6d796b..9db945304 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -229,4 +229,14 @@ public static String getBitString(byte[] bytes, final int precision) { return bitString; } + + public static String escapeLineString(String str) { + if (StringUtils.isBlank(str)) { + return str; + } + return str.replace("\r\n", "\\r\\n") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index 4c209a3c3..4bc0f90a4 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -4,7 +4,6 @@ import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; -import ai.chat2db.spi.sql.Chat2DBContext; import org.apache.commons.lang3.StringUtils; import java.util.Objects; @@ -29,13 +28,6 @@ public String getSqlValueString(SQLDataValue dataValue) { public String getJdbcValue(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { - // mysql -> [date]->0000:00:00 - if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { - String stringValue = dataValue.getStringValue(); - if (Objects.nonNull(stringValue)) { - return stringValue; - } - } return null; } if (value instanceof String emptyStr) { @@ -51,13 +43,6 @@ public String getJdbcValue(JDBCDataValue dataValue) { public String getJdbcValueString(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { - // mysql -> [date]->0000:00:00 - if (Chat2DBContext.getDBConfig().getDbType().equalsIgnoreCase("mysql")) { - String stringValue = dataValue.getStringValue(); - if (Objects.nonNull(stringValue)) { - return EasyStringUtils.escapeAndQuoteString(stringValue); - } - } return "NULL"; } if (value instanceof String stringValue) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 24f7b13c7..6538d0149 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -1,5 +1,6 @@ package ai.chat2db.spi.jdbc; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.enums.DmlType; @@ -218,16 +219,21 @@ protected String buildBaseInsertSql(String databaseName, String schemaName, Stri StringBuilder script = new StringBuilder(); script.append("INSERT INTO "); - buildTableName(null, null, tableName, script); + buildTableName(databaseName, schemaName, tableName, script); + + buildColumns(columnList, script); + + script.append(" VALUES "); + return script.toString(); + } + + protected void buildColumns(List columnList, StringBuilder script) { if (CollectionUtils.isNotEmpty(columnList)) { script.append(" (") .append(String.join(",", columnList)) .append(") "); } - - script.append(" VALUES "); - return script.toString(); } protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { @@ -252,7 +258,8 @@ protected void buildTableName(String databaseName, String schemaName, String tab */ public String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList) { String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); - return baseSql + "(" + String.join(",", valueList) + ");"; + List list = valueList.stream().map(EasyStringUtils::escapeString).toList(); + return baseSql + "(" + String.join(",", list) + ");"; } /** @@ -267,7 +274,7 @@ public String buildSingleInsertSql(String databaseName, String schemaName, Strin public String buildMultiInsertSql(String databaseName, String schemaName, String tableName, List columnList, List> valueLists) { String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); String valuesPart = valueLists.stream() - .map(values -> "(" + String.join(",", values) + ")") + .map(values -> "(" + String.join(",", values.stream().map(EasyStringUtils::escapeString).toList()) + ")") .collect(Collectors.joining(",\n")); return baseSql + valuesPart + ";"; } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 4e3d0a330..7fa9e3568 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -189,6 +189,9 @@ public String getBinaryDataString() { InputStream binaryStream = null; try { binaryStream = getBinaryStream(); + if (Objects.isNull(binaryStream)) { + return null; + } // 检查流是否支持 mark 操作，不支持则用 BufferedInputStream 包装 if (!binaryStream.markSupported()) { binaryStream = new BufferedInputStream(binaryStream); @@ -232,7 +235,7 @@ private String converterBinaryData(long size, InputStream binaryStream) throws I if (isBigSize(unit) && limitSize) { return String.format("[%s] %s", getType(), lobInfo); } - return "0x" + getBlobHexString(); + return "0x" + BaseEncoding.base16().encode(binaryStream.readAllBytes()); } return switch (fileTypeEnum) { diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index 2b6fcb010..2d7c7dd0e 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -12,7 +12,6 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.parser.SQLParserUtils; -import com.google.common.collect.Lists; import com.oceanbase.tools.sqlparser.oracle.PlSqlLexer; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Function; @@ -140,8 +139,8 @@ public static List parse(String sql, DbType dbType) { List sqls = sqlSplitter.split(sql); return sqls.stream().map(splitSqlString -> SQLParserUtils.removeComment(splitSqlString.getStr(), dbType)).collect(Collectors.toList()); } - }catch (Exception e){ - log.error("sqlSplitter error",e); + } catch (Exception e) { + log.error("sqlSplitter error", e); } try { if (DbType.mysql.equals(dbType) || @@ -152,8 +151,8 @@ public static List parse(String sql, DbType dbType) { sqlSplitProcessor.setDelimiter(";"); return split(sqlSplitProcessor, sql, dbType); } - }catch (Exception e){ - log.error("sqlSplitProcessor error",e); + } catch (Exception e) { + log.error("sqlSplitProcessor error", e); } // sql = removeDelimiter(sql); if (StringUtils.isBlank(sql)) { @@ -246,7 +245,7 @@ public static String getSqlValue(String value, String dataType) { if (value == null) { return null; } - if("".equals(value)){ + if ("".equals(value)) { return "''"; } if (DEFAULT_VALUE.equals(value)) { @@ -305,4 +304,31 @@ private static List split(SqlSplitProcessor processor, String sql, DbTyp return sqls.stream().map(splitSqlString -> SQLParserUtils.removeComment(splitSqlString.getStr(), dbType)).collect(Collectors.toList()); } + public static String quoteObjectName(String name) { + return quoteObjectName(name, "\""); + } + + public static String quoteObjectName(String name, String quoteSymbol) { + if (StringUtils.isNotBlank(name)) { + boolean startsWithQuote = name.startsWith(quoteSymbol); + boolean endsWithQuote = name.endsWith(quoteSymbol); + + if (!startsWithQuote && !endsWithQuote) { + // 如果前后都没有quoteSymbol + return quoteSymbol + name + quoteSymbol; + } else if (startsWithQuote && !endsWithQuote) { + // 如果只有前面有quoteSymbol + return quoteSymbol + quoteSymbol + name + quoteSymbol; + } else if (!startsWithQuote) { + // 如果只有后面有quoteSymbol + return quoteSymbol + name + quoteSymbol + quoteSymbol; + } + // 如果前后都有quoteSymbol，直接返回原字符串 + return name; + } + // 如果name为空或仅包含空白字符，返回原字符串 + return name; + } + + } \ No newline at end of file From 2d8f9baa7791ea3257294cdbedf13c82cbc930da Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 8 Jul 2024 19:17:23 +0800 Subject: [PATCH 43/73] fix(sql-builder): escape line --- .../src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 6538d0149..587f2b564 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -258,7 +258,7 @@ protected void buildTableName(String databaseName, String schemaName, String tab */ public String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList) { String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); - List list = valueList.stream().map(EasyStringUtils::escapeString).toList(); + List list = valueList.stream().map(EasyStringUtils::escapeLineString).toList(); return baseSql + "(" + String.join(",", list) + ");"; } @@ -274,7 +274,7 @@ public String buildSingleInsertSql(String databaseName, String schemaName, Strin public String buildMultiInsertSql(String databaseName, String schemaName, String tableName, List columnList, List> valueLists) { String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); String valuesPart = valueLists.stream() - .map(values -> "(" + String.join(",", values.stream().map(EasyStringUtils::escapeString).toList()) + ")") + .map(values -> "(" + String.join(",", values.stream().map(EasyStringUtils::escapeLineString).toList()) + ")") .collect(Collectors.joining(",\n")); return baseSql + valuesPart + ";"; } From 615c1bfcd503525232bed4c9cb5fc80a7709d558 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 8 Jul 2024 19:22:03 +0800 Subject: [PATCH 44/73] refactor(JDBCDataValue): optimize line separator usage --- .../src/main/java/ai/chat2db/spi/model/JDBCDataValue.java | 1 + 1 file changed, 1 insertion(+) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 7fa9e3568..d1f057741 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -137,6 +137,7 @@ public String getClobString() { String line; while ((line = reader.readLine()) != null) { + // TODO: 优化换行符 builder.append(line).append("\n"); } return builder.toString(); From 5370cea7f1671c2c56f0c07da65643f8b0c530a9 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Mon, 8 Jul 2024 19:34:08 +0800 Subject: [PATCH 45/73] Value processor remove base --- .../chat2db/spi/jdbc/BaseValueProcessor.java | 85 ++++++------------- .../spi/jdbc/DefaultValueProcessor.java | 57 +++++++++++-- .../ai/chat2db/spi/model/AsyncContext.java | 8 +- .../ai/chat2db/spi/util/ResultSetUtils.java | 4 +- .../java/ai/chat2db/spi/util/SqlUtils.java | 2 +- 5 files changed, 83 insertions(+), 73 deletions(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java index 4bc0f90a4..96cfcf802 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/BaseValueProcessor.java @@ -1,61 +1,24 @@ -package ai.chat2db.spi.jdbc; - -import ai.chat2db.server.tools.common.util.EasyStringUtils; -import ai.chat2db.spi.ValueProcessor; -import ai.chat2db.spi.model.JDBCDataValue; -import ai.chat2db.spi.model.SQLDataValue; -import org.apache.commons.lang3.StringUtils; - -import java.util.Objects; - -/** - * @author: zgq - * @date: 2024年05月30日 15:33 - */ -public abstract class BaseValueProcessor implements ValueProcessor { - - @Override - public String getSqlValueString(SQLDataValue dataValue) { - if (Objects.isNull(dataValue.getValue())) { - return "NULL"; - } - return convertSQLValueByType(dataValue); - - } - - - @Override - public String getJdbcValue(JDBCDataValue dataValue) { - Object value = dataValue.getObject(); - if (Objects.isNull(value)) { - return null; - } - if (value instanceof String emptyStr) { - if (StringUtils.isBlank(emptyStr)) { - return emptyStr; - } - } - return convertJDBCValueByType(dataValue); - } - - - @Override - public String getJdbcValueString(JDBCDataValue dataValue) { - Object value = dataValue.getObject(); - if (Objects.isNull(value)) { - return "NULL"; - } - if (value instanceof String stringValue) { - if (StringUtils.isBlank(stringValue)) { - return EasyStringUtils.quoteString(stringValue); - } - } - return convertJDBCValueStrByType(dataValue); - } - - public abstract String convertSQLValueByType(SQLDataValue dataValue); - - public abstract String convertJDBCValueByType(JDBCDataValue dataValue); - - public abstract String convertJDBCValueStrByType(JDBCDataValue dataValue); -} +//package ai.chat2db.spi.jdbc; +// +//import ai.chat2db.server.tools.common.util.EasyStringUtils; +//import ai.chat2db.spi.ValueProcessor; +//import ai.chat2db.spi.model.JDBCDataValue; +//import ai.chat2db.spi.model.SQLDataValue; +//import org.apache.commons.lang3.StringUtils; +// +//import java.util.Objects; +// +///** +// * @author: zgq +// * @date: 2024年05月30日 15:33 +// */ +//public abstract class BaseValueProcessor implements ValueProcessor { +// +// +// +// public abstract String convertSQLValueByType(SQLDataValue dataValue); +// +// public abstract String convertJDBCValueByType(JDBCDataValue dataValue); +// +// public abstract String convertJDBCValueStrByType(JDBCDataValue dataValue); +//} diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java index 8e63debe6..65f43e9fe 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultValueProcessor.java @@ -1,32 +1,75 @@ package ai.chat2db.spi.jdbc; import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import java.util.Objects; + /** * @author: zgq * @date: 2024年05月24日 14:30 */ -public class DefaultValueProcessor extends BaseValueProcessor { +public class DefaultValueProcessor implements ValueProcessor { + + @Override + public String getSqlValueString(SQLDataValue dataValue) { + if (Objects.isNull(dataValue.getValue())) { + return "NULL"; + } + return convertSQLValueByType(dataValue); + + } + + + @Override + public String getJdbcValue(JDBCDataValue dataValue) { +// Object value = dataValue.getObject(); +// if (Objects.isNull(dataValue.getObject())) { +// return null; +// } +// if (value instanceof String emptySry) { +// if (StringUtils.isBlank(emptySry)) { +// return emptySry; +// } +// } + return convertJDBCValueByType(dataValue); + } @Override + public String getJdbcValueString(JDBCDataValue dataValue) { +// Object value = dataValue.getObject(); +// if (Objects.isNull(value)) { +// return "NULL"; +// } +// if (value instanceof String stringValue) { +// if (StringUtils.isBlank(stringValue)) { +// return EasyStringUtils.quoteString(stringValue); +// } +// } + return convertJDBCValueStrByType(dataValue); + } + public String convertSQLValueByType(SQLDataValue dataValue) { return getString(dataValue.getValue()); } - @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { return dataValue.getString(); } - @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return getString(dataValue.getString()); + String value = dataValue.getString(); + if (value == null) { + return "NULL"; + } + return getString(value); } @@ -35,9 +78,9 @@ private boolean isNumber(String value) { } private String getString(String value) { - if (isNumber(value)) { - return value; - } +// if (isNumber(value)) { +// return value; +// } return EasyStringUtils.escapeAndQuoteString(value); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 8f30b3100..5e58a7c85 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -59,7 +59,7 @@ private void asyncCallBack(Context context) { // 更新时间逐渐变长避免频繁更新 callUpdate(); Thread.sleep(2000 * n); - if (n < 15) { + if (n < 5) { n++; } } @@ -98,6 +98,9 @@ public void setProgress(Integer progress) { if (progress == null) { return; } + if (progress > 100) { + progress = 100; + } this.progress = progress; } @@ -110,7 +113,7 @@ public void error(String message) { info.append(message + "\n"); } - public void stop(){ + public void stop() { this.finish = true; } @@ -141,5 +144,4 @@ public void write(String message) { private StringBuffer error = new StringBuffer(); - } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java index 9c872dc04..5856a0a04 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/ResultSetUtils.java @@ -115,7 +115,9 @@ public static String getString(ResultSet rs, int columnIndex){ if (obj == null) { return null; } - if (obj instanceof BigDecimal bigDecimal) { + if(obj instanceof String){ + return (String) obj; + }else if (obj instanceof BigDecimal bigDecimal) { return bigDecimal.toPlainString(); } else if (obj instanceof Double d) { return BigDecimal.valueOf(d).toPlainString(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index 7122acc61..fd2449144 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -293,7 +293,7 @@ private static List split(SqlSplitProcessor processor, String sql, DbTyp String bufferStr = buffer.toString(); if (bufferStr.trim().length() != 0) { // if buffer is not empty, there will be some errors in syntax - log.info("sql processor's buffer is not empty, there may be some errors. buffer={}", bufferStr); +// log.info("sql processor's buffer is not empty, there may be some errors. buffer={}", bufferStr); int lastSqlOffset; if (sqls.size() == 0) { int index = sql.indexOf(bufferStr.trim(), 0); From 50636792a72f6f62618231e54e9912e486c2299b Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Mon, 8 Jul 2024 21:18:39 +0800 Subject: [PATCH 46/73] Value processor remove base --- .../src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index da2bd198f..87648959f 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -222,7 +222,7 @@ public void exportTableData(Connection connection, String databaseName, String s String valueString = valueProcessor.getJdbcValueString(jdbcDataValue); valueList.add(valueString); } - String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, columnList, valueList); + String insertSql = sqlBuilder.buildSingleInsertSql(null, null, tableName, columnList, valueList); asyncContext.write(insertSql); valueList.clear(); } From 874914d689c8fd2f7254eb7807647425d75bb155 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Mon, 8 Jul 2024 22:02:44 +0800 Subject: [PATCH 47/73] Value processor remove base --- .../src/main/java/ai/chat2db/spi/model/AsyncContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java index 5e58a7c85..17e8b5eab 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/AsyncContext.java @@ -98,8 +98,8 @@ public void setProgress(Integer progress) { if (progress == null) { return; } - if (progress > 100) { - progress = 100; + if (progress >= 100) { + progress = 99; } this.progress = progress; } From 19a9060b1e6ff383e986b272bc26763f48f2d449 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Mon, 8 Jul 2024 22:10:31 +0800 Subject: [PATCH 48/73] Some typos and unit tests. --- .../domain/core/impl/ConfigServiceImpl.java | 1 - .../DataSourceAccessBusinessServiceTest.java | 62 ++++++ .../core/DataSourceAccessServiceTest.java | 115 +++++++++++ .../test/core/DataSourceServiceTest.java | 188 ++++++++++++++++++ .../start/test/core/DatabaseServiceTest.java | 2 +- .../test/core/DlTemplateServiceTest.java | 168 ++++++++++++++++ .../test/dialect/OracleDialectProperties.java | 22 +- .../dialect/PostgresqlDialectProperties.java | 11 - 8 files changed, 543 insertions(+), 26 deletions(-) create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessBusinessServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/ConfigServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/ConfigServiceImpl.java index a7d383008..1ebe3757e 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/ConfigServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/ConfigServiceImpl.java @@ -9,7 +9,6 @@ import ai.chat2db.server.domain.core.converter.ConfigConverter; import ai.chat2db.server.domain.repository.Dbutils; import ai.chat2db.server.domain.repository.entity.SystemConfigDO; -import ai.chat2db.server.domain.repository.mapper.ChartMapper; import ai.chat2db.server.domain.repository.mapper.SystemConfigMapper; import ai.chat2db.server.tools.base.wrapper.result.ActionResult; import ai.chat2db.server.tools.base.wrapper.result.DataResult; diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessBusinessServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessBusinessServiceTest.java new file mode 100644 index 000000000..4c0643bba --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessBusinessServiceTest.java @@ -0,0 +1,62 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.DataSource; +import ai.chat2db.server.domain.api.service.DataSourceAccessBusinessService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DataSourceAccessBusinessServiceTest extends TestApplication { + + @Autowired + private DataSourceAccessBusinessService dataSourceAccessBusinessService; + + /** + * 1. First, determine whether it is a private data source (PRIVATE) based on the type of the data source. + * If it is a private data source, determine whether the currently logged-in user is the owner of the data source. + * If so, allow the operation, otherwise throw a permission exception. + *

+ * 2. If the currently logged-in user is an administrator userLoginIdentity(true, **), the operation is allowed. + * If the currently logged-in user is a common user, determine whether the user has permission to access the data source. + * If so, the operation is allowed. + *

+ * 3. If the team to which the currently logged-in user belongs has permission to access the data source, the operation is allowed. + *

+ * 4. If none of the above conditions are met, a permission exception is thrown. + */ + @Test + public void testCheckPermission() { +// userLoginIdentity(false, 3L); + userLoginIdentity(true, 2L); + + DataSource source = new DataSource(); +// source.setKind("PRIVATE"); + source.setKind("SHARED"); + source.setUserId(5L); + source.setId(3L); + + ActionResult actionResult = dataSourceAccessBusinessService.checkPermission(source); + assertNotNull(actionResult); + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessServiceTest.java new file mode 100644 index 000000000..cc82ed378 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceAccessServiceTest.java @@ -0,0 +1,115 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.DataSourceAccess; +import ai.chat2db.server.domain.api.param.datasource.DataSourceSelector; +import ai.chat2db.server.domain.api.param.datasource.access.DataSourceAccessComprehensivePageQueryParam; +import ai.chat2db.server.domain.api.param.datasource.access.DataSourceAccessCreatParam; +import ai.chat2db.server.domain.api.param.datasource.access.DataSourceAccessPageQueryParam; +import ai.chat2db.server.domain.api.param.datasource.access.DataSourceAccessSelector; +import ai.chat2db.server.domain.api.service.DataSourceAccessService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.PageResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DataSourceAccessServiceTest extends TestApplication { + + @Autowired + private DataSourceAccessService dataSourceAccessService; + + @Test + public void testPageQuery() { +// userLoginIdentity(true,5L); + userLoginIdentity(false, 2L); + + + DataSourceAccessPageQueryParam queryParam = new DataSourceAccessPageQueryParam(); + queryParam.setDataSourceId(TestUtils.nextLong()); +// queryParam.setAccessObjectType("TEAM"); + queryParam.setAccessObjectType("USER"); + queryParam.setAccessObjectId(TestUtils.nextLong()); + queryParam.setPageNo(3); + queryParam.setPageSize(5); + + // Returns false by default + queryParam.setEnableReturnCount(true); + + + DataSourceAccessSelector accessSelector = new DataSourceAccessSelector(); + accessSelector.setAccessObject(true); + accessSelector.setDataSource(true); + accessSelector.setDataSourceSelector(new DataSourceSelector(true)); + + PageResult result = dataSourceAccessService.pageQuery(queryParam, accessSelector); + assertNotNull(result); + + } + + @Test + public void testComprehensivePageQuery() { + + userLoginIdentity(false, 2L); +// userLoginIdentity(true,5L); + + DataSourceAccessComprehensivePageQueryParam param = new DataSourceAccessComprehensivePageQueryParam(); + param.setPageNo(1); + param.setPageSize(10); + param.setEnableReturnCount(true); + param.setDataSourceId(TestUtils.nextLong()); + param.setAccessObjectType("USER"); +// param.setAccessObjectType("TEAM"); + param.setAccessObjectId(TestUtils.nextLong()); + param.setUserOrTeamSearchKey("test"); + param.setDataSourceSearchKey("m"); + + DataSourceAccessSelector selector = new DataSourceAccessSelector(); + selector.setAccessObject(true); + selector.setDataSource(true); + selector.setDataSourceSelector(new DataSourceSelector(true)); + + PageResult result = dataSourceAccessService.comprehensivePageQuery(param, selector); + assertNotNull(result); + } + + @Test + public void testCreateAndDelete() { + + userLoginIdentity(false, 8L); +// userLoginIdentity(true,6L); + + DataSourceAccessCreatParam creatParam = new DataSourceAccessCreatParam(); + creatParam.setDataSourceId(TestUtils.nextLong()); + creatParam.setAccessObjectId(TestUtils.nextLong()); + creatParam.setAccessObjectType("USER"); +// creatParam.setAccessObjectType("TEAM"); + + DataResult result = dataSourceAccessService.create(creatParam); + assertNotNull(result); + ActionResult delete = dataSourceAccessService.delete(result.getData()); + assertNotNull(delete); + + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java new file mode 100644 index 000000000..8adbfdb87 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java @@ -0,0 +1,188 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.DataSource; +import ai.chat2db.server.domain.api.param.datasource.*; +import ai.chat2db.server.domain.api.service.DataSourceService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.param.OrderBy; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.base.wrapper.result.PageResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import ai.chat2db.spi.config.DriverConfig; +import ai.chat2db.spi.model.Database; +import ai.chat2db.spi.model.KeyValue; +import ai.chat2db.spi.model.SSHInfo; +import ai.chat2db.spi.model.SSLInfo; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DataSourceServiceTest extends TestApplication { + + @Autowired + private DataSourceService dataSourceService; + + @Autowired + private List dialectPropertiesList; + + @Test + public void testCreateWithPermission() { +// userLoginIdentity(true, 1L); + userLoginIdentity(false, 2L); + + DataSourceCreateParam createParam = new DataSourceCreateParam(); + createParam.setKind("PRIVATE"); +// createParam.setKind("SHARED"); + createParam.setDriverConfig(new DriverConfig()); + + DataResult withPermission = dataSourceService.createWithPermission(createParam); + assertNotNull(withPermission.getData()); + + } + + @Test + public void testUpdateWithPermission() { +// userLoginIdentity(true, 7L); + userLoginIdentity(false, 2L); + + DataSourceUpdateParam updateParam = new DataSourceUpdateParam(); + updateParam.setId(4L); + updateParam.setDriverConfig(new DriverConfig()); + updateParam.setPassword("123456"); + + DataResult result = dataSourceService.updateWithPermission(updateParam); + ActionResult delete = dataSourceService.deleteWithPermission(4L); + assertNotNull(result.getData()); + assertNotNull(delete); + + } + + @Test + public void testQueryById() { + userLoginIdentity(false, 2L); +// userLoginIdentity(true, 7L); + + DataResult result = dataSourceService.queryById(3L); + ListResult dataSourceListResult = dataSourceService.listQuery(new ArrayList<>(), null); + assertNotNull(result.getData()); + assertNotNull(dataSourceListResult.getData()); + } + + @Test + public void testQueryExistent() { + userLoginIdentity(false, 2L); +// userLoginIdentity(true, 7L); + + DataSourceSelector selector = new DataSourceSelector(); + selector.setEnvironment(true); +// selector.setEnvironment(false); + + DataResult result = dataSourceService.queryExistent(3L, null); + assertNotNull( result.getData(),"Data should not be null"); + } + + @Test + public void testCopyByIdWithPermission() { + userLoginIdentity(false, 2L); +// userLoginIdentity(true, 7L); + + DataResult longDataResult = dataSourceService.copyByIdWithPermission(3L); + assertNotNull(longDataResult.getData()); + + } + + @Test + public void testQueryPage() { + userLoginIdentity(false,6L); +// userLoginIdentity(true,9L); + + DataSourcePageQueryParam queryParam = new DataSourcePageQueryParam(); + queryParam.setSearchKey("test"); + queryParam.setPageNo(1); + queryParam.setPageSize(10); + + DataSourceSelector selector = new DataSourceSelector(); + selector.setEnvironment(true); +// selector.setEnvironment(false); + + PageResult result = dataSourceService.queryPage(queryParam, selector); + assertNotNull(result.getData()); + } + + @Test + public void testQueryPageWithPermission() { +// userLoginIdentity(false,3L); + userLoginIdentity(true,9L); + + DataSourcePageQueryParam queryParam = new DataSourcePageQueryParam(); + queryParam.setSearchKey("test"); + queryParam.setKind("PRIVATE"); +// queryParam.setKind("SHARED"); + queryParam.setPageNo(1); + queryParam.setPageSize(10); + queryParam.setOrderByList(new ArrayList()); + + DataSourceSelector selector = new DataSourceSelector(); + selector.setEnvironment(true); + + PageResult result = dataSourceService.queryPageWithPermission(queryParam, selector); + assertNotNull(result.getData()); + + } + + @Test + public void testPreConnect() { + + for (DialectProperties dialectProperties : dialectPropertiesList) { + + DataSourcePreConnectParam param = new DataSourcePreConnectParam(); + param.setType(dialectProperties.getDbType()); + param.setUser(dialectProperties.getUsername()); + param.setUrl(dialectProperties.getUrl()); + param.setPassword(dialectProperties.getPassword()); + param.setPort(String.valueOf(dialectProperties.getPort())); + param.setHost("183.247.151.185"); + param.setSsh(new SSHInfo()); + param.setSsl(new SSLInfo()); + param.setExtendInfo(new ArrayList()); + + ActionResult result = dataSourceService.preConnect(param); + assertNotNull(result); + + Long consoleId= TestUtils.nextLong(); + Long dataSourceId= TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties,dataSourceId,consoleId); + ListResult connect = dataSourceService.connect(dataSourceId); + assertNotNull(connect.getData()); + + dataSourceService.close(dataSourceId); + + } + } + + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java index 04676f380..f619d2098 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DatabaseServiceTest.java @@ -36,7 +36,7 @@ public class DatabaseServiceTest extends TestApplication { @Test public void testQueryAll() { - // MYSQL ORACLE POSTGRESQL + // MYSQL ORACLE POSTGRESQL MONGODB MARIADB for (DialectProperties dialectProperties : dialectPropertiesList) { String dbType = dialectProperties.getDbType(); Long dataSourceId = TestUtils.nextLong(); diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java new file mode 100644 index 000000000..dd96ba4cb --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java @@ -0,0 +1,168 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.param.DlExecuteParam; +import ai.chat2db.server.domain.api.service.DlTemplateService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import ai.chat2db.spi.model.ExecuteResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + + +public class DlTemplateServiceTest extends TestApplication { + + @Autowired + private DlTemplateService dlTemplateService; + + @Autowired + private List dialectPropertiesList; + + // MYSQL: ali_dbhub_test -- test_data + // POSTGRESQL: ali_dbhub_test -- test -- test_data + // ORACLE: TEST_USER -- test_data + @Test + public void testExecute() { + + userLoginIdentity(false, 6L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = 11L; + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + String testData = dialectProperties.getCrateTableSql("test_data006"); + DlExecuteParam dlExecuteParam = new DlExecuteParam(); + dlExecuteParam.setSql(testData); + dlExecuteParam.setConsoleId(consoleId); + dlExecuteParam.setDataSourceId(dataSourceId); + dlExecuteParam.setTableName("test_data006"); + dlExecuteParam.setPageNo(1); + dlExecuteParam.setPageSize(10); + dlExecuteParam.setPageSizeAll(false); + if (dialectProperties.getDbType().equals("POSTGRESQL")) { + dlExecuteParam.setDatabaseName(dialectProperties.getDatabaseName()); + dlExecuteParam.setSchemaName("public"); + } else if (dialectProperties.getDbType().equals("ORACLE")) { + dlExecuteParam.setDatabaseName(""); + dlExecuteParam.setSchemaName("TEST_USER"); + } else if (dialectProperties.getDbType().equals("MYSQL")) { + dlExecuteParam.setDatabaseName(dialectProperties.getDatabaseName()); + dlExecuteParam.setSchemaName(""); + } else { + continue; + } + + + ListResult execute = dlTemplateService.execute(dlExecuteParam); + Assertions.assertTrue(execute.getSuccess(), execute.errorMessage()); + + } + + } + + @Test + public void testExecuteSelectTable() { + + userLoginIdentity(false, 3L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = 20858L; + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + if (dialectProperties.getDbType().equals("MYSQL")) { + DlExecuteParam dlExecuteParam = new DlExecuteParam(); + dlExecuteParam.setConsoleId(consoleId); + dlExecuteParam.setDataSourceId(dataSourceId); + dlExecuteParam.setTableName("test_data004"); + dlExecuteParam.setPageNo(1); + dlExecuteParam.setPageSize(10); + dlExecuteParam.setPageSizeAll(false); + + ListResult execute = dlTemplateService.executeSelectTable(dlExecuteParam); + Assertions.assertTrue(execute.getSuccess(), execute.errorMessage()); + } + + } + } + + @Test + public void testExecuteUpdate() { + + userLoginIdentity(false, 7L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + String testData = dialectProperties.getInsertSql("test_data006", new Timestamp(new Date().getTime()), 1L, "test"); + DlExecuteParam dlExecuteParam = new DlExecuteParam(); + dlExecuteParam.setSql(testData); + dlExecuteParam.setConsoleId(consoleId); + dlExecuteParam.setDataSourceId(dataSourceId); + dlExecuteParam.setTableName("test_data006"); + dlExecuteParam.setPageNo(1); + dlExecuteParam.setPageSize(10); + dlExecuteParam.setPageSizeAll(false); + if (dialectProperties.getDbType().equals("POSTGRESQL")) { + dlExecuteParam.setDatabaseName(dialectProperties.getDatabaseName()); + dlExecuteParam.setSchemaName("public"); + } else if (dialectProperties.getDbType().equals("ORACLE")) { + dlExecuteParam.setDatabaseName(""); + dlExecuteParam.setSchemaName("TEST_USER"); + } else if (dialectProperties.getDbType().equals("MYSQL")) { + dlExecuteParam.setDatabaseName(dialectProperties.getDatabaseName()); + dlExecuteParam.setSchemaName(""); + } else { + continue; + } + + + DataResult result = dlTemplateService.executeUpdate(dlExecuteParam); + Assertions.assertTrue(result.getSuccess(), result.errorMessage()); + + } + } + + @Test + public void testCount() { + + } + + @Test + public void testUpdateSelectResult() { + + } + + @Test + public void testGetOrderBySql() { + + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java index 0eac03a52..58f86e4ef 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java @@ -16,12 +16,12 @@ public String getDbType() { @Override public String getUrl() { - return "jdbc:oracle:thin:@183.247.151.185:11521:XE"; + return "jdbc:oracle:thin:@192.168.0.120:1521:XE"; } @Override public String getErrorUrl() { - return "jdbc:oracle:thin:@183.247.151.185:11521:XE1"; + return "jdbc:oracle:thin:@192.168.0.120:1521:XE1"; } @Override @@ -41,16 +41,12 @@ public String getDatabaseName() { @Override public String getCrateTableSql(String tableName) { - return "CREATE TABLE `" + tableName + "`\n\t" - + "(\n\t" - + " `id` bigint PRIMARY KEY AUTO_INCREMENT NOT NULL COMMENT 'Primary key auto-increment',\n\t" - + " `date` datetime(3) not null COMMENT 'date',\n\t" - + " `number` bigint COMMENT 'long integer',\n\t" - + " `string` VARCHAR(100) default 'DATA' COMMENT 'name',\n\t" - + " index " + tableName + "_idx_date (date desc) comment 'date index',\n\t" - + " unique " + tableName + "_uk_number (number) comment 'unique index',\n\t" - + " index " + tableName + "_idx_number_string (number, date) comment 'Union index'\n\t" - + ") COMMENT ='Test table';"; + return "CREATE TABLE TEST_USER." + tableName + " (\n" + + " id NUMBER PRIMARY KEY,\n" + + " created_date DATE,\n" + + " amount INT,\n" + + " string VARCHAR2(100)\n" + + ");"; } @Override @@ -60,7 +56,7 @@ public String getDropTableSql(String tableName) { @Override public String getInsertSql(String tableName, Date date, Long number, String string) { - return "INSERT INTO `" + tableName + "` (date,number,string) VALUES ('" + DateUtil.format(date, + return "INSERT INTO TEST_USER." + tableName + " (date,number,string) VALUES ('" + DateUtil.format(date, DatePattern.NORM_DATETIME_MS_FORMAT) + "','" + number + "','" + string + "');"; } diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java index ac81c7cb1..8c7172fc4 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java @@ -58,17 +58,6 @@ public String getCrateTableSql(String tableName) { + " number int,\n" + " string varchar(100) default 'DATA'\n" + ");\n"; - sql += "comment on table " + tableName + " is 'Test table';\n"; - sql += "comment on column " + tableName + ".id is 'Primary key auto-increment';\n"; - sql += "comment on column " + tableName + ".date is 'date';\n"; - sql += "comment on column " + tableName + ".number is 'long integer';\n"; - sql += "comment on column " + tableName + ".string is 'name';\n"; - sql += "create index " + tableName + "idx_date on " + tableName + " (date desc);"; - sql += "create unique index " + tableName + "_uk_number on " + tableName + " (number);"; - sql += "create index " + tableName + "_idx_number_string on " + tableName + " (number, date);"; - sql += "comment on index " + tableName + "_uk_number is 'date index';"; - sql += "comment on index " + tableName + "_uk_number is 'unique index';"; - sql += "comment on index " + tableName + "_idx_number_string is 'Union index';"; return sql; } From 6552f2dc5005ee9d9b382521809fc69201655c66 Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Mon, 8 Jul 2024 22:20:21 +0800 Subject: [PATCH 49/73] Some typos and unit tests. --- .../server/start/test/dialect/MariadbDialectProperties.java | 4 ++-- .../server/start/test/dialect/MongodbDialectProperties.java | 2 +- .../server/start/test/dialect/MysqlDialectProperties.java | 4 ++-- .../server/start/test/dialect/OracleDialectProperties.java | 4 ++-- .../start/test/dialect/PostgresqlDialectProperties.java | 4 ++-- .../java/ai/chat2db/server/start/test/dialect/TestUtils.java | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java index f4eedd53a..bb2a6a9b3 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MariadbDialectProperties.java @@ -14,12 +14,12 @@ public String getDbType() { @Override public String getUrl() { - return "jdbc:mariadb://183.247.151.185:13303/"; + return "jdbc:mariadb://localhost:3303/"; } @Override public String getErrorUrl() { - return "jdbc:mariadb://error:13303/"; + return "jdbc:mariadb://error:3303/"; } @Override diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java index b687e9a23..9ba2944bc 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MongodbDialectProperties.java @@ -14,7 +14,7 @@ public String getDbType() { @Override public String getUrl() { - return "mongodb://183.247.151.185:27017/"; + return "mongodb://localhost:27017/"; } @Override diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java index 1146f45d6..f8d821ce9 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/MysqlDialectProperties.java @@ -22,12 +22,12 @@ public String getDbType() { @Override public String getUrl() { - return "jdbc:mysql://183.247.151.185:13306"; + return "jdbc:mysql://localhost:3306"; } @Override public String getErrorUrl() { - return "jdbc:mysql://error.rm-8vb099vo8309mcngk.mysql.zhangbei.rds.aliyuncs.com:13306"; + return "jdbc:mysql://error.rm-8vb099vo8309mcngk.mysql.zhangbei.rds.aliyuncs.com:3306"; } @Override diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java index 58f86e4ef..2719d26d3 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/OracleDialectProperties.java @@ -16,12 +16,12 @@ public String getDbType() { @Override public String getUrl() { - return "jdbc:oracle:thin:@192.168.0.120:1521:XE"; + return "jdbc:oracle:thin:@localhost:1521:XE"; } @Override public String getErrorUrl() { - return "jdbc:oracle:thin:@192.168.0.120:1521:XE1"; + return "jdbc:oracle:thin:@localhost:1521:XE1"; } @Override diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java index 8c7172fc4..9a9bc8cfa 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/PostgresqlDialectProperties.java @@ -25,12 +25,12 @@ public String getDbType() { @Override public String getUrl() { - return "jdbc:postgresql://183.247.151.185:15431/ali_dbhub_test"; + return "jdbc:postgresql://localhost:5431/ali_dbhub_test"; } @Override public String getErrorUrl() { - return "jdbc:postgresql://error:15431/ali_dbhub"; + return "jdbc:postgresql://error:5431/ali_dbhub"; } @Override diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java index 6003c8adc..c9de2812a 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/dialect/TestUtils.java @@ -49,7 +49,7 @@ public static void buildContext(DialectProperties dialectProperties,Long dataSou ConnectInfo connectInfo = new ConnectInfo(); connectInfo.setUser(dialectProperties.getUsername()); connectInfo.setPort(dialectProperties.getPort()); - connectInfo.setHost("183.247.151.185"); + connectInfo.setHost("localhost"); connectInfo.setSsh(new SSHInfo()); connectInfo.setConsoleId(consoleId); connectInfo.setDataSourceId(dataSourceId); From 7a95f126b290586557c706bd69a0c509386f846e Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Tue, 9 Jul 2024 10:48:18 +0800 Subject: [PATCH 50/73] remove ';' in builder sql --- .../chat2db/plugin/clickhouse/ClickHouseMetaData.java | 2 +- .../ai/chat2db/plugin/sqlserver/SqlServerDBManage.java | 2 +- .../api/controller/rdb/data/sql/SqlDataExporter.java | 10 +++++++--- .../main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java | 2 +- .../java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java | 6 +++--- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index 6ebdd82b4..ff85f7142 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -61,7 +61,7 @@ public List functions(Connection connection, String databaseName, Stri @Override public List databases(Connection connection) { - List list = SQLExecutor.getInstance().execute(connection, "SELECT name FROM system.databases;;", resultSet -> { + List list = SQLExecutor.getInstance().execute(connection, "SELECT name FROM system.databases;", resultSet -> { List databases = new ArrayList<>(); try { while (resultSet.next()) { diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 36331e708..1fb2997fb 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -112,7 +112,7 @@ public void exportTableData(Connection connection, String databaseName, String s valueList.add(valueString); } String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, columnList, valueList); - asyncContext.write(insertSql); + asyncContext.write(insertSql+";"); valueList.clear(); } asyncContext.write("go \n"); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java index bc2a330ad..8ae1cf719 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java @@ -12,6 +12,7 @@ import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.util.ResultSetUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Component; import java.io.*; @@ -94,11 +95,14 @@ private void exportSingleInsert(Connection connection, String querySql, Boolean while (resultSet.next()) { List rowData = extractRowData(resultSet, valueProcessor); String sql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, header, rowData); - sqlList.add(sql); - if (sqlList.size() >= BATCH_SIZE || resultSet.isLast()) { + sqlList.add(sql+";"); + if (sqlList.size() >= BATCH_SIZE) { writeSqlList(writer, sqlList); } } + if(CollectionUtils.isNotEmpty(sqlList)){ + writeSqlList(writer, sqlList); + } }); TaskManager.increaseCurrent(); } @@ -113,7 +117,7 @@ private void exportMultiInsert(Connection connection, String querySql, Boolean c dataList.add(extractRowData(resultSet, valueProcessor)); } String sql = sqlBuilder.buildMultiInsertSql(databaseName, schemaName, tableName, header, dataList); - writer.println(sql); + writer.println(sql+";"); writer.flush(); }); TaskManager.increaseCurrent(); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 87648959f..4d6a79312 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -223,7 +223,7 @@ public void exportTableData(Connection connection, String databaseName, String s valueList.add(valueString); } String insertSql = sqlBuilder.buildSingleInsertSql(null, null, tableName, columnList, valueList); - asyncContext.write(insertSql); + asyncContext.write(insertSql+";"); valueList.clear(); } }); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 587f2b564..6498efb91 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -259,7 +259,7 @@ protected void buildTableName(String databaseName, String schemaName, String tab public String buildSingleInsertSql(String databaseName, String schemaName, String tableName, List columnList, List valueList) { String baseSql = buildBaseInsertSql(databaseName, schemaName, tableName, columnList); List list = valueList.stream().map(EasyStringUtils::escapeLineString).toList(); - return baseSql + "(" + String.join(",", list) + ");"; + return baseSql + "(" + String.join(",", list) + ")"; } /** @@ -276,7 +276,7 @@ public String buildMultiInsertSql(String databaseName, String schemaName, String String valuesPart = valueLists.stream() .map(values -> "(" + String.join(",", values.stream().map(EasyStringUtils::escapeLineString).toList()) + ")") .collect(Collectors.joining(",\n")); - return baseSql + valuesPart + ";"; + return baseSql + valuesPart; } @@ -299,7 +299,7 @@ public String buildUpdateSql(String databaseName, String schemaName, String tabl .collect(Collectors.toList()); script.append(String.join(" AND ", whereClauses)); } - return script + ";"; + return script.toString(); } private List getPrimaryColumns(List

headerList) { From ab96987d928a9f118a7860372b2f913e304cdb34 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Tue, 9 Jul 2024 11:52:53 +0800 Subject: [PATCH 51/73] remove ';' in builder sql --- .../chat2db/server/tools/common/util/EasyStringUtils.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 9db945304..ff2bd4d4c 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -234,9 +234,11 @@ public static String escapeLineString(String str) { if (StringUtils.isBlank(str)) { return str; } - return str.replace("\r\n", "\\r\\n") - .replace("\n", "\\n") - .replace("\r", "\\r"); + return str; + // TODO Need to be implemented in the future with different data types +// return str.replace("\r\n", "\\r\\n") +// .replace("\n", "\\n") +// .replace("\r", "\\r"); } } From 0f23e12e50770ef1ab1d6015948ed29371315ee8 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 11:58:08 +0800 Subject: [PATCH 52/73] refactor(Oraclesql-builder): wrap objectName --- .../oracle/builder/OracleSqlBuilder.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java index ef93beffd..a81e9c0ae 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/builder/OracleSqlBuilder.java @@ -6,8 +6,13 @@ import ai.chat2db.spi.model.Table; import ai.chat2db.spi.model.TableColumn; import ai.chat2db.spi.model.TableIndex; +import ai.chat2db.spi.util.SqlUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import java.util.List; +import java.util.stream.Collectors; + public class OracleSqlBuilder extends DefaultSqlBuilder { @Override public String buildCreateTableSql(Table table) { @@ -149,4 +154,26 @@ public String pageLimit(String sql, int offset, int pageNo, int pageSize) { // } // return sqlBuilder.toString(); // } + + + @Override + protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { + if (StringUtils.isNotBlank(databaseName)) { + script.append(SqlUtils.quoteObjectName(databaseName)).append('.'); + } + script.append(SqlUtils.quoteObjectName(tableName)); + } + + /** + * @param columnList + * @param script + */ + @Override + protected void buildColumns(List columnList, StringBuilder script) { + if (CollectionUtils.isNotEmpty(columnList)) { + script.append(" (") + .append(columnList.stream().map(SqlUtils::quoteObjectName).collect(Collectors.joining(","))) + .append(") "); + } + } } From ea5a7e89fff22a6f7cae69cf8f9df6b9bc2c7afb Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 14:35:13 +0800 Subject: [PATCH 53/73] refactor(OracleDBManage): fix get tableDDL --- .../chat2db/plugin/oracle/OracleDBManage.java | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index c2d9af998..e2407ae12 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -12,26 +12,23 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.Objects; @Slf4j public class OracleDBManage extends DefaultDBManage implements DBManage { - private static String TABLE_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TABLE', table_name) as ddl FROM all_tables WHERE owner = '%s' AND table_name = '%s'"; private static String TABLE_COMMENT_SQL = "SELECT 'COMMENT ON TABLE ' || table_name || ' IS ''' || comments || ''';' AS table_comment_ddl FROM user_tab_comments WHERE table_name = '%s'"; private static String TABLE_COLUMN_COMMENT_SQL = "SELECT 'COMMENT ON COLUMN ' || table_name || '.' || column_name || ' IS ''' || comments || ''';' AS column_comment_ddl " + "FROM user_col_comments " + "WHERE table_name = '%s' " + "AND comments IS NOT NULL"; private static String VIEW_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('VIEW', view_name) as ddl FROM all_views WHERE owner = '%s' AND view_name = '%s'"; - private String PROCEDURE_LIST_DDL ="SELECT object_name FROM all_procedures where OWNER = '%s' and OBJECT_TYPE='PROCEDURE'"; + private String PROCEDURE_LIST_DDL = "SELECT object_name FROM all_procedures where OWNER = '%s' and OBJECT_TYPE='PROCEDURE'"; private static String PROCEDURE_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('PROCEDURE', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; private static String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', trigger_name) AS ddl FROM all_triggers WHERE owner = '%s' AND trigger_name = '%s'"; private static String FUNCTION_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('FUNCTION', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; - public void exportDatabase(Connection connection, String databaseName, String schemaName,AsyncContext asyncContext) throws SQLException { - exportTables(connection,databaseName, schemaName, asyncContext); + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, asyncContext, schemaName); exportProcedures(connection, schemaName, asyncContext); exportTriggers(connection, schemaName, asyncContext); @@ -42,27 +39,25 @@ private void exportTables(Connection connection, String databaseName, String sch try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"TABLE", "SYSTEM TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); - exportTable(connection,databaseName, schemaName, tableName, asyncContext); + exportTable(connection, databaseName, schemaName, tableName, asyncContext); } } } public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(TABLE_DDL_SQL, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("DROP TABLE ").append(schemaName).append(".").append(tableName).append(";") - .append(resultSet.getString("ddl")).append(";").append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - exportTableComments(connection, tableName, asyncContext); - exportTableColumnsComments(connection, tableName, asyncContext); - if (asyncContext.isContainsData()) { - exportTableData(connection,databaseName,schemaName, tableName, asyncContext); - } + String tableDDL = Chat2DBContext.getMetaData().tableDDL(connection, databaseName, schemaName, tableName); + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("DROP TABLE ").append(schemaName).append(".").append(tableName).append(";") + .append(tableDDL).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); + + exportTableComments(connection, tableName, asyncContext); + exportTableColumnsComments(connection, tableName, asyncContext); + if (asyncContext.isContainsData()) { + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } + } private void exportTableComments(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { @@ -108,8 +103,8 @@ private void exportView(Connection connection, AsyncContext asyncContext, String } private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(PROCEDURE_LIST_DDL,schemaName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + String sql = String.format(PROCEDURE_LIST_DDL, schemaName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { while (resultSet.next()) { String procedureName = resultSet.getString("object_name"); exportProcedure(connection, schemaName, procedureName, asyncContext); @@ -185,11 +180,11 @@ public void connectDatabase(Connection connection, String database) { } @Override - public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName,boolean copyData) throws SQLException { + public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName, boolean copyData) throws SQLException { String sql = ""; - if(copyData){ + if (copyData) { sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName; - }else { + } else { sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName + " WHERE 1=0"; } SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); From 14777c784cf59aba81275775ec2ee51ad26737de Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 15:02:22 +0800 Subject: [PATCH 54/73] fix(JDBCDataValue): handle null bytes in getBlobHexString Avoid NullPointerException by checking for null byte array before encoding in getBlobHexString method. fix(OracleDBManage): properly quote object names in SQL builder --- .../main/java/ai/chat2db/plugin/oracle/OracleDBManage.java | 4 +++- .../src/main/java/ai/chat2db/spi/model/JDBCDataValue.java | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index e2407ae12..b32d09ca3 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -1,11 +1,13 @@ package ai.chat2db.plugin.oracle; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; import ai.chat2db.spi.model.AsyncContext; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; +import ai.chat2db.spi.util.SqlUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -48,7 +50,7 @@ private void exportTables(Connection connection, String databaseName, String sch public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String tableDDL = Chat2DBContext.getMetaData().tableDDL(connection, databaseName, schemaName, tableName); StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("DROP TABLE ").append(schemaName).append(".").append(tableName).append(";") + sqlBuilder.append("DROP TABLE ").append(SqlUtils.quoteObjectName(tableName)).append(";") .append(tableDDL).append(";").append("\n"); asyncContext.write(sqlBuilder.toString()); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index d1f057741..8ff42015c 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -95,7 +95,11 @@ public Blob getBlob() { } public String getBlobHexString() { - return BaseEncoding.base16().encode(getBytes()); + byte[] bytes = getBytes(); + if (Objects.isNull(bytes)) { + return "NULL"; + } + return BaseEncoding.base16().encode(bytes); } public BigDecimal getBigDecimal() { From 14b5ba519c01bc78b34dd515959fcdb0fdb7ab0a Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 16:04:58 +0800 Subject: [PATCH 55/73] fix(chat2db): fix oracle long raw null --- .../plugin/oracle/value/sub/OracleLongRawProcessor.java | 8 +++++++- .../src/main/java/ai/chat2db/spi/model/JDBCDataValue.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java index 2d030319b..6f3634c10 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java @@ -5,6 +5,8 @@ import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; +import java.util.Objects; + /** * @author: zgq * @date: 2024年07月07日 16:58 @@ -25,7 +27,11 @@ public String convertJDBCValueByType(JDBCDataValue dataValue) { @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + String blobHexString = dataValue.getBlobHexString(); + if (Objects.isNull(blobHexString)) { + return "NULL"; + } + return EasyStringUtils.quoteString(blobHexString); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 8ff42015c..685b3308a 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -97,7 +97,7 @@ public Blob getBlob() { public String getBlobHexString() { byte[] bytes = getBytes(); if (Objects.isNull(bytes)) { - return "NULL"; + return null; } return BaseEncoding.base16().encode(bytes); } From fbf34e7aa8fcc27010cfdd4d118050912322a333 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 17:27:13 +0800 Subject: [PATCH 56/73] fix(chat2db-oracle): correct timestamp scale --- .../oracle/value/sub/OracleTimeStampTZProcessor.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java index 182ea8582..99f4af54e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleTimeStampTZProcessor.java @@ -21,12 +21,14 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - //2024-06-05 17:41:27.52 +0:00 -> String timeStampString = dataValue.getStringValue(); int scale = dataValue.getScale(); int lastSpaceIndex = timeStampString.lastIndexOf(" "); - int nanosLength = lastSpaceIndex - timeStampString.indexOf(".") - 1; - if (scale != 0 && nanosLength < scale) { + int lastDotIndex = timeStampString.indexOf("."); + int nanosLength = lastSpaceIndex - lastDotIndex - 1; + if (scale == 0) { + return timeStampString.substring(0, lastDotIndex) + timeStampString.substring(lastDotIndex + 2); + } else if (nanosLength < scale) { // 计算需要补充的零的数量 int zerosToAdd = scale - nanosLength; StringBuilder sb = new StringBuilder(timeStampString); From d5c72b955c8a3a22a8e19dc54cb23b16e41eda87 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 20:51:49 +0800 Subject: [PATCH 57/73] fix(chat2db-oracle): fix schema struct export --- .../chat2db/plugin/oracle/OracleDBManage.java | 72 ++++------- .../chat2db/plugin/oracle/OracleMetaData.java | 115 ++++++++++-------- 2 files changed, 85 insertions(+), 102 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index b32d09ca3..62ab54339 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -1,20 +1,21 @@ package ai.chat2db.plugin.oracle; -import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.DBManage; import ai.chat2db.spi.jdbc.DefaultDBManage; -import ai.chat2db.spi.model.AsyncContext; +import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.util.SqlUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; @Slf4j public class OracleDBManage extends DefaultDBManage implements DBManage { @@ -23,12 +24,6 @@ public class OracleDBManage extends DefaultDBManage implements DBManage { "FROM user_col_comments " + "WHERE table_name = '%s' " + "AND comments IS NOT NULL"; - private static String VIEW_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('VIEW', view_name) as ddl FROM all_views WHERE owner = '%s' AND view_name = '%s'"; - private String PROCEDURE_LIST_DDL = "SELECT object_name FROM all_procedures where OWNER = '%s' and OBJECT_TYPE='PROCEDURE'"; - private static String PROCEDURE_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('PROCEDURE', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; - private static String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', trigger_name) AS ddl FROM all_triggers WHERE owner = '%s' AND trigger_name = '%s'"; - private static String FUNCTION_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('FUNCTION', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; - public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, asyncContext, schemaName); @@ -93,36 +88,26 @@ private void exportViews(Connection connection, AsyncContext asyncContext, Strin } } - private void exportView(Connection connection, AsyncContext asyncContext, String schemaName, String viewName) throws SQLException { - String sql = String.format(VIEW_DDL_SQL, schemaName, viewName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("ddl")).append(";").append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } + private void exportView(Connection connection, AsyncContext asyncContext, String schemaName, String viewName) { + Table view = Chat2DBContext.getMetaData().view(connection, null, schemaName, viewName); + asyncContext.write(view.getDdl() + ";" + "\n"); } - private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(PROCEDURE_LIST_DDL, schemaName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - while (resultSet.next()) { - String procedureName = resultSet.getString("object_name"); + private void exportProcedures(Connection connection, String schemaName, AsyncContext asyncContext) { + List procedures = Chat2DBContext.getMetaData().procedures(connection, null, schemaName); + if (CollectionUtils.isNotEmpty(procedures)) { + for (Procedure procedure : procedures) { + String procedureName = procedure.getProcedureName(); exportProcedure(connection, schemaName, procedureName, asyncContext); } } + } - private void exportProcedure(Connection connection, String schemaName, String procedureName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(PROCEDURE_DDL_SQL, schemaName, procedureName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("ddl")).append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } + private void exportProcedure(Connection connection, String schemaName, String procedureName, AsyncContext asyncContext) { + Procedure procedure = Chat2DBContext.getMetaData().procedure(connection, null, schemaName, procedureName); + asyncContext.write(procedure.getProcedureBody() + "\n"); + } private void exportTriggers(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { @@ -135,15 +120,10 @@ private void exportTriggers(Connection connection, String schemaName, AsyncConte } } - private void exportTrigger(Connection connection, String schemaName, String triggerName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(TRIGGER_DDL_SQL, schemaName, triggerName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("ddl")).append(";").append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } + private void exportTrigger(Connection connection, String schemaName, String triggerName, AsyncContext asyncContext) { + Trigger trigger = Chat2DBContext.getMetaData().trigger(connection, null, schemaName, triggerName); + asyncContext.write(trigger.getTriggerBody() + ";" + "\n"); + } private void exportFunctions(Connection connection, String schemaName, AsyncContext asyncContext) throws SQLException { @@ -155,15 +135,9 @@ private void exportFunctions(Connection connection, String schemaName, AsyncCont } } - private void exportFunction(Connection connection, String schemaName, String functionName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(FUNCTION_DDL_SQL, schemaName, functionName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("ddl")).append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } + private void exportFunction(Connection connection, String schemaName, String functionName, AsyncContext asyncContext) { + Function function = Chat2DBContext.getMetaData().function(connection, null, schemaName, functionName); + asyncContext.write(function.getFunctionBody() + "\n"); } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 27a67b639..48cba1495 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -1,12 +1,5 @@ package ai.chat2db.plugin.oracle; -import java.io.Reader; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; -import java.util.stream.Collectors; - import ai.chat2db.plugin.oracle.builder.OracleSqlBuilder; import ai.chat2db.plugin.oracle.type.OracleColumnTypeEnum; import ai.chat2db.plugin.oracle.type.OracleDefaultValueEnum; @@ -24,20 +17,31 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.io.Reader; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; + @Slf4j public class OracleMetaData extends DefaultMetaService implements MetaData { private static final String TABLE_DDL_SQL = "select dbms_metadata.get_ddl('TABLE','%s','%s') as sql from dual"; - private List systemSchemas = Arrays.asList("ANONYMOUS","APEX_030200","APEX_PUBLIC_USER","APPQOSSYS","BI","CTXSYS","DBSNMP","DIP","EXFSYS","FLOWS_FILES","HR","IX","MDDATA","MDSYS","MGMT_VIEW","OE","OLAPSYS","ORACLE_OCM","ORDDATA","ORDPLUGINS","ORDSYS","OUTLN","OWBSYS","OWBSYS_AUDIT","PM","SCOTT","SH","SI_INFORMTN_SCHEMA","SPATIAL_CSW_ADMIN_USR","SPATIAL_WFS_ADMIN_USR","SYS","SYSMAN","SYSTEM","WMSYS","XDB","XS$NULL"); + private List systemSchemas = Arrays.asList("ANONYMOUS", "APEX_030200", "APEX_PUBLIC_USER", "APPQOSSYS", "BI", "CTXSYS", "DBSNMP", "DIP", "EXFSYS", "FLOWS_FILES", "HR", "IX", "MDDATA", "MDSYS", "MGMT_VIEW", "OE", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "OWBSYS_AUDIT", "PM", "SCOTT", "SH", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "WMSYS", "XDB", "XS$NULL"); - private String PROCEDURE_LIST_DDL ="SELECT object_name FROM all_procedures where OWNER = '%s' and OBJECT_TYPE='PROCEDURE'"; + private String PROCEDURE_LIST_DDL = """ + SELECT OBJECT_NAME, OBJECT_TYPE + FROM ALL_OBJECTS + WHERE OBJECT_TYPE IN ('PROCEDURE') + AND OWNER = '%s'"""; @Override public List procedures(Connection connection, String databaseName, String schemaName) { - String sql = String.format(PROCEDURE_LIST_DDL,schemaName); + String sql = String.format(PROCEDURE_LIST_DDL, schemaName); ArrayList procedures = new ArrayList<>(); - SQLExecutor.getInstance().execute(connection, sql, resultSet->{ + SQLExecutor.getInstance().execute(connection, sql, resultSet -> { while (resultSet.next()) { Procedure procedure = new Procedure(); procedure.setProcedureName(resultSet.getString("object_name")); @@ -108,7 +112,7 @@ public List columns(Connection connection, String databaseName, Str // // Fields of the LONG type cannot be retrieved using getObject. They need to be accessed using getCharacterStream, and must be read first in the sequence. Reader reader = resultSet.getCharacterStream("DATA_DEFAULT"); - if(reader != null){ + if (reader != null) { StringBuilder sb = new StringBuilder(); int charValue; while ((charValue = reader.read()) != -1) { @@ -116,19 +120,19 @@ public List columns(Connection connection, String databaseName, Str } tableColumn.setDefaultValue(sb.toString()); } - }catch (Exception e){ - log.error("getDefaultValue error",e); + } catch (Exception e) { + log.error("getDefaultValue error", e); } tableColumn.setName(resultSet.getString("COLUMN_NAME")); String dataType = resultSet.getString("DATA_TYPE"); - if(dataType.contains("(")){ - dataType = dataType.substring(0,dataType.indexOf("(")).trim(); + if (dataType.contains("(")) { + dataType = dataType.substring(0, dataType.indexOf("(")).trim(); } tableColumn.setColumnType(dataType); Integer dataPrecision = resultSet.getInt("DATA_PRECISION"); - if(resultSet.getString("DATA_PRECISION") != null) { + if (resultSet.getString("DATA_PRECISION") != null) { tableColumn.setColumnSize(dataPrecision); - }else { + } else { tableColumn.setColumnSize(resultSet.getInt("DATA_LENGTH")); } // Object dataDefault = resultSet.getObject(7); @@ -137,8 +141,6 @@ public List columns(Connection connection, String databaseName, Str // } - - tableColumn.setComment(resultSet.getString("COMMENTS")); tableColumn.setNullable("Y".equalsIgnoreCase(resultSet.getString("NULLABLE")) ? 1 : 0); tableColumn.setOrdinalPosition(resultSet.getInt("COLUMN_ID")); @@ -155,18 +157,23 @@ public List columns(Connection connection, String databaseName, Str }); } - private static String FUNCTION_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('FUNCTION', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; + private static String ROUTINES_SQL + = "SELECT LINE, TEXT " + + "FROM ALL_SOURCE " + + "WHERE TYPE = '%s' AND OWNER = '%s' AND NAME = '%s'" + + "ORDER BY LINE"; + @Override public Function function(Connection connection, @NotEmpty String databaseName, String schemaName, String functionName) { - String sql = String.format(FUNCTION_DDL_SQL, schemaName, functionName); + String sql = String.format(ROUTINES_SQL, "FUNCTION", schemaName, functionName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { Function function = new Function(); function.setDatabaseName(databaseName); function.setSchemaName(schemaName); function.setFunctionName(functionName); if (resultSet.next()) { - function.setFunctionBody(resultSet.getString("ddl")); + function.setFunctionBody("CREATE " + resultSet.getString("TEXT")); } return function; @@ -188,11 +195,11 @@ public List indexes(Connection connection, String databaseName, Stri String pkSql = String.format(SELECT_PK_SQL, schemaName, tableName); Set pkSet = new HashSet<>(); SQLExecutor.getInstance().execute(connection, pkSql, resultSet -> { - while (resultSet.next()) { - pkSet.add(resultSet.getString("CONSTRAINT_NAME")); - } - return null; - } + while (resultSet.next()) { + pkSet.add(resultSet.getString("CONSTRAINT_NAME")); + } + return null; + } ); String sql = String.format(SELECT_TABLE_INDEX, schemaName, tableName); @@ -255,55 +262,57 @@ private TableIndexColumn getTableIndexColumn(ResultSet resultSet) throws SQLExce public List triggers(Connection connection, String databaseName, String schemaName) { List triggers = new ArrayList<>(); return SQLExecutor.getInstance().execute(connection, String.format(TRIGGER_SQL_LIST, schemaName), - resultSet -> { - while (resultSet.next()) { - String triggerName = resultSet.getString("TRIGGER_NAME"); - Trigger trigger = new Trigger(); - trigger.setTriggerName(triggerName==null?"":triggerName.trim()); - trigger.setSchemaName(schemaName); - trigger.setDatabaseName(databaseName); - triggers.add(trigger); - } - return triggers; - }); + resultSet -> { + while (resultSet.next()) { + String triggerName = resultSet.getString("TRIGGER_NAME"); + Trigger trigger = new Trigger(); + trigger.setTriggerName(triggerName == null ? "" : triggerName.trim()); + trigger.setSchemaName(schemaName); + trigger.setDatabaseName(databaseName); + triggers.add(trigger); + } + return triggers; + }); } - private static String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', trigger_name) AS ddl FROM all_triggers WHERE owner = '%s' AND trigger_name = '%s'"; + + private static final String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', '%s', '%s') as ddl FROM DUAL"; + @Override public Trigger trigger(Connection connection, @NotEmpty String databaseName, String schemaName, String triggerName) { - - - String sql = String.format(TRIGGER_DDL_SQL, schemaName, triggerName); + String sql = String.format(TRIGGER_DDL_SQL, triggerName, schemaName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { Trigger trigger = new Trigger(); trigger.setDatabaseName(databaseName); trigger.setSchemaName(schemaName); trigger.setTriggerName(triggerName); - if (resultSet.next()) { + while (resultSet.next()) { trigger.setTriggerBody(resultSet.getString("ddl")); } return trigger; }); } - private static String PROCEDURE_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('PROCEDURE', object_name) as ddl FROM all_procedures WHERE owner = '%s' AND object_name = '%s'"; + @Override public Procedure procedure(Connection connection, @NotEmpty String databaseName, String schemaName, String procedureName) { - String sql = String.format(PROCEDURE_DDL_SQL, schemaName, procedureName); + String sql = String.format(ROUTINES_SQL, "PROCEDURE", schemaName, procedureName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { Procedure procedure = new Procedure(); - if (resultSet.next()) { - procedure.setDatabaseName(databaseName); - procedure.setSchemaName(schemaName); - procedure.setProcedureName(procedureName); - procedure.setProcedureBody(resultSet.getString("ddl")); + procedure.setDatabaseName(databaseName); + procedure.setSchemaName(schemaName); + procedure.setProcedureName(procedureName); + StringBuilder bodyBuilder = new StringBuilder("CREATE "); + while (resultSet.next()) { + bodyBuilder.append(resultSet.getString("TEXT")).append("\n"); } + procedure.setProcedureBody(bodyBuilder.toString()); return procedure; }); } - private static String VIEW_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('VIEW', view_name) as ddl FROM all_views WHERE owner = '%s' AND view_name = '%s'"; + private static String VIEW_DDL_SQL = "SELECT VIEW_NAME, TEXT FROM ALL_VIEWS WHERE OWNER = '%s' AND VIEW_NAME = '%s'"; @Override public Table view(Connection connection, String databaseName, String schemaName, String viewName) { @@ -314,7 +323,7 @@ public Table view(Connection connection, String databaseName, String schemaName, table.setSchemaName(schemaName); table.setName(viewName); if (resultSet.next()) { - table.setDdl(resultSet.getString("ddl")); + table.setDdl("CREATE VIEW " + viewName + " AS " + resultSet.getString("TEXT")); } return table; }); From d70c4754d16f3a4ba86205346c8ed1b36ea838fd Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 21:02:52 +0800 Subject: [PATCH 58/73] fix(chat2db-oracle): fix oracle function struct --- .../main/java/ai/chat2db/plugin/oracle/OracleMetaData.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 48cba1495..0ffa4f929 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -172,9 +172,11 @@ public Function function(Connection connection, @NotEmpty String databaseName, S function.setDatabaseName(databaseName); function.setSchemaName(schemaName); function.setFunctionName(functionName); - if (resultSet.next()) { - function.setFunctionBody("CREATE " + resultSet.getString("TEXT")); + StringBuilder bodyBuilder = new StringBuilder("CREATE "); + while (resultSet.next()) { + bodyBuilder.append(resultSet.getString("TEXT")).append("\n"); } + function.setFunctionBody(bodyBuilder.toString()); return function; }); From cfcaa5b112ea811951de01d89fda0342e5931e46 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 9 Jul 2024 22:11:28 +0800 Subject: [PATCH 59/73] fix(chat2db-oracle): Temporary processing of binary data --- .../oracle/value/sub/OracleBlobProcessor.java | 19 +++++++++++++- .../value/sub/OracleLongRawProcessor.java | 25 ++++++++++++++++++- .../value/sub/OracleRawValueProcessor.java | 19 +++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java index 63b3b54e9..6e498af8d 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleBlobProcessor.java @@ -15,7 +15,24 @@ public class OracleBlobProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + // 0xabcd + return EasyStringUtils.quoteString(value.substring(2)); + } else { + //example: hello,world + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + boolean isDigit = (c >= '0' && c <= '9'); + boolean isUpperCaseHex = (c >= 'A' && c <= 'F'); + boolean isLowerCaseHex = (c >= 'a' && c <= 'f'); + if (!isDigit && !isUpperCaseHex && !isLowerCaseHex) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } + } + // example: abcd1234 + return EasyStringUtils.quoteString(value); + } } @Override diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java index 6f3634c10..dba376689 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java @@ -15,7 +15,25 @@ public class OracleLongRawProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getValue()); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + // 0xabcd + return EasyStringUtils.quoteString(value.substring(2)); + } else { + //example: hello,world + // TODO: Need to optimize recognition of hexadecimal strings + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + boolean isDigit = (c >= '0' && c <= '9'); + boolean isUpperCaseHex = (c >= 'A' && c <= 'F'); + boolean isLowerCaseHex = (c >= 'a' && c <= 'f'); + if (!isDigit && !isUpperCaseHex && !isLowerCaseHex) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } + } + // example: abcd1234 + return EasyStringUtils.quoteString(value); + } } @@ -34,4 +52,9 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { return EasyStringUtils.quoteString(blobHexString); } + public static void main(String[] args) { + String value = "0x123456"; + value = value.substring(2); + System.out.println("value = " + value); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java index db6517808..47d7ed365 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleRawValueProcessor.java @@ -14,7 +14,24 @@ public class OracleRawValueProcessor extends DefaultValueProcessor { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return EasyStringUtils.quoteString(dataValue.getValue()); + String value = dataValue.getValue(); + if (value.startsWith("0x")) { + // 0xabcd + return EasyStringUtils.quoteString(value.substring(2)); + } else { + //example: hello,world + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + boolean isDigit = (c >= '0' && c <= '9'); + boolean isUpperCaseHex = (c >= 'A' && c <= 'F'); + boolean isLowerCaseHex = (c >= 'a' && c <= 'f'); + if (!isDigit && !isUpperCaseHex && !isLowerCaseHex) { + return EasyStringUtils.quoteString(dataValue.getBlobHexString()); + } + } + // example: abcd1234 + return EasyStringUtils.quoteString(value); + } } From 025032cd05b5b3fa43310f52c37709c6eae45c25 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 10 Jul 2024 14:00:54 +0800 Subject: [PATCH 60/73] oracle column Type error fix --- .../chat2db/plugin/oracle/OracleDBManage.java | 11 +++- .../chat2db/plugin/oracle/OracleMetaData.java | 62 ++++++++++++------- .../tools/common/util/EasyStringUtils.java | 11 ++++ 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index 62ab54339..eae3c4bcf 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -24,6 +24,7 @@ public class OracleDBManage extends DefaultDBManage implements DBManage { "FROM user_col_comments " + "WHERE table_name = '%s' " + "AND comments IS NOT NULL"; + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, asyncContext, schemaName); @@ -159,11 +160,17 @@ public void connectDatabase(Connection connection, String database) { public void copyTable(Connection connection, String databaseName, String schemaName, String tableName, String newTableName, boolean copyData) throws SQLException { String sql = ""; if (copyData) { - sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName; + sql = "CREATE TABLE " + SqlUtils.quoteObjectName(newTableName) + " AS SELECT * FROM " + SqlUtils.quoteObjectName(tableName); } else { - sql = "CREATE TABLE " + newTableName + " AS SELECT * FROM " + tableName + " WHERE 1=0"; + sql = "CREATE TABLE " + SqlUtils.quoteObjectName(newTableName) + " AS SELECT * FROM " + SqlUtils.quoteObjectName(tableName) + " WHERE 1=0"; } SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); } + @Override + public void dropTable(Connection connection, String databaseName, String schemaName, String tableName) { + String sql = "DROP TABLE " + SqlUtils.quoteObjectName(tableName); + SQLExecutor.getInstance().execute(connection, sql, (resultSet) -> null); + } + } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 0ffa4f929..ef09ce709 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -15,6 +15,7 @@ import com.google.common.collect.Lists; import jakarta.validation.constraints.NotEmpty; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.io.Reader; @@ -32,10 +33,10 @@ public class OracleMetaData extends DefaultMetaService implements MetaData { private List systemSchemas = Arrays.asList("ANONYMOUS", "APEX_030200", "APEX_PUBLIC_USER", "APPQOSSYS", "BI", "CTXSYS", "DBSNMP", "DIP", "EXFSYS", "FLOWS_FILES", "HR", "IX", "MDDATA", "MDSYS", "MGMT_VIEW", "OE", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "OWBSYS_AUDIT", "PM", "SCOTT", "SH", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "WMSYS", "XDB", "XS$NULL"); private String PROCEDURE_LIST_DDL = """ - SELECT OBJECT_NAME, OBJECT_TYPE - FROM ALL_OBJECTS - WHERE OBJECT_TYPE IN ('PROCEDURE') - AND OWNER = '%s'"""; + SELECT OBJECT_NAME, OBJECT_TYPE + FROM ALL_OBJECTS + WHERE OBJECT_TYPE IN ('PROCEDURE') + AND OWNER = '%s'"""; @Override public List procedures(Connection connection, String databaseName, String schemaName) { @@ -101,9 +102,25 @@ public List
tables(Connection connection, String databaseName, String sch @Override public List columns(Connection connection, String databaseName, String schemaName, String tableName) { + List tableColumns = super.columns(connection, databaseName, schemaName, tableName); + if (CollectionUtils.isNotEmpty(tableColumns)) { + Map tableColumnMap = getTableColumns(connection, databaseName, schemaName, tableName); + for (TableColumn tableColumn : tableColumns) { + TableColumn column = tableColumnMap.get(tableColumn.getName()); + tableColumn.setUnit(column.getUnit()); + tableColumn.setComment(column.getComment()); + tableColumn.setDefaultValue(column.getDefaultValue()); + tableColumn.setOrdinalPosition(column.getOrdinalPosition()); + tableColumn.setNullable(column.getNullable()); + } + } + return tableColumns; + } + + private Map getTableColumns(Connection connection, String databaseName, String schemaName, String tableName) { + Map tableColumns = new HashMap<>(); String sql = String.format(SELECT_TAB_COLS, schemaName, tableName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { - List tableColumns = new ArrayList<>(); while (resultSet.next()) { TableColumn tableColumn = new TableColumn(); tableColumn.setTableName(tableName); @@ -151,10 +168,11 @@ public List columns(Connection connection, String databaseName, Str } else if ("C".equalsIgnoreCase(charUsed)) { tableColumn.setUnit("CHAR"); } - tableColumns.add(tableColumn); + tableColumns.put(tableColumn.getName(), tableColumn); } return tableColumns; }); + } private static String ROUTINES_SQL @@ -197,11 +215,11 @@ public List indexes(Connection connection, String databaseName, Stri String pkSql = String.format(SELECT_PK_SQL, schemaName, tableName); Set pkSet = new HashSet<>(); SQLExecutor.getInstance().execute(connection, pkSql, resultSet -> { - while (resultSet.next()) { - pkSet.add(resultSet.getString("CONSTRAINT_NAME")); - } - return null; - } + while (resultSet.next()) { + pkSet.add(resultSet.getString("CONSTRAINT_NAME")); + } + return null; + } ); String sql = String.format(SELECT_TABLE_INDEX, schemaName, tableName); @@ -264,17 +282,17 @@ private TableIndexColumn getTableIndexColumn(ResultSet resultSet) throws SQLExce public List triggers(Connection connection, String databaseName, String schemaName) { List triggers = new ArrayList<>(); return SQLExecutor.getInstance().execute(connection, String.format(TRIGGER_SQL_LIST, schemaName), - resultSet -> { - while (resultSet.next()) { - String triggerName = resultSet.getString("TRIGGER_NAME"); - Trigger trigger = new Trigger(); - trigger.setTriggerName(triggerName == null ? "" : triggerName.trim()); - trigger.setSchemaName(schemaName); - trigger.setDatabaseName(databaseName); - triggers.add(trigger); - } - return triggers; - }); + resultSet -> { + while (resultSet.next()) { + String triggerName = resultSet.getString("TRIGGER_NAME"); + Trigger trigger = new Trigger(); + trigger.setTriggerName(triggerName == null ? "" : triggerName.trim()); + trigger.setSchemaName(schemaName); + trigger.setDatabaseName(databaseName); + triggers.add(trigger); + } + return triggers; + }); } private static final String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', '%s', '%s') as ddl FROM DUAL"; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index ff2bd4d4c..016ba9328 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -241,4 +241,15 @@ public static String escapeLineString(String str) { // .replace("\r", "\\r"); } + + public static String sqlEscape(String str) { + if (StringUtils.isBlank(str)) { + return str; + } + str = str.trim(); + if (str.endsWith(";")) { + str = str.substring(0, str.length() - 1); + } + return str; + } } From 38fa3f100d7063eb9da09a2e38935248d986b721 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 10 Jul 2024 14:54:40 +0800 Subject: [PATCH 61/73] fix oracle type columnSize QA --- .../chat2db/plugin/oracle/OracleMetaData.java | 17 +++- .../oracle/type/OracleColumnTypeEnum.java | 95 +++++++++++-------- .../tools/common/util/EasyStringUtils.java | 2 + .../java/ai/chat2db/spi/util/SqlUtils.java | 15 ++- 4 files changed, 85 insertions(+), 44 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index ef09ce709..41012b126 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -106,12 +106,19 @@ public List columns(Connection connection, String databaseName, Str if (CollectionUtils.isNotEmpty(tableColumns)) { Map tableColumnMap = getTableColumns(connection, databaseName, schemaName, tableName); for (TableColumn tableColumn : tableColumns) { +// String columnType = SqlUtils.removeDigits(tableColumn.getColumnType()); +// if (columnType.startsWith("TIMESTAMP")) { +// tableColumn.setColumnSize(0); +// } + TableColumn column = tableColumnMap.get(tableColumn.getName()); - tableColumn.setUnit(column.getUnit()); - tableColumn.setComment(column.getComment()); - tableColumn.setDefaultValue(column.getDefaultValue()); - tableColumn.setOrdinalPosition(column.getOrdinalPosition()); - tableColumn.setNullable(column.getNullable()); + if (column != null) { + tableColumn.setUnit(column.getUnit()); + tableColumn.setComment(column.getComment()); + tableColumn.setDefaultValue(column.getDefaultValue()); + tableColumn.setOrdinalPosition(column.getOrdinalPosition()); + tableColumn.setNullable(column.getNullable()); + } } } return tableColumns; diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java index 6dcbd7601..b9b19f0a6 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -61,7 +62,6 @@ public enum OracleColumnTypeEnum implements ColumnBuilder { NATIONAL_CHAR_VARYING("NATIONAL CHAR VARYING", true, false, true, false, false, false, true, true, false, true), - NATIONAL_CHARACTER("NATIONAL CHARACTER", true, false, true, false, false, false, true, true, false, true), @@ -87,12 +87,17 @@ public enum OracleColumnTypeEnum implements ColumnBuilder { SMALLINT("SMALLINT", false, false, true, false, false, false, true, true, false, false), - TIMESTAMP("TIMESTAMP", true, false, true, false, false, false, true, true, false, false), + TIMESTAMP("TIMESTAMP", false, true, true, false, false, false, true, true, false, false), + + TIMESTAMP_WITH_LOCAL_TIME_ZONE("TIMESTAMP WITH LOCAL TIME ZONE", false, true, true, false, false, false, true, true, false, false), + + + TIMESTAMP_WITH_TIME_ZONE("TIMESTAMP WITH TIME ZONE", false, true, true, false, false, false, true, true, false, false), - TIMESTAMP_WITH_LOCAL_TIME_ZONE("TIMESTAMP WITH LOCAL TIME ZONE", true, false, true, false, false, false, true, true, false, false), + INTERVAL_YEAR_TO_MONTH("INTERVAL YEAR TO MONTH", true, false, true, false, false, false, true, true, false, false), - TIMESTAMP_WITH_TIME_ZONE("TIMESTAMP WITH TIME ZONE", true, false, true, false, false, false, true, true, false, false), + INTERVAL_DAY_TO_SECOND("INTERVAL DAY TO SECOND", true, true, true, false, false, false, true, true, false, false), UROWID("UROWID", true, false, true, false, false, false, true, true, false, false), @@ -104,7 +109,7 @@ public enum OracleColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static OracleColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } private static Map COLUMN_TYPE_MAP = Maps.newHashMap(); @@ -136,7 +141,7 @@ public String buildCreateColumnSql(TableColumn column) { script.append(buildDataType(column, type)).append(" "); - script.append(buildDefaultValue(column,type)).append(" "); + script.append(buildDefaultValue(column, type)).append(" "); script.append(buildNullable(column, type)).append(" "); @@ -144,11 +149,11 @@ public String buildCreateColumnSql(TableColumn column) { } - private String buildNullable(TableColumn column,OracleColumnTypeEnum type) { - if(!type.getColumnType().isSupportNullable()){ + private String buildNullable(TableColumn column, OracleColumnTypeEnum type) { + if (!type.getColumnType().isSupportNullable()) { return ""; } - if (column.getNullable()!=null && 1==column.getNullable()) { + if (column.getNullable() != null && 1 == column.getNullable()) { return "NULL"; } else { return "NOT NULL"; @@ -156,27 +161,27 @@ private String buildNullable(TableColumn column,OracleColumnTypeEnum type) { } private String buildDefaultValue(TableColumn column, OracleColumnTypeEnum type) { - if(!type.getColumnType().isSupportDefaultValue() || StringUtils.isEmpty(column.getDefaultValue())){ + if (!type.getColumnType().isSupportDefaultValue() || StringUtils.isEmpty(column.getDefaultValue())) { return ""; } - if("EMPTY_STRING".equalsIgnoreCase(column.getDefaultValue().trim())){ + if ("EMPTY_STRING".equalsIgnoreCase(column.getDefaultValue().trim())) { return StringUtils.join("DEFAULT ''"); } - if("NULL".equalsIgnoreCase(column.getDefaultValue().trim())){ + if ("NULL".equalsIgnoreCase(column.getDefaultValue().trim())) { return StringUtils.join("DEFAULT NULL"); } - return StringUtils.join("DEFAULT ",column.getDefaultValue()); + return StringUtils.join("DEFAULT ", column.getDefaultValue()); } private String buildDataType(TableColumn column, OracleColumnTypeEnum type) { String columnType = type.columnType.getTypeName(); if (Arrays.asList(CHAR, CHAR_VARYING, CHARACTER, CHARACTER_VARYING, - NVARCHAR2, VARCHAR, VARCHAR2,NATIONAL_CHAR, - NATIONAL_CHAR_VARYING,NATIONAL_CHARACTER, - NATIONAL_CHARACTER_VARYING,NCHAR,NCHAR_VARYING).contains(type)) { + NVARCHAR2, VARCHAR, VARCHAR2, NATIONAL_CHAR, + NATIONAL_CHAR_VARYING, NATIONAL_CHARACTER, + NATIONAL_CHARACTER_VARYING, NCHAR, NCHAR_VARYING).contains(type)) { StringBuilder script = new StringBuilder(); script.append(columnType); if (column.getColumnSize() != null && StringUtils.isEmpty(column.getUnit())) { @@ -187,26 +192,41 @@ private String buildDataType(TableColumn column, OracleColumnTypeEnum type) { return script.toString(); } - if (Arrays.asList(DECIMAL, FLOAT, NUMBER, UROWID,RAW,TIMESTAMP).contains(type)) { + if (Arrays.asList(DECIMAL, FLOAT, NUMBER, UROWID, RAW).contains(type)) { StringBuilder script = new StringBuilder(); script.append(columnType); if (column.getColumnSize() != null && column.getDecimalDigits() == null) { script.append("(").append(column.getColumnSize()).append(")"); - } else if (column.getColumnSize() != null && column.getDecimalDigits() != null) { + } else if (column.getColumnSize() != null && column.getDecimalDigits() != null) { script.append("(").append(column.getColumnSize()).append(",").append(column.getDecimalDigits()).append(")"); } return script.toString(); } - - if (Arrays.asList(TIMESTAMP_WITH_TIME_ZONE,TIMESTAMP_WITH_LOCAL_TIME_ZONE).contains(type)) { - StringBuilder script = new StringBuilder(); - if(column.getColumnSize() == null){ - script.append(columnType); - }else { - String [] split = columnType.split("TIMESTAMP"); - script.append("TIMESTAMP").append("(").append(column.getColumnSize()).append(")").append(split[1]); - } - return script.toString(); + if (Arrays.asList(TIMESTAMP).contains(type)) { + int decimalDigits = column.getDecimalDigits() != null ? column.getDecimalDigits() : 6; + String valueTemplate = "TIMESTAMP(%s)"; + return String.format(valueTemplate, decimalDigits); + } + if (Arrays.asList(TIMESTAMP_WITH_TIME_ZONE).contains(type)) { + int decimalDigits = column.getDecimalDigits() != null ? column.getDecimalDigits() : 6; + String valueTemplate = "TIMESTAMP(%s) WITH TIME ZONE"; + return String.format(valueTemplate, decimalDigits); + } + if (Arrays.asList(TIMESTAMP_WITH_LOCAL_TIME_ZONE).contains(type)) { + int decimalDigits = column.getDecimalDigits() != null ? column.getDecimalDigits() : 6; + String valueTemplate = "TIMESTAMP(%s) WITH LOCAL TIME ZONE"; + return String.format(valueTemplate, decimalDigits); + } + if (Arrays.asList(INTERVAL_DAY_TO_SECOND).contains(type)) { + int columnSize = column.getColumnSize() != null ? column.getColumnSize() : 2; + int decimalDigits = column.getDecimalDigits() != null ? column.getDecimalDigits() : 6; + String valueTemplate = "INTERVAL DAY(%s) TO SECOND(%s)"; + return String.format(valueTemplate, columnSize, decimalDigits); + } + if (Arrays.asList(INTERVAL_YEAR_TO_MONTH).contains(type)) { + int columnSize = column.getColumnSize() != null ? column.getColumnSize() : 2; + String valueTemplate = "INTERVAL YEAR(%s) TO MONTH"; + return String.format(valueTemplate, columnSize); } @@ -214,30 +234,29 @@ private String buildDataType(TableColumn column, OracleColumnTypeEnum type) { } - @Override public String buildModifyColumn(TableColumn tableColumn) { if (EditStatus.DELETE.name().equals(tableColumn.getEditStatus())) { StringBuilder script = new StringBuilder(); - script.append("ALTER TABLE "). append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); + script.append("ALTER TABLE ").append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); script.append(" ").append("DROP COLUMN ").append("\"").append(tableColumn.getName()).append("\""); return script.toString(); } if (EditStatus.ADD.name().equals(tableColumn.getEditStatus())) { StringBuilder script = new StringBuilder(); - script.append("ALTER TABLE "). append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); + script.append("ALTER TABLE ").append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); script.append(" ").append("ADD (").append(buildCreateColumnSql(tableColumn)).append(")"); return script.toString(); } if (EditStatus.MODIFY.name().equals(tableColumn.getEditStatus())) { StringBuilder script = new StringBuilder(); - script.append("ALTER TABLE "). append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); - script.append(" ").append("MODIFY (").append(buildModifyColumnSql(tableColumn,tableColumn.getOldColumn())).append(") \n" ); + script.append("ALTER TABLE ").append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); + script.append(" ").append("MODIFY (").append(buildModifyColumnSql(tableColumn, tableColumn.getOldColumn())).append(") \n"); if (!StringUtils.equalsIgnoreCase(tableColumn.getOldName(), tableColumn.getName())) { script.append(";"); - script.append("ALTER TABLE "). append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); + script.append("ALTER TABLE ").append("\"").append(tableColumn.getSchemaName()).append("\".\"").append(tableColumn.getTableName()).append("\""); script.append(" ").append("RENAME COLUMN ").append("\"").append(tableColumn.getOldName()).append("\"").append(" TO ").append("\"").append(tableColumn.getName()).append("\""); } @@ -247,7 +266,7 @@ public String buildModifyColumn(TableColumn tableColumn) { return ""; } - public String buildModifyColumnSql(TableColumn column,TableColumn oldColumn) { + public String buildModifyColumnSql(TableColumn column, TableColumn oldColumn) { OracleColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType().toUpperCase()); if (type == null) { return ""; @@ -258,16 +277,16 @@ public String buildModifyColumnSql(TableColumn column,TableColumn oldColumn) { script.append(buildDataType(column, type)).append(" "); - script.append(buildDefaultValue(column,type)).append(" "); + script.append(buildDefaultValue(column, type)).append(" "); - if(oldColumn.getNullable() != column.getNullable()) { + if (oldColumn.getNullable() != column.getNullable()) { script.append(buildNullable(column, type)).append(" "); } return script.toString(); } - public static List getTypes(){ + public static List getTypes() { return Arrays.stream(OracleColumnTypeEnum.values()).map(columnTypeEnum -> columnTypeEnum.getColumnType() ).toList(); diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java index 016ba9328..4349d1638 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/util/EasyStringUtils.java @@ -252,4 +252,6 @@ public static String sqlEscape(String str) { } return str; } + + } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index fd2449144..668d81764 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -175,7 +175,7 @@ public static List parse(String sql, DbType dbType, boolean removeCommen try { return splitWithCreateEvent(sql, dbType); } catch (Exception e1) { - if(removeComment) { + if (removeComment) { return SQLParserUtils.splitAndRemoveComment(sql, dbType); }{ return SQLParserUtils.split(sql, dbType); @@ -341,5 +341,18 @@ public static String quoteObjectName(String name, String quoteSymbol) { return name; } + /** + * String input = "INTERVAL DAY(2) TO SECOND(6)"; + * remove (2) and (6) + * + * @param input + * @return + */ + public static String removeDigits(String input) { + if (StringUtils.isBlank(input)) { + return input; + } + return input.replaceAll("\\(\\d+\\)", ""); + } } \ No newline at end of file From 212a527fc02f716fb2663668c5d0ebaebae3529e Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 10 Jul 2024 15:00:58 +0800 Subject: [PATCH 62/73] fix oracle type columnSize QA --- .../main/java/ai/chat2db/plugin/oracle/OracleMetaData.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 41012b126..8e4cb6b9f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -12,6 +12,7 @@ import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.util.SortUtils; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Lists; import jakarta.validation.constraints.NotEmpty; import lombok.extern.slf4j.Slf4j; @@ -106,11 +107,7 @@ public List columns(Connection connection, String databaseName, Str if (CollectionUtils.isNotEmpty(tableColumns)) { Map tableColumnMap = getTableColumns(connection, databaseName, schemaName, tableName); for (TableColumn tableColumn : tableColumns) { -// String columnType = SqlUtils.removeDigits(tableColumn.getColumnType()); -// if (columnType.startsWith("TIMESTAMP")) { -// tableColumn.setColumnSize(0); -// } - + tableColumn.setColumnType(SqlUtils.removeDigits(tableColumn.getColumnType())); TableColumn column = tableColumnMap.get(tableColumn.getName()); if (column != null) { tableColumn.setUnit(column.getUnit()); From c10a7fea86efb6322b03e831819934e5ba507230 Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Wed, 10 Jul 2024 15:04:04 +0800 Subject: [PATCH 63/73] fix oracle columnType --- .../ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java index b9b19f0a6..16ff67de4 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/type/OracleColumnTypeEnum.java @@ -105,6 +105,8 @@ public enum OracleColumnTypeEnum implements ColumnBuilder { VARCHAR2("VARCHAR2", true, false, true, false, false, false, true, true, false, true), + XMLTYPE("XMLTYPE", false, false, true, false, false, false, true, true, false, false), + ; private ColumnType columnType; From 2560061e6867729df5e7f390bb53e5a2f0448aa5 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 10 Jul 2024 20:05:14 +0800 Subject: [PATCH 64/73] feat(chat2db-oracle): fix oracle DDL --- .../chat2db/plugin/oracle/OracleDBManage.java | 35 +------ .../chat2db/plugin/oracle/OracleMetaData.java | 93 ++++++++++++++----- 2 files changed, 71 insertions(+), 57 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index eae3c4bcf..209e86175 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -19,11 +19,6 @@ @Slf4j public class OracleDBManage extends DefaultDBManage implements DBManage { - private static String TABLE_COMMENT_SQL = "SELECT 'COMMENT ON TABLE ' || table_name || ' IS ''' || comments || ''';' AS table_comment_ddl FROM user_tab_comments WHERE table_name = '%s'"; - private static String TABLE_COLUMN_COMMENT_SQL = "SELECT 'COMMENT ON COLUMN ' || table_name || '.' || column_name || ' IS ''' || comments || ''';' AS column_comment_ddl " + - "FROM user_col_comments " + - "WHERE table_name = '%s' " + - "AND comments IS NOT NULL"; public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { exportTables(connection, databaseName, schemaName, asyncContext); @@ -45,40 +40,14 @@ private void exportTables(Connection connection, String databaseName, String sch public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { String tableDDL = Chat2DBContext.getMetaData().tableDDL(connection, databaseName, schemaName, tableName); - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("DROP TABLE ").append(SqlUtils.quoteObjectName(tableName)).append(";") - .append(tableDDL).append(";").append("\n"); - asyncContext.write(sqlBuilder.toString()); - - exportTableComments(connection, tableName, asyncContext); - exportTableColumnsComments(connection, tableName, asyncContext); + String sqlBuilder = "DROP TABLE " + SqlUtils.quoteObjectName(tableName) + ";" + tableDDL + "\n"; + asyncContext.write(sqlBuilder); if (asyncContext.isContainsData()) { exportTableData(connection, databaseName, schemaName, tableName, asyncContext); } } - private void exportTableComments(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(TABLE_COMMENT_SQL, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - if (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("table_comment_ddl")).append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } - } - - private void exportTableColumnsComments(Connection connection, String tableName, AsyncContext asyncContext) throws SQLException { - String sql = String.format(TABLE_COLUMN_COMMENT_SQL, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - while (resultSet.next()) { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append(resultSet.getString("column_comment_ddl")).append("\n"); - asyncContext.write(sqlBuilder.toString()); - } - } - } private void exportViews(Connection connection, AsyncContext asyncContext, String schemaName) throws SQLException { try (ResultSet resultSet = connection.getMetaData().getTables(null, schemaName, null, new String[]{"VIEW"})) { diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 8e4cb6b9f..819c4188f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -5,6 +5,7 @@ import ai.chat2db.plugin.oracle.type.OracleDefaultValueEnum; import ai.chat2db.plugin.oracle.type.OracleIndexTypeEnum; import ai.chat2db.plugin.oracle.value.OracleValueProcessor; +import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.ValueProcessor; @@ -30,14 +31,23 @@ public class OracleMetaData extends DefaultMetaService implements MetaData { private static final String TABLE_DDL_SQL = "select dbms_metadata.get_ddl('TABLE','%s','%s') as sql from dual"; + private static final String TABLE_COMMENT_SQL = "select owner, table_name, comments from ALL_TAB_COMMENTS where OWNER = '%s' and TABLE_NAME = '%s'"; + private static final String TABLE_COLUMN_COMMENT_SQL = """ + SELECT owner, table_name, column_name, comments + FROM all_col_comments + WHERE owner = '%s' and table_name = '%s' and comments is not null"""; private List systemSchemas = Arrays.asList("ANONYMOUS", "APEX_030200", "APEX_PUBLIC_USER", "APPQOSSYS", "BI", "CTXSYS", "DBSNMP", "DIP", "EXFSYS", "FLOWS_FILES", "HR", "IX", "MDDATA", "MDSYS", "MGMT_VIEW", "OE", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "OWBSYS_AUDIT", "PM", "SCOTT", "SH", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "WMSYS", "XDB", "XS$NULL"); - private String PROCEDURE_LIST_DDL = """ - SELECT OBJECT_NAME, OBJECT_TYPE - FROM ALL_OBJECTS - WHERE OBJECT_TYPE IN ('PROCEDURE') - AND OWNER = '%s'"""; + private static final String PROCEDURE_LIST_DDL = """ + SELECT OBJECT_NAME, OBJECT_TYPE + FROM ALL_OBJECTS + WHERE OBJECT_TYPE IN ('PROCEDURE') + AND OWNER = '%s'"""; + public static final String TABLE_INDEX_DDL_SQL = """ + SELECT DBMS_METADATA.GET_DDL('INDEX', index_name, table_owner) AS ddl + FROM all_indexes + WHERE table_owner = '%s' AND table_name = '%s'"""; @Override public List procedures(Connection connection, String databaseName, String schemaName) { @@ -61,18 +71,53 @@ public List schemas(Connection connection, String databaseName) { @Override public String tableDDL(Connection connection, String databaseName, String schemaName, String tableName) { + // TODO: only_read user can not get ddl String sql = String.format(TABLE_DDL_SQL, tableName, schemaName); - return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + String tableCommentSql = String.format(TABLE_COMMENT_SQL, schemaName, tableName); + String tableColumnCommentSql = String.format(TABLE_COLUMN_COMMENT_SQL, schemaName, tableName); + String tableIndexSql = String.format(TABLE_INDEX_DDL_SQL, schemaName, tableName); + StringBuilder ddlBuilder = new StringBuilder(); + SQLExecutor.getInstance().execute(connection, sql, resultSet -> { try { if (resultSet.next()) { - return resultSet.getString("sql"); + ddlBuilder.append(resultSet.getString("sql")); } } catch (SQLException e) { throw new RuntimeException(e); } + }); + SQLExecutor.getInstance().execute(connection, tableCommentSql, resultSet -> { + if (resultSet.next()) { + String tableComment = resultSet.getString("comments"); + if (StringUtils.isNotBlank(tableComment)) { + ddlBuilder.append("\nCOMMENT ON TABLE ").append(SqlUtils.quoteObjectName(tableName)).append(" IS ") + .append(EasyStringUtils.escapeAndQuoteString(tableComment)).append(";"); + } + } + }); + SQLExecutor.getInstance().execute(connection, tableColumnCommentSql, resultSet -> { + while (resultSet.next()) { + String columnName = resultSet.getString("column_name"); + String columnComment = resultSet.getString("comments"); + if (StringUtils.isNotBlank(columnComment)) { + ddlBuilder.append("\nCOMMENT ON COLUMN ") + .append(SqlUtils.quoteObjectName(tableName)).append(".") + .append(SqlUtils.quoteObjectName(columnName)).append(" IS ") + .append(EasyStringUtils.escapeAndQuoteString(columnComment)).append(";"); + } + } + }); - return null; + SQLExecutor.getInstance().execute(connection, tableIndexSql, resultSet -> { + while (resultSet.next()) { + String ddl = resultSet.getString("ddl"); + if (StringUtils.isNotBlank(ddl)) { + ddlBuilder.append("\n").append(ddl).append(";"); + } + } }); + return ddlBuilder.toString(); + } private static String SELECT_TABLE_SQL = "SELECT A.OWNER, A.TABLE_NAME, B.COMMENTS " + @@ -219,11 +264,11 @@ public List indexes(Connection connection, String databaseName, Stri String pkSql = String.format(SELECT_PK_SQL, schemaName, tableName); Set pkSet = new HashSet<>(); SQLExecutor.getInstance().execute(connection, pkSql, resultSet -> { - while (resultSet.next()) { - pkSet.add(resultSet.getString("CONSTRAINT_NAME")); - } - return null; - } + while (resultSet.next()) { + pkSet.add(resultSet.getString("CONSTRAINT_NAME")); + } + return null; + } ); String sql = String.format(SELECT_TABLE_INDEX, schemaName, tableName); @@ -286,17 +331,17 @@ private TableIndexColumn getTableIndexColumn(ResultSet resultSet) throws SQLExce public List triggers(Connection connection, String databaseName, String schemaName) { List triggers = new ArrayList<>(); return SQLExecutor.getInstance().execute(connection, String.format(TRIGGER_SQL_LIST, schemaName), - resultSet -> { - while (resultSet.next()) { - String triggerName = resultSet.getString("TRIGGER_NAME"); - Trigger trigger = new Trigger(); - trigger.setTriggerName(triggerName == null ? "" : triggerName.trim()); - trigger.setSchemaName(schemaName); - trigger.setDatabaseName(databaseName); - triggers.add(trigger); - } - return triggers; - }); + resultSet -> { + while (resultSet.next()) { + String triggerName = resultSet.getString("TRIGGER_NAME"); + Trigger trigger = new Trigger(); + trigger.setTriggerName(triggerName == null ? "" : triggerName.trim()); + trigger.setSchemaName(schemaName); + trigger.setDatabaseName(databaseName); + triggers.add(trigger); + } + return triggers; + }); } private static final String TRIGGER_DDL_SQL = "SELECT DBMS_METADATA.GET_DDL('TRIGGER', '%s', '%s') as ddl FROM DUAL"; From 3961610e5097d987d3be1deef76791723aec6d0c Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 10 Jul 2024 20:42:16 +0800 Subject: [PATCH 65/73] feat(chat2db-oracle): exclude PUINDEX --- .../chat2db/plugin/oracle/OracleMetaData.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 819c4188f..07116b41b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -44,10 +44,16 @@ public class OracleMetaData extends DefaultMetaService implements MetaData { FROM ALL_OBJECTS WHERE OBJECT_TYPE IN ('PROCEDURE') AND OWNER = '%s'"""; - public static final String TABLE_INDEX_DDL_SQL = """ - SELECT DBMS_METADATA.GET_DDL('INDEX', index_name, table_owner) AS ddl - FROM all_indexes - WHERE table_owner = '%s' AND table_name = '%s'"""; + private static final String TABLE_INDEX_DDL_SQL = """ + SELECT DBMS_METADATA.GET_DDL('INDEX', index_name, table_owner) AS ddl, + index_name AS INDEX_NAME + FROM all_indexes + WHERE table_owner = '%s' AND table_name = '%s'"""; + private static final String PU_INDEX_NAME_SQL = """ + SELECT DISTINCT AC.INDEX_NAME + FROM ALL_CONSTRAINTS AC + WHERE AC.OWNER = '%s' AND AC.TABLE_NAME = '%s' + AND AC.CONSTRAINT_TYPE IN ('P', 'U')"""; @Override public List procedures(Connection connection, String databaseName, String schemaName) { @@ -76,6 +82,7 @@ public String tableDDL(Connection connection, String databaseName, String schema String tableCommentSql = String.format(TABLE_COMMENT_SQL, schemaName, tableName); String tableColumnCommentSql = String.format(TABLE_COLUMN_COMMENT_SQL, schemaName, tableName); String tableIndexSql = String.format(TABLE_INDEX_DDL_SQL, schemaName, tableName); + String PUIndexSql = String.format(PU_INDEX_NAME_SQL, schemaName, tableName); StringBuilder ddlBuilder = new StringBuilder(); SQLExecutor.getInstance().execute(connection, sql, resultSet -> { try { @@ -107,9 +114,22 @@ public String tableDDL(Connection connection, String databaseName, String schema } } }); - + List indexNames = SQLExecutor.getInstance().execute(connection, PUIndexSql, resultSet -> { + List PUIndexNames = new ArrayList<>(); + while (resultSet.next()) { + String indexName = resultSet.getString("index_name"); + if (StringUtils.isNotBlank(indexName)) { + PUIndexNames.add(indexName); + } + } + return PUIndexNames; + }); SQLExecutor.getInstance().execute(connection, tableIndexSql, resultSet -> { while (resultSet.next()) { + String indexName = resultSet.getString("INDEX_NAME"); + if (CollectionUtils.isNotEmpty(indexNames) && indexNames.contains(indexName)) { + continue; + } String ddl = resultSet.getString("ddl"); if (StringUtils.isNotBlank(ddl)) { ddlBuilder.append("\n").append(ddl).append(";"); From 493f934a2c8e09f707b27c07a521345c3e88bebc Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Thu, 11 Jul 2024 13:56:32 +0800 Subject: [PATCH 66/73] fix columnType error --- .../type/ClickHouseColumnTypeEnum.java | 3 +- .../plugin/db2/type/DB2ColumnTypeEnum.java | 3 +- .../plugin/dm/type/DMColumnTypeEnum.java | 5 +- .../plugin/hive/type/HiveColumnTypeEnum.java | 3 +- .../kingbase/type/KingBaseColumnTypeEnum.java | 3 +- .../mysql/type/MysqlColumnTypeEnum.java | 3 +- .../mysql/value/MysqlValueProcessor.java | 2 +- .../oracle/value/OracleValueProcessor.java | 2 +- .../type/PostgreSQLColumnTypeEnum.java | 3 +- .../sqlite/type/SqliteColumnTypeEnum.java | 3 +- .../plugin/sqlserver/SqlServerDBManage.java | 3 +- .../plugin/sqlserver/SqlServerMetaData.java | 5 +- .../type/SqlServerColumnTypeEnum.java | 3 +- .../rdb/RdbDmlExportController.java | 56 ++++++++++--------- .../rdb/data/sql/SqlDataExporter.java | 4 +- .../controller/task/biz/TaskBizService.java | 32 +++++------ .../java/ai/chat2db/spi/ValueProcessor.java | 2 +- .../ai/chat2db/spi/jdbc/DefaultDBManage.java | 2 +- .../chat2db/spi/jdbc/DefaultMetaService.java | 10 +++- .../chat2db/spi/jdbc/DefaultSqlBuilder.java | 7 ++- .../spi/jdbc/DefaultValueProcessor.java | 2 +- .../java/ai/chat2db/spi/model/Command.java | 5 ++ .../java/ai/chat2db/spi/sql/SQLExecutor.java | 36 +++++++----- 23 files changed, 119 insertions(+), 78 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java index d53c90cc7..2222e97c2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/type/ClickHouseColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -67,7 +68,7 @@ public enum ClickHouseColumnTypeEnum implements ColumnBuilder { } public static ClickHouseColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public static List getTypes() { diff --git a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/type/DB2ColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/type/DB2ColumnTypeEnum.java index 81465314d..e4729d504 100644 --- a/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/type/DB2ColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-db2/src/main/java/ai/chat2db/plugin/db2/type/DB2ColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -133,7 +134,7 @@ public enum DB2ColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static DB2ColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } private static Map COLUMN_TYPE_MAP = Maps.newHashMap(); diff --git a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java index cd422fbde..22cf80846 100644 --- a/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-dm/src/main/java/ai/chat2db/plugin/dm/type/DMColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -123,11 +124,13 @@ public enum DMColumnTypeEnum implements ColumnBuilder { VARCHAR2("VARCHAR2", true, false, true, false, false, false, true, true, false, true), + DATETIME("DATETIME", false, false, true, false, false, false, true, true, false, false), ; private ColumnType columnType; public static DMColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + String type = SqlUtils.removeDigits(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(type); } private static Map COLUMN_TYPE_MAP = Maps.newHashMap(); diff --git a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/type/HiveColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/type/HiveColumnTypeEnum.java index 2c19fd0ff..95f98785e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/type/HiveColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/type/HiveColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -71,7 +72,7 @@ public enum HiveColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static HiveColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public ColumnType getColumnType() { diff --git a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/type/KingBaseColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/type/KingBaseColumnTypeEnum.java index 1eec1a778..e673e9227 100644 --- a/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/type/KingBaseColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-kingbase/src/main/java/ai/chat2db/plugin/kingbase/type/KingBaseColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -87,7 +88,7 @@ public enum KingBaseColumnTypeEnum implements ColumnBuilder { } public static KingBaseColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public static List getTypes() { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlColumnTypeEnum.java index d901b9b3f..81b7caa96 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/type/MysqlColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -116,7 +117,7 @@ public enum MysqlColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static MysqlColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public ColumnType getColumnType() { diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index 0f5aad138..08a0a148d 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -42,7 +42,7 @@ public String getJdbcValue(JDBCDataValue dataValue) { @Override - public String getJdbcValueString(JDBCDataValue dataValue) { + public String getJdbcSqlValueString(JDBCDataValue dataValue) { Object value = dataValue.getObject(); if (Objects.isNull(value)) { // mysql -> example: [date]->0000-00-00 diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 3c640e730..746ed852c 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -36,7 +36,7 @@ public String getJdbcValue(JDBCDataValue dataValue) { @Override - public String getJdbcValueString(JDBCDataValue dataValue) { + public String getJdbcSqlValueString(JDBCDataValue dataValue) { if (OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName().equalsIgnoreCase(dataValue.getType())) { return convertJDBCValueStrByType(dataValue); } diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/type/PostgreSQLColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/type/PostgreSQLColumnTypeEnum.java index c0b6a6213..c0f96f0ed 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/type/PostgreSQLColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/type/PostgreSQLColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -76,7 +77,7 @@ public enum PostgreSQLColumnTypeEnum implements ColumnBuilder { } public static PostgreSQLColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public static List getTypes() { diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/type/SqliteColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/type/SqliteColumnTypeEnum.java index 5aded889d..efda80230 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/type/SqliteColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlite/src/main/java/ai/chat2db/plugin/sqlite/type/SqliteColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -27,7 +28,7 @@ public enum SqliteColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static SqliteColumnTypeEnum getByType(String dataType) { - return COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); } public ColumnType getColumnType() { diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java index 1fb2997fb..93d1ed951 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerDBManage.java @@ -16,7 +16,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Objects; public class SqlServerDBManage extends DefaultDBManage implements DBManage { private String tableDDLFunction @@ -108,7 +107,7 @@ public void exportTableData(Connection connection, String databaseName, String s for (int i = 1; i <= metaData.getColumnCount(); i++) { ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); - String valueString = valueProcessor.getJdbcValueString(jdbcDataValue); + String valueString = valueProcessor.getJdbcSqlValueString(jdbcDataValue); valueList.add(valueString); } String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, columnList, valueList); diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerMetaData.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerMetaData.java index 2924c4c1d..18c6f0c68 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/SqlServerMetaData.java @@ -17,6 +17,7 @@ import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; import ai.chat2db.spi.util.SortUtils; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Lists; import jakarta.validation.constraints.NotEmpty; import org.apache.commons.lang3.StringUtils; @@ -131,7 +132,9 @@ public List columns(Connection connection, String databaseName, Str column.setOldName(resultSet.getString("COLUMN_NAME")); column.setName(resultSet.getString("COLUMN_NAME")); //column.setColumnType(resultSet.getString("COLUMN_TYPE")); - column.setColumnType(resultSet.getString("DATA_TYPE").toUpperCase()); + String dataType = resultSet.getString("DATA_TYPE").toUpperCase(); + column.setColumnType(SqlUtils.removeDigits(dataType)); + //column.setDataType(resultSet.getInt("DATA_TYPE")); column.setDefaultValue(resultSet.getString("COLUMN_DEFAULT")); //column.setAutoIncrement(resultSet.getString("EXTRA").contains("auto_increment")); diff --git a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/type/SqlServerColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/type/SqlServerColumnTypeEnum.java index 7be1a04ce..033254d20 100644 --- a/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/type/SqlServerColumnTypeEnum.java +++ b/chat2db-server/chat2db-plugins/chat2db-sqlserver/src/main/java/ai/chat2db/plugin/sqlserver/type/SqlServerColumnTypeEnum.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.enums.EditStatus; import ai.chat2db.spi.model.ColumnType; import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -103,7 +104,7 @@ public enum SqlServerColumnTypeEnum implements ColumnBuilder { private ColumnType columnType; public static SqlServerColumnTypeEnum getByType(String dataType) { - SqlServerColumnTypeEnum typeEnum = COLUMN_TYPE_MAP.get(dataType.toUpperCase()); + SqlServerColumnTypeEnum typeEnum = COLUMN_TYPE_MAP.get(SqlUtils.removeDigits(dataType.toUpperCase())); if (typeEnum == null) { return OTHER; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java index cbb263180..0f554a2cc 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDmlExportController.java @@ -7,6 +7,8 @@ import java.time.LocalDateTime; import java.util.List; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.SQLUtils.FormatOption; @@ -103,9 +105,9 @@ public void export(@Valid @RequestBody DataExportRequest request, HttpServletRes response.setCharacterEncoding("utf-8"); String fileName = URLEncoder.encode( - tableName + "_" + LocalDateTime.now().format(DatePattern.PURE_DATETIME_FORMATTER), - StandardCharsets.UTF_8) - .replaceAll("\\+", "%20"); + tableName + "_" + LocalDateTime.now().format(DatePattern.PURE_DATETIME_FORMATTER), + StandardCharsets.UTF_8) + .replaceAll("\\+", "%20"); if (exportType == ExportTypeEnum.CSV) { doExportCsv(sql, response, fileName); @@ -115,26 +117,27 @@ public void export(@Valid @RequestBody DataExportRequest request, HttpServletRes } private void doExportCsv(String sql, HttpServletResponse response, String fileName) - throws IOException { + throws IOException { response.setContentType("text/csv"); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".csv"); ExcelWrapper excelWrapper = new ExcelWrapper(); try { + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); ExcelWriterBuilder excelWriterBuilder = EasyExcel.write(response.getOutputStream()) - .charset(StandardCharsets.UTF_8) - .excelType(ExcelTypeEnum.CSV); + .charset(StandardCharsets.UTF_8) + .excelType(ExcelTypeEnum.CSV); excelWrapper.setExcelWriterBuilder(excelWriterBuilder); SQLExecutor.getInstance().execute(Chat2DBContext.getConnection(), sql, headerList -> { excelWriterBuilder.head( - EasyCollectionUtils.toList(headerList, header -> Lists.newArrayList(header.getName()))); + EasyCollectionUtils.toList(headerList, header -> Lists.newArrayList(header.getName()))); excelWrapper.setExcelWriter(excelWriterBuilder.build()); excelWrapper.setWriteSheet(EasyExcel.writerSheet(0).build()); }, dataList -> { List> writeDataList = Lists.newArrayList(); writeDataList.add(dataList); excelWrapper.getExcelWriter().write(writeDataList, excelWrapper.getWriteSheet()); - }, false); + }, jdbcDataValue -> valueProcessor.getJdbcValue(jdbcDataValue), false); } finally { if (excelWrapper.getExcelWriter() != null) { excelWrapper.getExcelWriter().finish(); @@ -143,29 +146,32 @@ private void doExportCsv(String sql, HttpServletResponse response, String fileNa } private void doExportInsert(String sql, HttpServletResponse response, String fileName, DbType dbType, - String tableName) - throws IOException { + String tableName) + throws IOException { response.setContentType("text/sql"); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".sql"); - + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + SqlBuilder sqlBuilder = Chat2DBContext.getMetaData().getSqlBuilder(); try (PrintWriter printWriter = response.getWriter()) { + List headerColumns = Lists.newArrayList(); InsertWrapper insertWrapper = new InsertWrapper(); SQLExecutor.getInstance().execute(Chat2DBContext.getConnection(), sql, - headerList -> insertWrapper.setHeaderList( - EasyCollectionUtils.toList(headerList, header -> new SQLIdentifierExpr(header.getName()))) - , dataList -> { - SQLInsertStatement sqlInsertStatement = new SQLInsertStatement(); - sqlInsertStatement.setDbType(dbType); - sqlInsertStatement.setTableSource(new SQLExprTableSource(tableName)); - sqlInsertStatement.getColumns().addAll(insertWrapper.getHeaderList()); - ValuesClause valuesClause = new ValuesClause(); - for (String s : dataList) { - valuesClause.addValue(s); + headerList -> { + headerList.forEach(sqlIdentifierExpr -> headerColumns.add(sqlIdentifierExpr.getName())); } - sqlInsertStatement.setValues(valuesClause); - - printWriter.println(SQLUtils.toSQLString(sqlInsertStatement, dbType, INSERT_FORMAT_OPTION) + ";"); - }, false); + , dataList -> { + SQLInsertStatement sqlInsertStatement = new SQLInsertStatement(); + sqlInsertStatement.setDbType(dbType); + sqlInsertStatement.setTableSource(new SQLExprTableSource(tableName)); + sqlInsertStatement.getColumns().addAll(insertWrapper.getHeaderList()); + ValuesClause valuesClause = new ValuesClause(); + for (String s : dataList) { + valuesClause.addValue(s); + } + sqlInsertStatement.setValues(valuesClause); + String sqls = sqlBuilder.buildSingleInsertSql(null, null, tableName, headerColumns, dataList); + printWriter.println(sqls + ";"); + }, jdbcDataValue -> valueProcessor.getJdbcSqlValueString(jdbcDataValue), false); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java index 8ae1cf719..b9f985fb8 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SqlDataExporter.java @@ -146,7 +146,7 @@ private List extractRowData(ResultSet resultSet, ValueProcessor valuePro List rowData = new ArrayList<>(metaData.getColumnCount()); for (int i = 1; i <= metaData.getColumnCount(); i++) { JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); - rowData.add(valueProcessor.getJdbcValueString(jdbcDataValue)); + rowData.add(valueProcessor.getJdbcSqlValueString(jdbcDataValue)); } return rowData; } @@ -158,7 +158,7 @@ private Map extractRowDataAsMap(ResultSet resultSet, ValueProces for (int i = 1; i <= metaData.getColumnCount(); i++) { JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); String columnName = metaData.getColumnName(i); - String jdbcValueString = valueProcessor.getJdbcValueString(jdbcDataValue); + String jdbcValueString = valueProcessor.getJdbcSqlValueString(jdbcDataValue); if (primaryKeyMap.containsKey(columnName)) { primaryKeyMap.put(columnName, jdbcValueString); } else { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java index d52e8cecf..4b136184f 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java @@ -19,6 +19,8 @@ import ai.chat2db.server.web.api.controller.rdb.factory.ExportServiceFactory; import ai.chat2db.server.web.api.controller.rdb.request.DataExportRequest; import ai.chat2db.server.web.api.controller.rdb.vo.TableVO; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.model.Table; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; @@ -32,8 +34,6 @@ import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; -import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.visitor.VisitorFeature; import com.alibaba.excel.EasyExcel; @@ -250,6 +250,7 @@ private void doExportCsv(String sql, File file) { .charset(StandardCharsets.UTF_8) .excelType(ExcelTypeEnum.CSV); excelWrapper.setExcelWriterBuilder(excelWriterBuilder); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); SQLExecutor.getInstance().execute(Chat2DBContext.getConnection(), sql, headerList -> { excelWriterBuilder.head( EasyCollectionUtils.toList(headerList, header -> Lists.newArrayList(header.getName()))); @@ -259,7 +260,7 @@ private void doExportCsv(String sql, File file) { List> writeDataList = Lists.newArrayList(); writeDataList.add(dataList); excelWrapper.getExcelWriter().write(writeDataList, excelWrapper.getWriteSheet()); - }, false); + }, jdbcDataValue -> valueProcessor.getJdbcValue(jdbcDataValue),false); } finally { if (excelWrapper.getExcelWriter() != null) { excelWrapper.getExcelWriter().finish(); @@ -272,22 +273,19 @@ private void doExportInsert(String sql, File file, DbType dbType, throws IOException { try (PrintWriter printWriter = new PrintWriter(file, StandardCharsets.UTF_8.name())) { RdbDmlExportController.InsertWrapper insertWrapper = new RdbDmlExportController.InsertWrapper(); + ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); + SqlBuilder sqlBuilder = Chat2DBContext.getSqlBuilder(); + String databaseName = Chat2DBContext.getConnectInfo().getDatabaseName(); + String schemaName = Chat2DBContext.getConnectInfo().getSchemaName(); + List headerColumns = Lists.newArrayList(); SQLExecutor.getInstance().execute(Chat2DBContext.getConnection(), sql, - headerList -> insertWrapper.setHeaderList( - EasyCollectionUtils.toList(headerList, header -> new SQLIdentifierExpr(header.getName()))) + headerList -> { + headerList.forEach(header -> headerColumns.add(header.getName())); + } , dataList -> { - SQLInsertStatement sqlInsertStatement = new SQLInsertStatement(); - sqlInsertStatement.setDbType(dbType); - sqlInsertStatement.setTableSource(new SQLExprTableSource(tableName)); - sqlInsertStatement.getColumns().addAll(insertWrapper.getHeaderList()); - SQLInsertStatement.ValuesClause valuesClause = new SQLInsertStatement.ValuesClause(); - for (String s : dataList) { - valuesClause.addValue(s); - } - sqlInsertStatement.setValues(valuesClause); - - printWriter.println(SQLUtils.toSQLString(sqlInsertStatement, dbType, INSERT_FORMAT_OPTION) + ";"); - }, false); + String insertSql = sqlBuilder.buildSingleInsertSql(databaseName, schemaName, tableName, headerColumns, dataList); + printWriter.println(insertSql + ";"); + }, jdbcDataValue -> valueProcessor.getJdbcSqlValueString(jdbcDataValue), false); } } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java index 953a2ccc1..c8776feb4 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/ValueProcessor.java @@ -36,5 +36,5 @@ public interface ValueProcessor { * * @return 一个格式化后的字符串，可以直接用于DML语句中，确保数据的正确插入或更新。 */ - String getJdbcValueString(JDBCDataValue dataValue); + String getJdbcSqlValueString(JDBCDataValue dataValue); } diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java index 4d6a79312..9efafff38 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultDBManage.java @@ -219,7 +219,7 @@ public void exportTableData(Connection connection, String databaseName, String s for (int i = 1; i <= metaData.getColumnCount(); i++) { ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); JDBCDataValue jdbcDataValue = new JDBCDataValue(resultSet, metaData, i, false); - String valueString = valueProcessor.getJdbcValueString(jdbcDataValue); + String valueString = valueProcessor.getJdbcSqlValueString(jdbcDataValue); valueList.add(valueString); } String insertSql = sqlBuilder.buildSingleInsertSql(null, null, tableName, columnList, valueList); diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java index 07f2ece5b..93854f2d7 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultMetaService.java @@ -4,6 +4,7 @@ import ai.chat2db.spi.*; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.SQLExecutor; +import ai.chat2db.spi.util.SqlUtils; import com.google.common.collect.Lists; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -110,7 +111,14 @@ public List procedures(Connection connection, String databaseName, St @Override public List columns(Connection connection, String databaseName, String schemaName, String tableName) { - return SQLExecutor.getInstance().columns(connection, StringUtils.isEmpty(databaseName) ? null : databaseName, StringUtils.isEmpty(schemaName) ? null : schemaName, tableName, null); + List columns = SQLExecutor.getInstance().columns(connection, StringUtils.isEmpty(databaseName) ? null : databaseName, StringUtils.isEmpty(schemaName) ? null : schemaName, tableName, null); + if (CollectionUtils.isNotEmpty(columns)) { + for (TableColumn column : columns) { + String columnType = SqlUtils.removeDigits(column.getColumnType()); + column.setColumnType(columnType); + } + } + return columns; } @Override diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java index 6498efb91..cfffd4521 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/jdbc/DefaultSqlBuilder.java @@ -3,6 +3,7 @@ import ai.chat2db.server.tools.common.util.EasyStringUtils; import ai.chat2db.spi.MetaData; import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.ValueProcessor; import ai.chat2db.spi.enums.DmlType; import ai.chat2db.spi.model.*; import ai.chat2db.spi.sql.Chat2DBContext; @@ -371,6 +372,8 @@ private String getInsertSql(String tableName, List
headerList, List headerList, List R execute(Connection connection, String sql, ResultSetFunction fun } } } catch (Exception e) { - log.error("execute:{}", sql,e); + log.error("execute:{}", sql, e); throw new RuntimeException(e); } return null; @@ -77,7 +77,7 @@ public void execute(Connection connection, String sql, ResultSetConsumer consume } } } catch (Exception e) { - log.error("execute:{}", sql,e); + log.error("execute:{}", sql, e); throw new RuntimeException(e); } } @@ -87,11 +87,14 @@ public void execute(Connection connection, String sql, ResultSetConsumer consume // execute(connection, sql, headerConsumer, rowConsumer, true, valueHandler); // } - public void execute(Connection connection, String sql, Consumer> headerConsumer, - Consumer> rowConsumer, boolean limitSize) { + public void execute( + Connection connection, String sql, + Consumer> headerConsumer, + Consumer> rowConsumer, + java.util.function.Function valueFunction, + boolean limitSize) { Assert.notNull(sql, "SQL must not be null"); - - ValueProcessor valueProcessor = Chat2DBContext.getMetaData().getValueProcessor(); try (Statement stmt = connection.createStatement();) { boolean query = stmt.execute(sql); // Represents the query @@ -110,7 +113,8 @@ public void execute(Connection connection, String sql, Consumer> he while (rs.next()) { List row = Lists.newArrayListWithExpectedSize(col); for (int i = 1; i <= col; i++) { - row.add(valueProcessor.getJdbcValue(new JDBCDataValue(rs, resultSetMetaData, i, limitSize))); + JDBCDataValue jdbcDataValue = new JDBCDataValue(rs, resultSetMetaData, i, limitSize); + row.add(valueFunction.apply(jdbcDataValue)); } rowConsumer.accept(row); } @@ -119,7 +123,7 @@ public void execute(Connection connection, String sql, Consumer> he } } } catch (SQLException e) { - log.error("execute:{}", sql,e); + log.error("execute:{}", sql, e); throw new RuntimeException(e); } } @@ -381,7 +385,7 @@ public List columns(Connection connection, String databaseName, Str tableName, String columnName) { try (ResultSet resultSet = connection.getMetaData().getColumns(databaseName, schemaName, tableName, - columnName)) { + columnName)) { return ResultSetUtils.toObjectList(resultSet, TableColumn.class); } catch (Exception e) { throw new RuntimeException(e); @@ -400,8 +404,8 @@ public List columns(Connection connection, String databaseName, Str public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { List tableIndices = Lists.newArrayList(); try (ResultSet resultSet = connection.getMetaData().getIndexInfo(databaseName, schemaName, tableName, - false, - false)) { + false, + false)) { List tableIndexColumns = ResultSetUtils.toObjectList(resultSet, TableIndexColumn.class); tableIndexColumns.stream().filter(c -> c.getIndexName() != null).collect( Collectors.groupingBy(TableIndexColumn::getIndexName)).entrySet() @@ -493,8 +497,10 @@ public List execute(Command command) { // parse sql String type = Chat2DBContext.getConnectInfo().getDbType(); DbType dbType = JdbcUtils.parse2DruidDbType(type); - List sqlList = SqlUtils.parse(command.getScript(), dbType,true); - + List sqlList = Lists.newArrayList(command.getScript()); + if(!command.isSingle()) { + sqlList = SqlUtils.parse(command.getScript(), dbType, true); + } if (CollectionUtils.isEmpty(sqlList)) { throw new BusinessException("dataSource.sqlAnalysisError"); } @@ -577,7 +583,7 @@ private void addRowNumber(ExecuteResult executeResult, int pageNo, int pageSize) Header rowNumberHeader = Header.builder() .name(I18nUtils.getMessage("sqlResult.rowNumber")) .dataType(DataTypeEnum.CHAT2DB_ROW_NUMBER - .getCode()).build(); + .getCode()).build(); executeResult.setHeaderList(EasyCollectionUtils.union(Arrays.asList(rowNumberHeader), headers)); // Add row number @@ -632,7 +638,7 @@ public void execute(Connection connection, String sql, int batchSize, ResultSetC } } } catch (Exception e) { - log.error("execute error:{}", sql,e); + log.error("execute error:{}", sql, e); throw new RuntimeException(e); } } From b332bd2d67d50533eca53cc2dd33f80cf4201cc3 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Thu, 11 Jul 2024 16:40:06 +0800 Subject: [PATCH 67/73] Temporarily cancel the export trigger --- .../chat2db/plugin/oracle/OracleDBManage.java | 2 +- .../chat2db/plugin/oracle/OracleMetaData.java | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java index 209e86175..fe91d6d63 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleDBManage.java @@ -24,8 +24,8 @@ public void exportDatabase(Connection connection, String databaseName, String sc exportTables(connection, databaseName, schemaName, asyncContext); exportViews(connection, asyncContext, schemaName); exportProcedures(connection, schemaName, asyncContext); - exportTriggers(connection, schemaName, asyncContext); exportFunctions(connection, schemaName, asyncContext); +// exportTriggers(connection, schemaName, asyncContext); } private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java index 07116b41b..63902bf9b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/OracleMetaData.java @@ -87,7 +87,7 @@ public String tableDDL(Connection connection, String databaseName, String schema SQLExecutor.getInstance().execute(connection, sql, resultSet -> { try { if (resultSet.next()) { - ddlBuilder.append(resultSet.getString("sql")); + ddlBuilder.append(resultSet.getString("sql")).append(";"); } } catch (SQLException e) { throw new RuntimeException(e); @@ -259,11 +259,15 @@ public Function function(Connection connection, @NotEmpty String databaseName, S function.setDatabaseName(databaseName); function.setSchemaName(schemaName); function.setFunctionName(functionName); - StringBuilder bodyBuilder = new StringBuilder("CREATE "); + StringBuilder bodyBuilder = new StringBuilder("CREATE OR REPLACE "); while (resultSet.next()) { bodyBuilder.append(resultSet.getString("TEXT")).append("\n"); } - function.setFunctionBody(bodyBuilder.toString()); + String functionBody = bodyBuilder.toString().trim(); + if (!functionBody.endsWith("/")) { + functionBody += "\n/"; + } + function.setFunctionBody(functionBody); return function; }); @@ -391,11 +395,15 @@ public Procedure procedure(Connection connection, @NotEmpty String databaseName, procedure.setDatabaseName(databaseName); procedure.setSchemaName(schemaName); procedure.setProcedureName(procedureName); - StringBuilder bodyBuilder = new StringBuilder("CREATE "); + StringBuilder bodyBuilder = new StringBuilder("CREATE OR REPLACE "); while (resultSet.next()) { bodyBuilder.append(resultSet.getString("TEXT")).append("\n"); } - procedure.setProcedureBody(bodyBuilder.toString()); + String procedureBody = bodyBuilder.toString().trim(); // 去掉最后的空白字符 + if (!procedureBody.endsWith("/")) { + procedureBody += "\n/"; + } + procedure.setProcedureBody(procedureBody); return procedure; }); } @@ -412,7 +420,7 @@ public Table view(Connection connection, String databaseName, String schemaName, table.setSchemaName(schemaName); table.setName(viewName); if (resultSet.next()) { - table.setDdl("CREATE VIEW " + viewName + " AS " + resultSet.getString("TEXT")); + table.setDdl("CREATE OR REPLACE VIEW " + viewName + " AS " + resultSet.getString("TEXT")); } return table; }); From 07320fb2e89760eb5867065a6d80a58aa58eabcf Mon Sep 17 00:00:00 2001 From: SwallowGG <1558143046@qq.com> Date: Sat, 13 Jul 2024 15:05:55 +0800 Subject: [PATCH 68/73] fix columnType error --- .../src/main/java/ai/chat2db/spi/util/SqlUtils.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java index 668d81764..310e385dc 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/util/SqlUtils.java @@ -128,10 +128,12 @@ private static SQLTableSource getSQLExprTableSource(SQLTableSource sqlTableSourc public static List parse(String sql, DbType dbType, boolean removeComment) { List list = new ArrayList<>(); try { - if (StringUtils.isBlank(sql)) { return list; } + if (removeComment) { + sql = SQLParserUtils.removeComment(sql, dbType); + } try { if (DbType.oracle.equals(dbType)) { SqlSplitter sqlSplitter = new SqlSplitter(PlSqlLexer.class, ";", false); @@ -154,9 +156,6 @@ public static List parse(String sql, DbType dbType, boolean removeCommen } catch (Exception e) { log.error("sqlSplitProcessor error", e); } - if (removeComment) { - sql = SQLParserUtils.removeComment(sql, dbType); - } // sql = removeDelimiter(sql); if (StringUtils.isBlank(sql)) { return list; From cd68edc8af139086c9c93be92bc232106b7f1a8d Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Sat, 13 Jul 2024 17:37:30 +0800 Subject: [PATCH 69/73] Add Webhook --- .../api/param/message/MessageCreateParam.java | 40 ++++++++++ .../domain/api/service/WebhookSender.java | 13 ++++ .../chat2db-server-domain-core/pom.xml | 10 +++ .../enums/ExternalNotificationTypeEnum.java | 77 +++++++++++++++++++ .../core/notification/BaseWebhookSender.java | 42 ++++++++++ .../notification/DingTalkWebhookSender.java | 70 +++++++++++++++++ .../core/notification/LarkWebhookSender.java | 69 +++++++++++++++++ .../core/notification/WeComWebhookSender.java | 43 +++++++++++ .../start/test/core/WebhookServiceTest.java | 38 +++++++++ 9 files changed, 402 insertions(+) create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/message/MessageCreateParam.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/WebhookSender.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/enums/ExternalNotificationTypeEnum.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/BaseWebhookSender.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/DingTalkWebhookSender.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/LarkWebhookSender.java create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/WeComWebhookSender.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/WebhookServiceTest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/message/MessageCreateParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/message/MessageCreateParam.java new file mode 100644 index 000000000..f15098334 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/message/MessageCreateParam.java @@ -0,0 +1,40 @@ +package ai.chat2db.server.domain.api.param.message; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * @author Juechen + * @version : MessageCreateParam.java + */ +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class MessageCreateParam { + + /** + * 平台类型 + * @see ai.chat2db.server.domain.core.enums.ExternalNotificationTypeEnum + */ + private String platformType; + + /** + * 服务URL + */ + private String serviceUrl; + + /** + * 密钥 + */ + private String secretKey; + + /** + * 消息模版 + */ + private String textTemplate; + + +} diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/WebhookSender.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/WebhookSender.java new file mode 100644 index 000000000..fabfcf6af --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/service/WebhookSender.java @@ -0,0 +1,13 @@ +package ai.chat2db.server.domain.api.service; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; + +/** + * @author Juechen + * @version : WebhookSender.java + */ +public interface WebhookSender { + + void sendMessage(MessageCreateParam param); + +} diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml index 47a86c68a..4916c324b 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml @@ -121,5 +121,15 @@ chat2db-sqlserver ${revision} + + commons-codec + commons-codec + 1.11 + + + com.squareup.okhttp3 + okhttp + 4.9.1 + diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/enums/ExternalNotificationTypeEnum.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/enums/ExternalNotificationTypeEnum.java new file mode 100644 index 000000000..6afbdc6e8 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/enums/ExternalNotificationTypeEnum.java @@ -0,0 +1,77 @@ +package ai.chat2db.server.domain.core.enums; + +import ai.chat2db.server.domain.api.service.WebhookSender; +import ai.chat2db.server.domain.core.notification.DingTalkWebhookSender; +import ai.chat2db.server.domain.core.notification.LarkWebhookSender; +import ai.chat2db.server.domain.core.notification.WeComWebhookSender; +import ai.chat2db.server.tools.base.enums.BaseEnum; +import lombok.Getter; + +/** + * @author Juechen + * @version : ExternalNotificationTypeEnum.java + */ +@Getter +public enum ExternalNotificationTypeEnum implements BaseEnum { + + /** + * 企业微信 + */ + WECOM("WeCom", WeComWebhookSender.class), + + /** + * 钉钉 + */ + DINGTALK("DingTalk", DingTalkWebhookSender.class), + + /** + * 飞书 + */ + LARK("Lark", LarkWebhookSender.class), + + ; + + final String description; + + final Class webhookSender; + + + @Override + public String getCode() { + return this.name(); + } + + public static WebhookSender getWebhookSender(String platformType) { + String lowerCasePlatformType = platformType.toLowerCase(); + switch (lowerCasePlatformType) { + case "wecom": + return new WeComWebhookSender(); + case "dingtalk": + return new DingTalkWebhookSender(); + case "lark": + return new LarkWebhookSender(); + default: + return null; + } + } + + /** + * Get enum by name + * + * @param name + * @return + */ + public static ExternalNotificationTypeEnum getByName(String name) { + for (ExternalNotificationTypeEnum dbTypeEnum : ExternalNotificationTypeEnum.values()) { + if (dbTypeEnum.name().equalsIgnoreCase(name)) { + return dbTypeEnum; + } + } + return null; + } + + ExternalNotificationTypeEnum(String description, Class webhookSender) { + this.description = description; + this.webhookSender = webhookSender; + } +} diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/BaseWebhookSender.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/BaseWebhookSender.java new file mode 100644 index 000000000..c769f072a --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/BaseWebhookSender.java @@ -0,0 +1,42 @@ +package ai.chat2db.server.domain.core.notification; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; +import ai.chat2db.server.domain.api.service.WebhookSender; +import ai.chat2db.server.domain.core.enums.ExternalNotificationTypeEnum; +import org.springframework.stereotype.Service; + +@Service +public class BaseWebhookSender implements WebhookSender { + + /** + * Sends a message through the specified webhook platform. + * + * @param param The parameter object containing message details and platform type. + * @throws IllegalArgumentException if the provided param is null or invalid. + * @throws RuntimeException if an error occurs while attempting to send the message. + */ + public void sendMessage(MessageCreateParam param) throws IllegalArgumentException { + // Validate the input parameter to ensure it's not null and meets the necessary criteria. + if (param == null || param.getPlatformType() == null) { + throw new IllegalArgumentException("MessageCreateParam or its platform type cannot be null."); + } + + try { + // Attempt to retrieve the appropriate WebhookSender based on the platform type. + ExternalNotificationTypeEnum extern = ExternalNotificationTypeEnum.getByName(param.getPlatformType()); + WebhookSender sender = extern.getWebhookSender(param.getPlatformType()); + + // Guard clause for null sender. Ideally, getWebhookSender should prevent this, but it's good to be cautious. + if (sender == null) { + throw new RuntimeException("Failed to retrieve WebhookSender for platform type: " + param.getPlatformType()); + } + + // Send the message. Any exceptions thrown by sendMessage should be caught and handled here. + sender.sendMessage(param); + } catch (Exception e) { + // Wrap and re-throw any runtime exceptions as a checked exception specific to webhook sending. + throw new RuntimeException("An error occurred while sending the message.", e); + } + } + +} diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/DingTalkWebhookSender.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/DingTalkWebhookSender.java new file mode 100644 index 000000000..c4408fe82 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/DingTalkWebhookSender.java @@ -0,0 +1,70 @@ +package ai.chat2db.server.domain.core.notification; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; +import okhttp3.*; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.binary.Base64; +import org.springframework.stereotype.Service; + +/** + * @author Juechen + * @version : DingTalkWebhookSender.java + */ +@Service +public class DingTalkWebhookSender extends BaseWebhookSender { + + private static final String HMAC_SHA256_ALGORITHM = "HmacSHA256"; + + @Override + public void sendMessage(MessageCreateParam param) { + try { + OkHttpClient client = new OkHttpClient(); + String secret = param.getSecretKey(); + Long timestamp = System.currentTimeMillis(); + + String sign = generateSign(secret, timestamp); + + String webhookUrl = param.getServiceUrl() + "&sign=" + sign + "×tamp=" + timestamp; + + + String payload = "{\"msgtype\": \"text\",\"text\": {\"content\": \"" + param.getTextTemplate() + "\"}}"; + RequestBody requestBody = RequestBody.create(payload, MediaType.parse("application/json; charset=utf-8")); + + Request request = new Request.Builder() + .url(webhookUrl) + .post(requestBody) + .header("Content-Type", "application/json") + .build(); + + + Response response = client.newCall(request).execute(); + if (!response.isSuccessful()) { + throw new RuntimeException("Failed to send message: " + response.code()); + } + System.out.println(response.body().string()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } catch (InvalidKeyException e) { + throw new RuntimeException(e); + } + + } + + private static String generateSign(String secret, Long timestamp) throws NoSuchAlgorithmException, UnsupportedEncodingException, InvalidKeyException { + String stringToSign = timestamp + "\n" + secret; + Mac mac = Mac.getInstance(HMAC_SHA256_ALGORITHM); + mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); + byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); + return URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); + } +} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/LarkWebhookSender.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/LarkWebhookSender.java new file mode 100644 index 000000000..477297d33 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/LarkWebhookSender.java @@ -0,0 +1,69 @@ +package ai.chat2db.server.domain.core.notification; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; +import okhttp3.*; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.codec.binary.Base64; +import org.springframework.stereotype.Service; + +/** + * @author Juechen + * @version : LarkWebhookSender.java + */ +@Service +public class LarkWebhookSender extends BaseWebhookSender { + + private static final String HMAC_SHA256_ALGORITHM = "HmacSHA256"; + + @Override + public void sendMessage(MessageCreateParam param) { + try { + OkHttpClient client = new OkHttpClient(); + String webhookUrl = param.getServiceUrl(); + String secret = param.getSecretKey(); + int timestamp = (int) (System.currentTimeMillis() / 1000); + + String signature = GenSign(secret, timestamp); + + String payload = "{\"timestamp\": \"" + timestamp + + "\",\"sign\": \"" + signature + + "\",\"msg_type\":\"text\",\"content\":{\"text\":\""+ param.getTextTemplate() +"\"}}"; + RequestBody body = RequestBody.create(payload, MediaType.parse("application/json; charset=utf-8")); + + + Request request = new Request.Builder() + .url(webhookUrl) + .post(body) + .addHeader("Content-Type", "application/json") + .build(); + + Response response = client.newCall(request).execute(); + if (!response.isSuccessful()) { + throw new RuntimeException("Failed to send message: " + response.code()); + } + System.out.println(response.body().string()); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } catch (InvalidKeyException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private static String GenSign(String secret, int timestamp) throws NoSuchAlgorithmException, InvalidKeyException { + String stringToSign = timestamp + "\n" + secret; + Mac mac = Mac.getInstance(HMAC_SHA256_ALGORITHM); + mac.init(new SecretKeySpec(stringToSign.getBytes(StandardCharsets.UTF_8), HMAC_SHA256_ALGORITHM)); + byte[] signData = mac.doFinal(new byte[]{}); + return new String(Base64.encodeBase64(signData)); + } +} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/WeComWebhookSender.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/WeComWebhookSender.java new file mode 100644 index 000000000..ec4abeabd --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/notification/WeComWebhookSender.java @@ -0,0 +1,43 @@ +package ai.chat2db.server.domain.core.notification; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; +import okhttp3.*; +import org.springframework.stereotype.Service; + +import java.io.IOException; + +/** + * @author Juechen + * @version : WeComWebhookSender.java + */ +@Service +public class WeComWebhookSender extends BaseWebhookSender { + + @Override + public void sendMessage(MessageCreateParam param) { + try { + OkHttpClient client = new OkHttpClient(); + String webhookUrl = param.getServiceUrl(); + String text = param.getTextTemplate(); + + String payload = "{\"msgtype\": \"text\",\"text\": {\"content\": \"" + text + "\"}}"; + + RequestBody requestBody = RequestBody.create(payload, MediaType.parse("application/json; charset=utf-8")); + + Request request = new Request.Builder() + .url(webhookUrl) + .post(requestBody) + .addHeader("Content-Type", "application/json") + .build(); + + Response response = client.newCall(request).execute(); + if (!response.isSuccessful()) { + throw new RuntimeException("Failed to send message: " + response.code()); + } + System.out.println(response.body().string()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/WebhookServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/WebhookServiceTest.java new file mode 100644 index 000000000..930667e1b --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/WebhookServiceTest.java @@ -0,0 +1,38 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.param.message.MessageCreateParam; +import ai.chat2db.server.domain.core.notification.BaseWebhookSender; +import ai.chat2db.server.start.test.TestApplication; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * @author Juechen + * @version : WebhookServiceTest.java + */ +public class WebhookServiceTest extends TestApplication { + + @Autowired + private BaseWebhookSender baseWebhookSender; + + @Test + public void test() { + MessageCreateParam param = new MessageCreateParam(); +// param.setServiceUrl("https://oapi.dingtalk.com/robot/send?access_token=3dc1c8a55a3ba966d38fb37466c93c536ac210895304e2682966252ea8f8a252"); +// param.setSecretKey("SEC5058616c6ea2e5745abeb381d510579538ea5baa7cdd28a386c809289b1f1db9"); +// param.setPlatformType("DingTalk"); +// param.setTextTemplate("你好，钉钉！"); + + param.setServiceUrl("https://open.feishu.cn/open-apis/bot/v2/hook/da4c4585-b320-4a72-8fbe-920b48c4a0c9"); + param.setSecretKey("tm3p2x2IBs8Lh8cBiJo1F"); + param.setPlatformType("LaRK"); + param.setTextTemplate("你好，飞书"); + +// param.setServiceUrl("https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=346b7d1e-39bd-4146-89e4-bca5fe05f5b4"); +// param.setSecretKey(""); +// param.setPlatformType("WeCom"); +// param.setTextTemplate("你好，企业微信"); + baseWebhookSender.sendMessage(param); + } +} From ffc2bc0e1460443651d1a59c7aa73b96d7c16b6e Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 15 Jul 2024 18:04:16 +0800 Subject: [PATCH 70/73] fix(chat2db-mysql): correct string conversion methods in MysqlTimestampProcessor --- .../plugin/mysql/value/sub/MysqlTimestampProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java index 29a6cf60e..64e5c2f16 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/sub/MysqlTimestampProcessor.java @@ -19,12 +19,12 @@ public String convertSQLValueByType(SQLDataValue dataValue) { @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { - return new String(dataValue.getBytes()); + return dataValue.getStringValue(); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return EasyStringUtils.quoteString(new String(dataValue.getBytes())); + return EasyStringUtils.quoteString(dataValue.getStringValue()); } } From 95db332ad3d45ba24e4218c71c4f5e5968f3b0f2 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 15 Jul 2024 21:18:13 +0800 Subject: [PATCH 71/73] fix(chat2db-mysql): optimize valueProcessor --- .../plugin/mariadb/MariaDBMetaData.java | 6 + .../mariadb/value/MariaDBValueProcessor.java | 109 ++++++++++++++++++ .../factory/MariaDBValueProcessorFactory.java | 61 ++++++++++ .../value/sub/MariaDBBitProcessor.java | 76 ++++++++++++ .../value/sub/MariaDBGeometryProcessor.java | 80 +++++++++++++ .../value/sub/MariaDBTimestampProcessor.java | 30 +++++ .../value/sub/MariaDBYearProcessor.java | 31 +++++ .../mysql/value/MysqlValueProcessor.java | 38 +++++- .../factory/MysqlValueProcessorFactory.java | 3 +- .../oracle/value/OracleValueProcessor.java | 38 +++++- .../factory/OracleValueProcessorFactory.java | 2 +- .../value/sub/OracleLongRawProcessor.java | 5 - 12 files changed, 465 insertions(+), 14 deletions(-) create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/MariaDBValueProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/factory/MariaDBValueProcessorFactory.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBBitProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBGeometryProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBTimestampProcessor.java create mode 100644 chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBYearProcessor.java diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/MariaDBMetaData.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/MariaDBMetaData.java index 675cd3f40..32498610e 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/MariaDBMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/MariaDBMetaData.java @@ -1,9 +1,15 @@ package ai.chat2db.plugin.mariadb; +import ai.chat2db.plugin.mariadb.value.MariaDBValueProcessor; import ai.chat2db.plugin.mysql.MysqlMetaData; import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.ValueProcessor; public class MariaDBMetaData extends MysqlMetaData implements MetaData { + @Override + public ValueProcessor getValueProcessor() { + return new MariaDBValueProcessor(); + } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/MariaDBValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/MariaDBValueProcessor.java new file mode 100644 index 000000000..6b71b6b5d --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/MariaDBValueProcessor.java @@ -0,0 +1,109 @@ +package ai.chat2db.plugin.mariadb.value; + +import ai.chat2db.plugin.mariadb.value.factory.MariaDBValueProcessorFactory; +import ai.chat2db.plugin.mysql.value.MysqlValueProcessor; +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年05月24日 21:02 + *
+ * TODO: + * attribute: [zerofill] example tinyint[5] zerofill 34->00034 + */ +public class MariaDBValueProcessor extends MysqlValueProcessor { + + + private static final Logger log = LoggerFactory.getLogger(MariaDBValueProcessor.class); + + @Override + public String getJdbcValue(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + // example: [date]->0000-00-00 + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return stringValue; + } + return null; + } + if (value instanceof String emptyStr) { + if (StringUtils.isBlank(emptyStr)) { + return emptyStr; + } + } + return convertJDBCValueByType(dataValue); + } + + + @Override + public String getJdbcSqlValueString(JDBCDataValue dataValue) { + Object value = dataValue.getObject(); + if (Objects.isNull(value)) { + // example: [date]->0000-00-00 + String stringValue = dataValue.getStringValue(); + if (Objects.nonNull(stringValue)) { + return EasyStringUtils.escapeAndQuoteString(stringValue); + } + return "NULL"; + } + if (value instanceof String stringValue) { + if (StringUtils.isBlank(stringValue)) { + return EasyStringUtils.quoteString(stringValue); + } + } + return convertJDBCValueStrByType(dataValue); + } + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + try { + DefaultValueProcessor valueProcessor = MariaDBValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()); + if (Objects.isNull(valueProcessor)) { + return super.convertSQLValueByType(dataValue); + } + return valueProcessor.convertSQLValueByType(dataValue); + } catch (Exception e) { + log.warn("convertSQLValueByType error", e); + return super.convertSQLValueByType(dataValue); + } + } + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + String type = dataValue.getType(); + try { + DefaultValueProcessor valueProcessor = MariaDBValueProcessorFactory.getValueProcessor(type); + if (Objects.isNull(valueProcessor)) { + return super.convertJDBCValueByType(dataValue); + } + return valueProcessor.convertJDBCValueByType(dataValue); + } catch (Exception e) { + log.warn("convertJDBCValueByType error", e); + return super.convertJDBCValueByType(dataValue); + } + } + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + String type = dataValue.getType(); + try { + DefaultValueProcessor valueProcessor = MariaDBValueProcessorFactory.getValueProcessor(type); + if (Objects.isNull(valueProcessor)) { + return super.convertJDBCValueByType(dataValue); + } + return valueProcessor.convertJDBCValueStrByType(dataValue); + } catch (Exception e) { + log.warn("convertJDBCValueStrByType error", e); + return super.convertJDBCValueStrByType(dataValue); + } + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/factory/MariaDBValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/factory/MariaDBValueProcessorFactory.java new file mode 100644 index 000000000..80e96013c --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/factory/MariaDBValueProcessorFactory.java @@ -0,0 +1,61 @@ +package ai.chat2db.plugin.mariadb.value.factory; + +import ai.chat2db.plugin.mariadb.value.sub.MariaDBBitProcessor; +import ai.chat2db.plugin.mariadb.value.sub.MariaDBGeometryProcessor; +import ai.chat2db.plugin.mariadb.value.sub.MariaDBTimestampProcessor; +import ai.chat2db.plugin.mariadb.value.sub.MariaDBYearProcessor; +import ai.chat2db.plugin.mysql.type.MysqlColumnTypeEnum; +import ai.chat2db.plugin.mysql.value.sub.*; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; + +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年06月03日 23:16 + */ +public class MariaDBValueProcessorFactory { + + private static final Map PROCESSOR_MAP; + + static { + MariaDBGeometryProcessor mariaDBGeometryProcessor = new MariaDBGeometryProcessor(); + MysqlVarBinaryProcessor mysqlVarBinaryProcessor = new MysqlVarBinaryProcessor(); + MariaDBTimestampProcessor mariaDBTimestampProcessor = new MariaDBTimestampProcessor(); + MysqlTextProcessor mysqlTextProcessor = new MysqlTextProcessor(); + PROCESSOR_MAP = Map.ofEntries( + //text + Map.entry(MysqlColumnTypeEnum.TEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.TINYTEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.MEDIUMTEXT.name(), mysqlTextProcessor), + Map.entry(MysqlColumnTypeEnum.LONGTEXT.name(), mysqlTextProcessor), + // geometry + Map.entry(MysqlColumnTypeEnum.GEOMETRY.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.POINT.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.LINESTRING.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.POLYGON.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTIPOINT.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTILINESTRING.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.MULTIPOLYGON.name(), mariaDBGeometryProcessor), + Map.entry(MysqlColumnTypeEnum.GEOMETRYCOLLECTION.name(), mariaDBGeometryProcessor), + // binary + Map.entry(MysqlColumnTypeEnum.VARBINARY.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.BLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.LONGBLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.TINYBLOB.name(), mysqlVarBinaryProcessor), + Map.entry(MysqlColumnTypeEnum.MEDIUMBLOB.name(), mysqlVarBinaryProcessor), + // timestamp + Map.entry(MysqlColumnTypeEnum.TIMESTAMP.name(), mariaDBTimestampProcessor), + Map.entry(MysqlColumnTypeEnum.DATETIME.name(), mariaDBTimestampProcessor), + //others + Map.entry(MysqlColumnTypeEnum.YEAR.name(), new MariaDBYearProcessor()), + Map.entry(MysqlColumnTypeEnum.BIT.name(), new MariaDBBitProcessor()), + Map.entry(MysqlColumnTypeEnum.DECIMAL.name(), new MysqlDecimalProcessor()), + Map.entry(MysqlColumnTypeEnum.BINARY.name(), new MysqlBinaryProcessor()) + ); + } + + public static DefaultValueProcessor getValueProcessor(String type) { + return PROCESSOR_MAP.get(type); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBBitProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBBitProcessor.java new file mode 100644 index 000000000..46729d895 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBBitProcessor.java @@ -0,0 +1,76 @@ +package ai.chat2db.plugin.mariadb.value.sub; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.apache.commons.lang3.StringUtils; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Function; + +/** + * @author: zgq + * @date: 2024年06月01日 13:08 + */ +public class MariaDBBitProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return getString(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return getValue(dataValue, s -> s); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return getValue(dataValue, this::wrap); + } + + private String getValue(JDBCDataValue dataValue, Function function) { + try { + //mariadb tinyint(1) + if ((dataValue.getMetaData().getColumnType(dataValue.getColumnIndex()) == -7)) { + return String.valueOf(dataValue.getInt()); + } + } catch (SQLException e) { + super.convertJDBCValueByType(dataValue); + } + int precision = dataValue.getPrecision(); + byte[] bytes = dataValue.getBytes(); + if (precision == 1) { + //bit(1) [1 -> true] [0 -> false] + if (bytes.length == 1 && (bytes[0] == 0 || bytes[0] == 1)) { + return String.valueOf(dataValue.getBoolean()); + } + } + //bit(m) m: 2~64 + return function.apply(EasyStringUtils.getBitString(bytes, precision)); + } + + public String getString(String value) { + + if (Objects.equals("true", value.toLowerCase())) { + return "1"; + } + if (Objects.equals("false", value.toLowerCase())) { + return "0"; + } + if (StringUtils.isBlank(value)) { + return "NULL"; + } + return MysqlDmlValueTemplate.wrapBit(value); + } + + private String wrap(String value) { + return MysqlDmlValueTemplate.wrapBit(value); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBGeometryProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBGeometryProcessor.java new file mode 100644 index 000000000..2e9b0060c --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBGeometryProcessor.java @@ -0,0 +1,80 @@ +package ai.chat2db.plugin.mariadb.value.sub; + +import ai.chat2db.plugin.mysql.value.template.MysqlDmlValueTemplate; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.WKBReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; + +/** + * @author: zgq + * @date: 2024年06月01日 12:42 + */ +public class MariaDBGeometryProcessor extends DefaultValueProcessor { + + + private static final Logger log = LoggerFactory.getLogger(MariaDBGeometryProcessor.class); + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return MysqlDmlValueTemplate.wrapGeometry(dataValue.getValue()); + } + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + try { + Geometry dbGeometry = null; + byte[] geometryAsBytes = dataValue.getBytes(); + if (geometryAsBytes != null) { + if (geometryAsBytes.length < 5) { + throw new Exception("Invalid geometry inputStream - less than five bytes"); + } + + //first four bytes of the geometry are the SRID, + //followed by the actual WKB. Determine the SRID + //here + byte[] sridBytes = new byte[4]; + System.arraycopy(geometryAsBytes, 0, sridBytes, 0, 4); + boolean bigEndian = (geometryAsBytes[4] == 0x00); + + int srid = 0; + if (bigEndian) { + for (int i = 0; i < sridBytes.length; i++) { + srid = (srid << 8) + (sridBytes[i] & 0xff); + } + } else { + for (int i = 0; i < sridBytes.length; i++) { + srid += (sridBytes[i] & 0xff) << (8 * i); + } + } + + //use the JTS WKBReader for WKB parsing + WKBReader wkbReader = new WKBReader(); + + //copy the byte array, removing the first four + //SRID bytes + byte[] wkb = new byte[geometryAsBytes.length - 4]; + System.arraycopy(geometryAsBytes, 4, wkb, 0, wkb.length); + dbGeometry = wkbReader.read(wkb); + dbGeometry.setSRID(srid); + } + return dbGeometry != null ? dbGeometry.toString() : null; + } catch (Exception e) { + log.warn("Error converting database geometry", e); + return dataValue.getStringValue(); + } + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return MysqlDmlValueTemplate.wrapGeometry(convertJDBCValueByType(dataValue)); + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBTimestampProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBTimestampProcessor.java new file mode 100644 index 000000000..b6d4a6cc3 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBTimestampProcessor.java @@ -0,0 +1,30 @@ +package ai.chat2db.plugin.mariadb.value.sub; + +import ai.chat2db.server.tools.common.util.EasyStringUtils; +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * @author: zgq + * @date: 2024年06月01日 18:26 + */ +public class MariaDBTimestampProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getValue()); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getStringValue(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return EasyStringUtils.quoteString(dataValue.getStringValue()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBYearProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBYearProcessor.java new file mode 100644 index 000000000..e29543134 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-mariadb/src/main/java/ai/chat2db/plugin/mariadb/value/sub/MariaDBYearProcessor.java @@ -0,0 +1,31 @@ +package ai.chat2db.plugin.mariadb.value.sub; + +import ai.chat2db.spi.jdbc.DefaultValueProcessor; +import ai.chat2db.spi.model.JDBCDataValue; +import ai.chat2db.spi.model.SQLDataValue; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年07月15日 20:19 + */ +public class MariaDBYearProcessor extends DefaultValueProcessor { + + @Override + public String convertSQLValueByType(SQLDataValue dataValue) { + return dataValue.getValue(); + } + + + @Override + public String convertJDBCValueByType(JDBCDataValue dataValue) { + return dataValue.getStringValue(); + } + + + @Override + public String convertJDBCValueStrByType(JDBCDataValue dataValue) { + return dataValue.getStringValue(); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java index 08a0a148d..2345476ef 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/MysqlValueProcessor.java @@ -6,6 +6,8 @@ import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Objects; import java.util.Set; @@ -19,6 +21,7 @@ */ public class MysqlValueProcessor extends DefaultValueProcessor { public static final Set FUNCTION_SET = Set.of("now()", "default"); + private static final Logger log = LoggerFactory.getLogger(MysqlValueProcessor.class); @Override @@ -65,18 +68,47 @@ public String convertSQLValueByType(SQLDataValue dataValue) { if (FUNCTION_SET.contains(dataValue.getValue().toLowerCase())) { return dataValue.getValue(); } - return MysqlValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()).convertSQLValueByType(dataValue); + try { + DefaultValueProcessor valueProcessor = MysqlValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertSQLValueByType(dataValue); + } + } catch (Exception e) { + log.warn("convertSQLValueByType error", e); + return super.convertSQLValueByType(dataValue); + } + return super.convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return MysqlValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); + try { + DefaultValueProcessor valueProcessor = MysqlValueProcessorFactory.getValueProcessor(type); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertJDBCValueByType(dataValue); + } + } catch (Exception e) { + log.warn("convertJDBCValueByType error", e); + return super.convertJDBCValueByType(dataValue); + } + return super.convertJDBCValueByType(dataValue); + } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return MysqlValueProcessorFactory.getValueProcessor(type).convertJDBCValueStrByType(dataValue); + DefaultValueProcessor valueProcessor; + try { + valueProcessor = MysqlValueProcessorFactory.getValueProcessor(type); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertJDBCValueStrByType(dataValue); + } + } catch (Exception e) { + log.warn("convertJDBCValueStrByType error", e); + return super.convertJDBCValueStrByType(dataValue); + } + return super.convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java index 4e8ae2512..4a685e192 100644 --- a/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-mysql/src/main/java/ai/chat2db/plugin/mysql/value/factory/MysqlValueProcessorFactory.java @@ -52,7 +52,6 @@ public class MysqlValueProcessorFactory { } public static DefaultValueProcessor getValueProcessor(String type) { - DefaultValueProcessor processor = PROCESSOR_MAP.get(type); - return processor == null ? new DefaultValueProcessor() : processor; + return PROCESSOR_MAP.get(type); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java index 746ed852c..7841ec0ad 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/OracleValueProcessor.java @@ -7,6 +7,8 @@ import ai.chat2db.spi.model.JDBCDataValue; import ai.chat2db.spi.model.SQLDataValue; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Objects; @@ -17,6 +19,8 @@ public class OracleValueProcessor extends DefaultValueProcessor { + private static final Logger log = LoggerFactory.getLogger(OracleValueProcessor.class); + @Override public String getJdbcValue(JDBCDataValue dataValue) { if (OracleColumnTypeEnum.LONG_RAW.getColumnType().getTypeName().equalsIgnoreCase(dataValue.getType())) { @@ -54,19 +58,47 @@ public String getJdbcSqlValueString(JDBCDataValue dataValue) { @Override public String convertSQLValueByType(SQLDataValue dataValue) { - return OracleValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()).convertSQLValueByType(dataValue); + try { + DefaultValueProcessor valueProcessor = OracleValueProcessorFactory.getValueProcessor(dataValue.getDateTypeName()); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertSQLValueByType(dataValue); + } + } catch (Exception e) { + log.warn("convertSQLValueByType error", e); + return super.convertSQLValueByType(dataValue); + } + return super.convertSQLValueByType(dataValue); } @Override public String convertJDBCValueByType(JDBCDataValue dataValue) { String type = dataValue.getType(); - return OracleValueProcessorFactory.getValueProcessor(type).convertJDBCValueByType(dataValue); + try { + DefaultValueProcessor valueProcessor = OracleValueProcessorFactory.getValueProcessor(type); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertJDBCValueByType(dataValue); + } + } catch (Exception e) { + log.warn("convertJDBCValueByType error", e); + return super.convertJDBCValueByType(dataValue); + } + return super.convertJDBCValueByType(dataValue); } @Override public String convertJDBCValueStrByType(JDBCDataValue dataValue) { - return OracleValueProcessorFactory.getValueProcessor(dataValue.getType()).convertJDBCValueStrByType(dataValue); + String type = dataValue.getType(); + try { + DefaultValueProcessor valueProcessor = OracleValueProcessorFactory.getValueProcessor(type); + if (Objects.nonNull(valueProcessor)) { + return valueProcessor.convertJDBCValueStrByType(dataValue); + } + } catch (Exception e) { + log.warn("convertJDBCValueStrByType error", e); + return super.convertJDBCValueStrByType(dataValue); + } + return super.convertJDBCValueStrByType(dataValue); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java index fbd8d1dc9..ae5deeaeb 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/factory/OracleValueProcessorFactory.java @@ -49,7 +49,7 @@ public class OracleValueProcessorFactory { } public static DefaultValueProcessor getValueProcessor(String type) { - return PROCESSOR_MAP.getOrDefault(type, new DefaultValueProcessor()); + return PROCESSOR_MAP.get(type); } } diff --git a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java index dba376689..56347dfac 100644 --- a/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java +++ b/chat2db-server/chat2db-plugins/chat2db-oracle/src/main/java/ai/chat2db/plugin/oracle/value/sub/OracleLongRawProcessor.java @@ -52,9 +52,4 @@ public String convertJDBCValueStrByType(JDBCDataValue dataValue) { return EasyStringUtils.quoteString(blobHexString); } - public static void main(String[] args) { - String value = "0x123456"; - value = value.substring(2); - System.out.println("value = " + value); - } } From 60e47c2132664c29d388c9ecb1948cb9dac66c6a Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 16 Jul 2024 14:45:39 +0800 Subject: [PATCH 72/73] refactor(chat2db-spi): replace Clob reading logic with IOUtils.toString --- .../ai/chat2db/spi/model/JDBCDataValue.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java index 685b3308a..c03618804 100644 --- a/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java +++ b/chat2db-server/chat2db-spi/src/main/java/ai/chat2db/spi/model/JDBCDataValue.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; +import org.apache.commons.io.IOUtils; import org.apache.tika.Tika; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -12,10 +13,7 @@ import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.math.BigDecimal; import java.sql.*; import java.util.Objects; @@ -126,7 +124,7 @@ public String getBlobString() { public String getClobString() { Clob clob = getClob(); - try (BufferedReader reader = new BufferedReader(clob.getCharacterStream())) { + try (Reader reader = clob.getCharacterStream()) { long length = clob.length(); LOBInfo cLobInfo = getLobInfo(length); double size = cLobInfo.getSize(); @@ -137,14 +135,7 @@ public String getClobString() { if (limitSize && isBigSize(unit)) { return String.format("[%s] %s", getType(), cLobInfo); } - StringBuilder builder = new StringBuilder((int) (Math.ceil(size))); - String line; - - while ((line = reader.readLine()) != null) { - // TODO: 优化换行符 - builder.append(line).append("\n"); - } - return builder.toString(); + return IOUtils.toString(reader); } catch (IOException | SQLException e) { log.warn("Error while reading clob stream", e); return getStringValue(); From 82bb7c24026ffbc21a0c5a96bfebdd7b81ebf83d Mon Sep 17 00:00:00 2001 From: suyue <2016494681@qq.com> Date: Mon, 22 Jul 2024 16:03:55 +0800 Subject: [PATCH 73/73] unit test --- .../test/core/DataSourceServiceTest.java | 2 +- .../test/core/DlTemplateServiceTest.java | 81 +++++++++++++++++ .../test/core/EnvironmentServiceTest.java | 91 +++++++++++++++++++ .../start/test/core/FunctionServiceTest.java | 67 ++++++++++++++ .../test/core/JdbcDriverServiceTest.java | 69 ++++++++++++++ .../test/core/OperationLogServiceTest.java | 27 ++++++ 6 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/EnvironmentServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/FunctionServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/JdbcDriverServiceTest.java create mode 100644 chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/OperationLogServiceTest.java diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java index 8adbfdb87..5ab3fa7c1 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DataSourceServiceTest.java @@ -152,7 +152,7 @@ public void testPreConnect() { param.setUrl(dialectProperties.getUrl()); param.setPassword(dialectProperties.getPassword()); param.setPort(String.valueOf(dialectProperties.getPort())); - param.setHost("183.247.151.185"); + param.setHost("localhost"); param.setSsh(new SSHInfo()); param.setSsl(new SSLInfo()); param.setExtendInfo(new ArrayList()); diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java index dd96ba4cb..c081ebc69 100644 --- a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/DlTemplateServiceTest.java @@ -1,6 +1,9 @@ package ai.chat2db.server.start.test.core; +import ai.chat2db.server.domain.api.param.DlCountParam; import ai.chat2db.server.domain.api.param.DlExecuteParam; +import ai.chat2db.server.domain.api.param.OrderByParam; +import ai.chat2db.server.domain.api.param.UpdateSelectResultParam; import ai.chat2db.server.domain.api.service.DlTemplateService; import ai.chat2db.server.domain.repository.Dbutils; import ai.chat2db.server.start.test.TestApplication; @@ -12,11 +15,15 @@ import ai.chat2db.server.tools.common.model.LoginUser; import ai.chat2db.server.tools.common.util.ContextUtils; import ai.chat2db.spi.model.ExecuteResult; +import ai.chat2db.spi.model.Header; +import ai.chat2db.spi.model.OrderBy; +import ai.chat2db.spi.model.ResultOperation; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -140,16 +147,90 @@ public void testExecuteUpdate() { @Test public void testCount() { + userLoginIdentity(true, 8L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + DlCountParam param = new DlCountParam(); + DataResult count = null; + if (dialectProperties.getDbType().equals("MYSQL")) { + param.setSql("select * from test_data"); + count = dlTemplateService.count(param); + System.out.println("Mysql Total rows:" + count.getData()); + Assertions.assertTrue(count.getSuccess(), count.errorMessage()); + } else if (dialectProperties.getDbType().equals("ORACLE")) { + param.setSql("select * from TEST_USER.DEMO"); + count = dlTemplateService.count(param); + System.out.println("Oracle Total rows:" + count.getData()); + Assertions.assertTrue(count.getSuccess(), count.errorMessage()); + }else if (dialectProperties.getDbType().equals("POSTGRESQL")) { + param.setSql("select * from test.reference_table"); + count = dlTemplateService.count(param); + System.out.println("PG Total rows:" + count.getData()); + Assertions.assertTrue(count.getSuccess(), count.errorMessage()); + } else if (dialectProperties.getDbType().equals("MARIADB")) { + param.setSql("select * from test.test_data"); + count = dlTemplateService.count(param); + System.out.println("Mariadb Total rows:" + count.getData()); + Assertions.assertTrue(count.getSuccess(), count.errorMessage()); + } + } + } @Test public void testUpdateSelectResult() { + userLoginIdentity(true, 8L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + UpdateSelectResultParam param = new UpdateSelectResultParam(); + param.setTableName("test_data"); + param.setHeaderList(new ArrayList
()); + param.setOperations(new ArrayList()); + + DataResult result = dlTemplateService.updateSelectResult(param); + System.out.println("result:" + result.getData()); + Assertions.assertTrue(result.getSuccess(), result.errorMessage()); + } + } @Test public void testGetOrderBySql() { + userLoginIdentity(false, 4L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + if (dialectProperties.getDbType().equals("MYSQL")) { + ArrayList orderByList = new ArrayList<>(); + OrderBy orderBy1 = new OrderBy(); + orderBy1.setColumnName("number"); + orderBy1.setAsc(true); + orderByList.add(orderBy1); + + OrderByParam orderByParam = new OrderByParam(); + orderByParam.setOrderByList(orderByList); + orderByParam.setOriginSql("select * from test_data"); + + DataResult result = dlTemplateService.getOrderBySql(orderByParam); + System.out.println("Mysql Final Sql:" + result.getData()); + Assertions.assertTrue(result.getSuccess(), result.errorMessage()); + + } + } + } /** diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/EnvironmentServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/EnvironmentServiceTest.java new file mode 100644 index 000000000..84a651e61 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/EnvironmentServiceTest.java @@ -0,0 +1,91 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.model.Environment; +import ai.chat2db.server.domain.api.param.EnvironmentPageQueryParam; +import ai.chat2db.server.domain.api.service.EnvironmentService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.base.wrapper.result.PageResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Juechen + * @version : EnvironmentServiceTest.java + */ +public class EnvironmentServiceTest extends TestApplication { + + @Autowired + private EnvironmentService environmentService; + + @Autowired + private List dialectPropertiesList; + + @Test + public void testListQuery() { + + userLoginIdentity(false, 6L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + ArrayList list = new ArrayList<>(); + list.add(1L); + list.add(2L); + list.add(3L); + + ListResult query = environmentService.listQuery(list); + Assertions.assertTrue(query.getSuccess(), query.getErrorMessage()); + Assertions.assertFalse(query.getData().isEmpty(), "Result should not be empty for non-empty input list"); + } + } + + @Test + public void testPageQuery() { + + userLoginIdentity(false, 3L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + EnvironmentPageQueryParam param = new EnvironmentPageQueryParam(); + param.setSearchKey("release"); +// param.setSearchKey("test"); + param.setPageNo(1); + param.setPageSize(10); + + PageResult query = environmentService.pageQuery(param); + Assertions.assertTrue(query.getSuccess(), query.getErrorMessage()); + Assertions.assertFalse(query.getData().isEmpty(), "Result should not be empty for non-empty input list"); + } + } + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } + +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/FunctionServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/FunctionServiceTest.java new file mode 100644 index 000000000..33b240a7b --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/FunctionServiceTest.java @@ -0,0 +1,67 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.service.FunctionService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.start.test.dialect.DialectProperties; +import ai.chat2db.server.start.test.dialect.TestUtils; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.base.wrapper.result.ListResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import ai.chat2db.spi.model.Function; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +/** + * @author Juechen + * @version : FunctionServiceTest.java + */ +public class FunctionServiceTest extends TestApplication { + + @Autowired + private FunctionService functionService; + + @Autowired + private List dialectPropertiesList; + + @Test + public void testFunctions() { + + userLoginIdentity(false, 3L); + + for (DialectProperties dialectProperties : dialectPropertiesList) { + Long dataSourceId = TestUtils.nextLong(); + Long consoleId = TestUtils.nextLong(); + TestUtils.buildContext(dialectProperties, dataSourceId, consoleId); + + ListResult functions = functionService.functions(dialectProperties.getDatabaseName(), null); + Assertions.assertTrue(functions.getSuccess(), functions.errorMessage()); + + if (dialectProperties.getDbType().equals("MYSQL")) { + DataResult detail = functionService.detail(dialectProperties.getDatabaseName(), null, "add_numbers"); + Assertions.assertTrue(detail.getSuccess(), detail.errorMessage()); + } + + } + } + + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/JdbcDriverServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/JdbcDriverServiceTest.java new file mode 100644 index 000000000..8ad28e1fc --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/JdbcDriverServiceTest.java @@ -0,0 +1,69 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.service.JdbcDriverService; +import ai.chat2db.server.domain.repository.Dbutils; +import ai.chat2db.server.start.test.TestApplication; +import ai.chat2db.server.tools.base.wrapper.result.ActionResult; +import ai.chat2db.server.tools.base.wrapper.result.DataResult; +import ai.chat2db.server.tools.common.model.Context; +import ai.chat2db.server.tools.common.model.LoginUser; +import ai.chat2db.server.tools.common.util.ContextUtils; +import ai.chat2db.spi.config.DBConfig; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @author Juechen + * @version : JdbcDriverServiceTest.java + */ +public class JdbcDriverServiceTest extends TestApplication { + + @Autowired + private JdbcDriverService jdbcDriverService; + + @Test + public void testGetDrivers() { + + userLoginIdentity(false, 2L); + + String dbType = "POSTGRESQL"; + DataResult drivers = jdbcDriverService.getDrivers(dbType); + Assertions.assertTrue(drivers.success(), drivers.errorMessage()); + } + + @Test + public void testUpload() { + + userLoginIdentity(false, 2L); + + String dbType = "MYSQL"; + ActionResult result = jdbcDriverService.upload(dbType, "com.mysql.cj.jdbc.Driver", "mysql-connector-java-8.0.30.jar"); + Assertions.assertTrue(result.success(), result.errorMessage()); + } + + @Test + public void testDownload() { + userLoginIdentity(false, 5L); + + String dbType = "ORACLE"; + DataResult drivers = jdbcDriverService.getDrivers(dbType); + Assertions.assertTrue(drivers.success(), drivers.errorMessage()); + } + + + + /** + * Save the current user identity (administrator or normal user) and user ID to the context and database session for subsequent use. + * + * @param isAdmin + * @param userId + */ + private static void userLoginIdentity(boolean isAdmin, Long userId) { + Context context = Context.builder().loginUser( + LoginUser.builder().admin(isAdmin).id(userId).build() + ).build(); + ContextUtils.setContext(context); + Dbutils.setSession(); + } +} diff --git a/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/OperationLogServiceTest.java b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/OperationLogServiceTest.java new file mode 100644 index 000000000..30f3490a1 --- /dev/null +++ b/chat2db-server/chat2db-server-start/src/test/java/ai/chat2db/server/start/test/core/OperationLogServiceTest.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.start.test.core; + +import ai.chat2db.server.domain.api.service.OperationLogService; +import ai.chat2db.server.start.test.TestApplication; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @author Juechen + * @version : OperationLogServiceTest.java + */ +public class OperationLogServiceTest extends TestApplication { + + @Autowired + private OperationLogService operationLogService; + + @Test + public void testCreate() { + operationLogService.create(null); + } + + @Test + public void testQueryPage() { + operationLogService.queryPage(null); + } + +}