Skip to content

Commit

Permalink
UPDATE: Fix how batches are inserted into db
Browse files Browse the repository at this point in the history
  • Loading branch information
aueb-wim committed Dec 5, 2017
1 parent 04b0b64 commit 569bf63
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 21 deletions.
4 changes: 2 additions & 2 deletions mipmapEngine/build.properties
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#Build Number for ANT. Do not edit!
#Wed Nov 22 16:04:54 EET 2017
build.number=2081
#Tue Dec 05 17:35:30 EET 2017
build.number=2167
major.version=1
minor.version=0
4 changes: 4 additions & 0 deletions mipmapEngine/src/it/unibas/spicy/model/algebra/query/operators/sql/ExecuteSQL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.io.Reader;
import java.io.StringReader;
import java.sql.Connection;
import java.sql.JDBCType;
import java.sql.SQLWarning;
import java.sql.Statement;
import java.util.HashSet;
Expand Down Expand Up @@ -110,7 +111,10 @@ public void executeScript(MappingTask mappingTask, AccessConfiguration accessCon
}

Statement statement = connection.createStatement();

System.out.println("Starting Data Translation" + new java.util.Date());
statement.execute(sqlScript);
System.out.println("Data Translation Ended with " + statement.getUpdateCount() + "\t insertions\t" + new java.util.Date());
SQLWarning warning = statement.getWarnings();
String notice = SpicyEngineConstants.PRIMARY_KEY_CONSTR_NOTICE;
while (warning != null){
Expand Down
2 changes: 2 additions & 0 deletions mipmapEngine/src/it/unibas/spicy/persistence/DAOHandleDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ public void createNewDatabase() throws DAOException{
dropSchemataQuery.append("drop schema if exists ").append(res.getString("schemaName")).append(" cascade;\n");
}
statement2.executeUpdate(dropSchemataQuery.toString());
// System.out.println("DROP SCHEMATA QUERY IN TEMP DB\n------------------------------\n" + dropSchemataQuery.toString());
}
else{
StringBuilder createDatabaseQuery = new StringBuilder();
createDatabaseQuery.append("create database ").append(SpicyEngineConstants.MAPPING_TASK_DB_NAME).append(";\n");
statement.executeUpdate(createDatabaseQuery.toString());
// System.out.println("CREATE DB QUERY IN TEMP DB\n------------------------------\n" + createDatabaseQuery.toString());
}
}
catch (Exception ex) {
Expand Down
4 changes: 1 addition & 3 deletions mipmapEngine/src/it/unibas/spicy/persistence/csv/DAOCsv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,9 +531,7 @@ private LeafNode createLeafNode(INode attributeNode, Object untypedValue) throws
LeafNode leafNodeInSchema = (LeafNode) attributeNode.getChild(0);
String type = leafNodeInSchema.getLabel();
Object typedValue = Types.getTypedValue(type, untypedValue);
//avenet 26/9/2017 - Add support of UTF-8
System.out.println("getInstanceByTable");
return new LeafNode(type, typedValue);
return new LeafNode(type, typedValue);
}

@SuppressWarnings("unchecked")
Expand Down
71 changes: 59 additions & 12 deletions mipmapEngine/src/it/unibas/spicy/persistence/relational/DAORelational.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ private TupleNode getTuple(DatabaseMetaData databaseMetaData, String catalog, St
String typeOfColumn = DAORelationalUtility.convertDBTypeToDataSourceType(columnType);
columnNode.addChild(new LeafNode(typeOfColumn));
tupleNode.addChild(columnNode);
// if(tableName.contains("encounter_mapping")) System.out.println("\n\tColumn Name: " + columnName + "(" + columnType + ") " + " type of column= " + typeOfColumn + "[IS_Nullable: " + isNullable + "]");
if (logger.isDebugEnabled()) logger.debug("\n\tColumn Name: " + columnName + "(" + columnType + ") " + " type of column= " + typeOfColumn + "[IS_Nullable: " + isNullable + "]");
}
//take out the last ',' character
Expand All @@ -263,6 +264,26 @@ private TupleNode getTuple(DatabaseMetaData databaseMetaData, String catalog, St
statement.executeUpdate("drop table if exists "+ table);
statement.executeUpdate("create table "+ table +" ("+ columns+ ")");

// if(tableName.contains("encounter_mapping")) {
// System.out.println("-----------------------------------------------\n"
// + "create table "+ table +" ("+ columns+ ")"
// + "\n-----------------------------------------------");
//
// System.out.println("Running query");
//
// ResultSet queryResult = statement.executeQuery("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE "
// + "table_name = '"+ tableName + "' ORDER BY ordinal_position;");
//
// System.out.println("Query run, waiting for results");
// while (queryResult.next()) {
// System.out.println(queryResult.getString("column_name") + "\t" +
// queryResult.getString("data_type") + "\t" + queryResult.getString("is_nullable"));
// }
//
// System.out.println("END of results");
// }
//

return tupleNode;
}

Expand Down Expand Up @@ -502,46 +523,72 @@ public void loadInstance(int scenarioNo, AccessConfiguration accessConfiguration
}
String newTablePath = tableName;
if (source){
newTablePath = SpicyEngineConstants.SOURCE_SCHEMA_NAME+ scenarioNo+".\""+tableName+"\"";
newTablePath = SpicyEngineConstants.SOURCE_SCHEMA_NAME+ scenarioNo+".\""+tableName+"\"";
}
else{
newTablePath = SpicyEngineConstants.TARGET_SCHEMA_NAME+ scenarioNo+".\""+tableName+"\"";
newTablePath = SpicyEngineConstants.TARGET_SCHEMA_NAME+ scenarioNo+".\""+tableName+"\"";
}
ResultSet countResult = statement.executeQuery("SELECT COUNT(*) AS instancesCount FROM " +tablePath+";");
int instancesCount = 1;
while(countResult.next()){
instancesCount = countResult.getInt("instancesCount");
}
}

ResultSet pKList = null;
pKList = databaseMetaData.getPrimaryKeys(null, null, tableName);
// ResultSet pKList = statement.executeQuery("SELECT c.column_name as keyname\n" + "FROM information_schema.key_column_usage AS c\n" +
// "LEFT JOIN information_schema.table_constraints AS t\n" +
// "ON t.constraint_name = c.constraint_name\n" +
// "WHERE t.table_name = '" + tablePath + "' AND t.constraint_type = 'PRIMARY KEY';");
String pKListString = "";
while (pKList.next()) {
pKListString += pKList.getString("COLUMN_NAME") + ",";
}
if (pKListString != "" )
pKListString = pKListString.substring(0, pKListString.length()-1);

int inCount = 0;
String viewName = tableName + "_MIPMapView";
String orderByClause = "";
if ( pKListString != "" )
orderByClause = " ORDER BY " + pKListString;
statement.executeUpdate("DROP VIEW IF EXISTS " + viewName + ";");
statement.executeUpdate("CREATE VIEW " + viewName + " AS SELECT * FROM " + tablePath + orderByClause + ";");
for (int i=0; i<=((instancesCount-1)/BATCH_SIZE); i++){
ResultSet instancesSet = statement.executeQuery("SELECT * FROM " +tablePath+" LIMIT "+BATCH_SIZE+" OFFSET "+BATCH_SIZE*i+";");
ResultSet instancesSet = statement.executeQuery("SELECT * FROM " +viewName + " LIMIT "+BATCH_SIZE+" OFFSET "+(BATCH_SIZE*i)+";");
ResultSetMetaData rsmd = instancesSet.getMetaData();
int columnsNumber = rsmd.getColumnCount();
String sql_insert_stmnt="";
String sql_insert_stmnt= "";
while (instancesSet.next()){
sql_insert_stmnt += "(";
String tmp_sql_insert_stmnt = "(";
for (int j=1; j<=columnsNumber; j++){
String columnValue = instancesSet.getString(j);
if (columnValue == null){
sql_insert_stmnt += " null,";
tmp_sql_insert_stmnt += " null,";
} else {
if(isTextColumn(rsmd.getColumnTypeName(j))){
sql_insert_stmnt += "'"+columnValue.replaceAll("'", "''") +"',";
tmp_sql_insert_stmnt += "'"+columnValue.replaceAll("'", "''") +"',";
} else {
sql_insert_stmnt += ""+columnValue +",";
tmp_sql_insert_stmnt += ""+columnValue +",";
}
}

}
//take out the last ',' character
sql_insert_stmnt = sql_insert_stmnt.substring(0, sql_insert_stmnt.length()-1);
sql_insert_stmnt += "),";
tmp_sql_insert_stmnt = tmp_sql_insert_stmnt.substring(0, tmp_sql_insert_stmnt.length()-1);
tmp_sql_insert_stmnt += "),";
// if (!inserted.contains(tmp_sql_insert_stmnt)) {
sql_insert_stmnt+=tmp_sql_insert_stmnt;
// inserted.add(tmp_sql_insert_stmnt);
// }
}
if (!sql_insert_stmnt.equals("")){
//take out the last ',' character
sql_insert_stmnt = sql_insert_stmnt.substring(0, sql_insert_stmnt.length()-1);
statementPostgres.executeUpdate("insert into "+newTablePath+" values "+sql_insert_stmnt+";");
inCount += statementPostgres.executeUpdate("insert into "+newTablePath+" values "+sql_insert_stmnt+";");
}
}
statement.executeUpdate("DROP VIEW IF EXISTS " + viewName + ";");
}
dataSource.addAnnotation(SpicyEngineConstants.LOADED_INSTANCES_FLAG, true);
}
Expand Down
9 changes: 5 additions & 4 deletions mipmapEngine/src/it/unibas/spicy/persistence/sql/ExportSQLInstances.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ private Connection getConnectionToDatabase(IConnectionFactory connectionFactory,
}

public void exportSQLInstances(MappingTask mappingTask, int scenarioNo, String driver, String uri,
String userName, String password) throws DAOException, SQLException, IOException{
String userName, String password) throws DAOException, SQLException, IOException{
//connection to Postgres
connectionFactory = new SimpleDbConnectionFactory();
Connection connection = getConnectionToPostgres(connectionFactory);
Expand Down Expand Up @@ -104,7 +104,7 @@ public void exportSQLInstances(MappingTask mappingTask, int scenarioNo, String d
ResultSet tableColumns = statement.executeQuery("SELECT column_name, data_type, is_nullable "+
" FROM information_schema.columns WHERE " + " table_schema = '" + SpicyEngineConstants.TARGET_SCHEMA_NAME+String.valueOf(scenarioNo)
+ "' AND table_name = '"+ tableName + "' ORDER BY ordinal_position;");

ResultSet pkConstraints = databaseMetaData.getPrimaryKeys(SpicyEngineConstants.MAPPING_TASK_DB_NAME,
SpicyEngineConstants.TARGET_SCHEMA_NAME+scenarioNo, tableName);

Expand Down Expand Up @@ -275,7 +275,7 @@ private boolean checkTablesIntegrityOfExistingDatabase(Connection connectionCrea
}

ArrayList<TableSchema> targetDatabaseTables = new ArrayList<>();
statement = connection.createStatement();
statement = connection.createStatement();
//get table names from target database
DatabaseMetaData databaseMetaData = connection.getMetaData();
ResultSet tableResultSet = databaseMetaData.getTables(SpicyEngineConstants.MAPPING_TASK_DB_NAME,
Expand All @@ -293,7 +293,6 @@ private boolean checkTablesIntegrityOfExistingDatabase(Connection connectionCrea
} else if (database == 1) {
t.addDataType(searchMappings(tableColumns.getString("data_type"), 1));
}

t.addIsNull(tableColumns.getString("is_nullable"));
t.addColumn(tableColumns.getString("column_name"));
}
Expand Down Expand Up @@ -372,7 +371,9 @@ private TableSchema searchTableName(String tableName, ArrayList<TableSchema> t){
private void insertIntoDbPerRow(ResultSet tableRows, String tableName, Statement statementCreateAndInsertToTable) throws SQLException{
ResultSetMetaData rsmd = tableRows.getMetaData();
int columnsNumber = rsmd.getColumnCount();
int rowsnumber = 0;
while(tableRows.next()){
rowsnumber++;
String insertIntoScript = "INSERT INTO " + tableName + " VALUES \n";
insertIntoScript += "(";
for(int i=1;i<=columnsNumber;i++){
Expand Down

0 comments on commit 569bf63

Please sign in to comment.