diff --git a/app/src/main/java/com/fieldbook/tracker/async/ImportRunnableTask.java b/app/src/main/java/com/fieldbook/tracker/async/ImportRunnableTask.java index b8f815476..814b78ab2 100644 --- a/app/src/main/java/com/fieldbook/tracker/async/ImportRunnableTask.java +++ b/app/src/main/java/com/fieldbook/tracker/async/ImportRunnableTask.java @@ -1,10 +1,13 @@ package com.fieldbook.tracker.async; +import android.app.AlertDialog; import android.app.ProgressDialog; import android.content.Context; +import android.content.DialogInterface; import android.content.SharedPreferences; import android.os.AsyncTask; import android.text.Html; +import android.util.Log; import androidx.preference.PreferenceManager; @@ -15,6 +18,7 @@ import com.fieldbook.tracker.objects.FieldFileObject; import com.fieldbook.tracker.objects.FieldObject; import com.fieldbook.tracker.preferences.GeneralKeys; +import com.fieldbook.tracker.utilities.StringUtil; import com.fieldbook.tracker.utilities.Utils; import java.lang.ref.WeakReference; @@ -33,8 +37,10 @@ public class ImportRunnableTask extends AsyncTask { int lineFail = -1; boolean fail; + private CharSequence failMessage; boolean uniqueFail; boolean containsDuplicates = false; + private static final String TAG = "ImportRunnableTask"; public ImportRunnableTask(Context context, FieldFileObject.FieldFileBase fieldFile, int idColPosition, String unique, String primary, String secondary) { @@ -82,9 +88,11 @@ protected Integer doInBackground(Integer... params) { } if (mFieldFile.hasSpecialCharacters()) { + Log.d(TAG, "doInBackground: Special characters found in file column names"); return 0; } + mFieldFile.open(); String[] data; String[] columns = mFieldFile.readNext(); @@ -107,7 +115,7 @@ protected Integer doInBackground(Integer... params) { //populate an array of indices that have a non empty column //later we will only add data rows with the non empty columns //also find the unique/primary/secondary indices - //later we will skip the rows if these are not present + //later we will return an error if these are not present if (!columns[i].isEmpty()) { if (!nonEmptyColumns.contains(columns[i])) { @@ -136,9 +144,7 @@ protected Integer doInBackground(Integer... params) { //start iterating over all the rows of the csv file only if we found the u/p/s indices if (uniqueIndex > -1 && primaryIndex > -1 && secondaryIndex > -1) { - int line = 0; - try { while (true) { data = mFieldFile.readNext(); @@ -164,6 +170,34 @@ protected Integer doInBackground(Integer... params) { controller.getDatabase().createFieldData(studyId, nonEmptyColumns, nonEmptyData); + } else { + fail = true; + + String fixFileMessage = mContext.get().getString(R.string.import_runnable_create_field_fix_file); + String missingIdMessageTemplate = mContext.get().getString(R.string.import_runnable_create_field_missing_identifier); + + String missingField = null; + String fieldValue = null; + + if (data[uniqueIndex].isEmpty()) { + missingField = mContext.get().getString(R.string.import_dialog_unique).toLowerCase(); + fieldValue = unique; + } else if (data[primaryIndex].isEmpty()) { + missingField = mContext.get().getString(R.string.import_dialog_primary).toLowerCase(); + fieldValue = primary; + } else if (data[secondaryIndex].isEmpty()) { + missingField = mContext.get().getString(R.string.import_dialog_secondary).toLowerCase(); + fieldValue = secondary; + } + + if (missingField != null) { + String missingIdMessage = String.format(missingIdMessageTemplate, missingField, fieldValue, line + 1); + failMessage = StringUtil.INSTANCE.applyBoldStyleToString( + String.format("%s\n\n%s", missingIdMessage, fixFileMessage), + fieldValue, + String.valueOf(line + 1) + ); + } } } @@ -171,8 +205,10 @@ protected Integer doInBackground(Integer... params) { } controller.getDatabase().setTransactionSuccessfull(); + Log.d(TAG, "doInBackground: Field data created successfully for study ID: " + studyId); } catch (Exception e) { + Log.e(TAG, "doInBackground: Exception at line " + line + ", Error: " + e.getMessage(), e); lineFail = line; @@ -185,6 +221,8 @@ protected Integer doInBackground(Integer... params) { controller.getDatabase().endTransaction(); } + } else { + Log.d(TAG, "doInBackground: Required indices not found. UniqueIndex: " + uniqueIndex + ", PrimaryIndex: " + primaryIndex + ", SecondaryIndex: " + secondaryIndex); } @@ -198,7 +236,7 @@ protected Integer doInBackground(Integer... params) { } catch (Exception e) { e.printStackTrace(); fail = true; - + failMessage = mContext.get().getString(R.string.import_runnable_create_field_data_failed); controller.getDatabase().close(); controller.getDatabase().open(); } @@ -215,28 +253,27 @@ protected void onPostExecute(Integer result) { if (dialog.isShowing()) dialog.dismiss(); - if (fail | uniqueFail | mFieldFile.hasSpecialCharacters()) { + // Display user feedback in an alert dialog + if (context != null && (uniqueFail || mFieldFile.hasSpecialCharacters())) { + CharSequence errorMessage = mFieldFile.getLastError(); + showAlertDialog(context, "Unable to Import", errorMessage); + } else if (context != null && fail ) { + showAlertDialog(context, "Unable to Import", failMessage); + } else if (containsDuplicates) { + showAlertDialog(context, "Import Warning", context.getString(R.string.import_runnable_duplicates_skipped)); + } + + if (fail || uniqueFail || mFieldFile.hasSpecialCharacters()) { controller.getDatabase().deleteField(result); SharedPreferences.Editor ed = preferences.edit(); ed.putString(GeneralKeys.FIELD_FILE, null); ed.putBoolean(GeneralKeys.IMPORT_FIELD_FINISHED, false); ed.apply(); - } - if (containsDuplicates) { - Utils.makeToast(context, context.getString(R.string.import_runnable_duplicates_skipped)); - } - if (fail) { - Utils.makeToast(context, context.getString(R.string.import_runnable_create_field_data_failed, lineFail)); - //makeToast(getString(R.string.import_error_general)); - } else if (uniqueFail && context != null) { - Utils.makeToast(context,context.getString(R.string.import_error_unique)); - } else if (mFieldFile.hasSpecialCharacters()) { - Utils.makeToast(context,context.getString(R.string.import_error_unique_characters_illegal)); } else { - SharedPreferences.Editor ed = preferences.edit(); + Log.d(TAG, "onPostExecute: Import successful. Field setup for ID: " + result); + SharedPreferences.Editor ed = preferences.edit(); CollectActivity.reloadData = true; - controller.queryAndLoadFields(); try { @@ -262,11 +299,26 @@ protected void onPostExecute(Integer result) { } private boolean verifyUniqueColumn(FieldFileObject.FieldFileBase fieldFile) { - HashMap check = fieldFile.getColumnSet(idColPosition); - if (check.isEmpty()) { + HashMap result = fieldFile.getColumnSet(unique, idColPosition); + if (result == null) { return false; } else { - return controller.getDatabase().checkUnique(check); + return controller.getDatabase().checkUnique(result); } } + + private void showAlertDialog(Context context, String title, CharSequence message) { + new AlertDialog.Builder(context, R.style.AppAlertDialog) + .setTitle(title) + .setMessage(message) + .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + dialog.dismiss(); + } + }) + .setCancelable(false) + .show(); + } + } diff --git a/app/src/main/java/com/fieldbook/tracker/objects/FieldFileObject.java b/app/src/main/java/com/fieldbook/tracker/objects/FieldFileObject.java index 109463cea..60d9edfc2 100644 --- a/app/src/main/java/com/fieldbook/tracker/objects/FieldFileObject.java +++ b/app/src/main/java/com/fieldbook/tracker/objects/FieldFileObject.java @@ -1,27 +1,25 @@ package com.fieldbook.tracker.objects; -import static org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BOOLEAN; -import static org.apache.poi.ss.usermodel.Cell.CELL_TYPE_NUMERIC; -import static org.apache.poi.ss.usermodel.Cell.CELL_TYPE_STRING; - import android.content.Context; import android.database.Cursor; import android.net.Uri; import android.provider.OpenableColumns; +import android.os.Build; +import android.text.Html; +import android.text.Spanned; +import android.util.Log; import androidx.annotation.Nullable; +import com.fieldbook.tracker.R; import com.fieldbook.tracker.utilities.CSVReader; +import com.fieldbook.tracker.utilities.StringUtil; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; -import org.apache.poi.ss.usermodel.FormulaEvaluator; -import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFCell; -import org.apache.poi.xssf.usermodel.XSSFFormulaEvaluator; import org.apache.poi.xssf.usermodel.XSSFRow; import org.apache.poi.xssf.usermodel.XSSFSheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; import java.io.IOException; import java.io.InputStream; @@ -31,9 +29,6 @@ import java.util.Iterator; import java.util.UUID; -import jxl.Workbook; -import jxl.WorkbookSettings; - //TODO when merged with xlsx edit getColumnSet public class FieldFileObject { public static FieldFileBase create(final Context ctx, final Uri path, @@ -77,6 +72,7 @@ public static String getExtensionFromClass(FieldFileBase fieldFile) { public abstract static class FieldFileBase { boolean openFail; boolean specialCharactersFail; + private CharSequence lastErrorMessage = ""; private final Uri path_; private final Context ctx; @@ -89,6 +85,48 @@ public abstract static class FieldFileBase { specialCharactersFail = false; } + protected void setLastError(CharSequence message) { + this.lastErrorMessage = message; + Log.e("FieldFileBase", message.toString()); + } + + public CharSequence getLastError() { + return this.lastErrorMessage; + } + + // Helper method to check unique values + protected boolean checkUnique(HashMap check, String value, String columnLabel, int rowIndex) { + String fixFileMessage = ctx.getString(R.string.import_runnable_create_field_fix_file); + + if (check.containsKey(value)) { + String duplicateErrorMessage = ctx.getString( + R.string.import_runnable_create_field_duplicate_unique_identifier, value, columnLabel, + 1 + ); + setLastError(StringUtil.INSTANCE.applyBoldStyleToString( + String.format("%s\n\n%s", duplicateErrorMessage, fixFileMessage), + value, columnLabel + )); + return false; + } + + for (char specialChar : new char[]{'/', '\\'}) { + if (value.contains(String.valueOf(specialChar))) { + String specialCharErrorMessage = ctx.getString( + R.string.import_runnable_create_field_special_character_error, value, columnLabel, rowIndex + 1, specialChar + ); + setLastError(StringUtil.INSTANCE.applyBoldStyleToString( + String.format("%s\n\n%s", specialCharErrorMessage, fixFileMessage), + value, columnLabel + )); + specialCharactersFail = true; + return false; + } + } + + check.put(value, value); + return true; + } + public final InputStream getInputStream() { try { return this.ctx.getContentResolver().openInputStream(this.path_); @@ -191,9 +229,7 @@ public boolean getOpenFailed() { abstract public String[] getColumns(); - // return {column name: column name} - // if columns are duplicated, return an empty HshMap - abstract public HashMap getColumnSet(int idColPosition); + abstract public HashMap getColumnSet(String unique, int idColPosition); // read file abstract public void open(); @@ -234,39 +270,34 @@ public String[] getColumns() { } } - public HashMap getColumnSet(int idColPosition) { + public HashMap getColumnSet(String columnLabel, int idColPosition) { + HashMap check = new HashMap<>(); try { openFail = false; - HashMap check = new HashMap<>(); InputStreamReader isr = new InputStreamReader(super.getInputStream()); CSVReader cr = new CSVReader(isr); String[] columns = cr.readNext(); + int rowIndex = 0; while (columns != null) { columns = cr.readNext(); - + rowIndex++; if (columns != null) { - String unique = columns[idColPosition]; - if (!unique.isEmpty()) { - if (check.containsKey(unique)) { - cr.close(); - return new HashMap<>(); - } else { - check.put(unique, unique); - } - - if (unique.contains("/") || unique.contains("\\")) { - specialCharactersFail = true; - } + if (!checkUnique(check, columns[idColPosition], columnLabel, rowIndex)) { + close(); + return null; // Return null to indicate an error has occurred } } } - return check; - } catch (Exception n) { + } catch (Exception e) { openFail = true; - n.printStackTrace(); - return new HashMap<>(); + e.printStackTrace(); + setLastError("Failed to process file: " + e.getMessage()); + return null; + } finally { + close(); } + return check; } public void open() { @@ -304,6 +335,8 @@ public static class FieldFileExcel extends FieldFileBase { private Workbook wb; private int current_row; + DataFormatter formatter = new DataFormatter(); + FieldFileExcel(final Context ctx, final Uri path) { super(ctx, path); } @@ -323,63 +356,92 @@ public boolean isOther() { public String[] getColumns() { try { openFail = false; - WorkbookSettings wbSettings = new WorkbookSettings(); - wbSettings.setUseTemporaryFileDuringWrite(true); - InputStream is = super.getInputStream(); if (is != null) { - wb = Workbook.getWorkbook(super.getInputStream(), wbSettings); - String[] importColumns = new String[wb.getSheet(0).getColumns()]; - - for (int s = 0; s < wb.getSheet(0).getColumns(); s++) { - importColumns[s] = wb.getSheet(0).getCell(s, 0).getContents(); + wb = new XSSFWorkbook(is); + Sheet sheet = wb.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) return new String[0]; + + String[] importColumns = new String[headerRow.getLastCellNum()]; + for (int cn = 0; cn < headerRow.getLastCellNum(); cn++) { + Cell cell = headerRow.getCell(cn, Row.RETURN_BLANK_AS_NULL); + importColumns[cn] = (cell == null) ? "" : formatter.formatCellValue(cell); } return importColumns; } - } catch (Exception ignore) { openFail = true; - return new String[0]; } - return new String[0]; } - public HashMap getColumnSet(int idColPosition) { + @Override + public HashMap getColumnSet(String columnLabel, int idColPosition) { HashMap check = new HashMap<>(); + try { + open(); + Sheet sheet = wb.getSheetAt(0); + int totalRows = sheet.getLastRowNum(); - for (int s = 0; s < wb.getSheet(0).getRows(); s++) { - String value = wb.getSheet(0).getCell(idColPosition, s).getContents(); + for (int rowIndex = 0; rowIndex <= totalRows; rowIndex++) { + Row row = sheet.getRow(rowIndex); + Cell cell = row.getCell(idColPosition, Row.RETURN_BLANK_AS_NULL); + String value = cell == null ? "" : formatter.formatCellValue(cell); - if (!value.isEmpty()) { - if (check.containsKey(value)) { - return new HashMap<>(); - } else { - check.put(value, value); + if (value.isEmpty() && isRowEmpty(row)) { + continue; // Skip the row if the specific cell is empty and the whole row is empty } - if (value.contains("/") || value.contains("\\")) { - specialCharactersFail = true; + if (!checkUnique(check, value, columnLabel, rowIndex)) { + return null; } } + } catch (Exception e) { + setLastError("Failed to process Excel file: " + e.getMessage()); + return null; + } finally { + close(); } return check; } + private boolean isRowEmpty(Row row) { + if (row == null) return true; + for (int cellNum = row.getFirstCellNum(); cellNum < row.getLastCellNum(); cellNum++) { + Cell cell = row.getCell(cellNum, Row.RETURN_BLANK_AS_NULL); + if (cell != null && cell.getCellType() != Cell.CELL_TYPE_BLANK) { + return false; + } + } + return true; + } + public void open() { current_row = 0; } public String[] readNext() { - if (current_row >= wb.getSheet(0).getRows()) { + Sheet sheet = wb.getSheetAt(0); + if (current_row > sheet.getLastRowNum()) { return null; } - String[] data = new String[wb.getSheet(0).getColumns()]; - for (int s = 0; s < wb.getSheet(0).getColumns(); s++) { - data[s] = wb.getSheet(0).getCell(s, current_row).getContents(); + Row row = sheet.getRow(current_row); + if (row == null) { + current_row++; + return new String[0]; } - current_row += 1; + + int numCells = row.getLastCellNum(); + String[] data = new String[numCells]; + + for (int cellIndex = 0; cellIndex < numCells; cellIndex++) { + Cell cell = row.getCell(cellIndex, Row.RETURN_BLANK_AS_NULL); + data[cellIndex] = (cell == null) ? "" : formatter.formatCellValue(cell); + } + + current_row++; return data; } @@ -401,6 +463,8 @@ public static class FieldFileXlsx extends FieldFileBase { private XSSFWorkbook wb; private int currentRow; + DataFormatter formatter = new DataFormatter(); + FieldFileXlsx(final Context ctx, final Uri path) { super(ctx, path); } @@ -454,29 +518,44 @@ public String[] getColumns() { return new String[0]; } - public HashMap getColumnSet(int idColPosition) { + @Override + public HashMap getColumnSet(String columnLabel, int idColPosition) { HashMap check = new HashMap<>(); + try { + open(); + XSSFSheet sheet = wb.getSheetAt(0); - XSSFSheet sheet = wb.getSheetAt(0); - - for (Iterator it = sheet.rowIterator(); it.hasNext(); ) { - XSSFRow row = (XSSFRow) it.next(); + for (Iterator it = sheet.rowIterator(); it.hasNext(); ) { + XSSFRow row = (XSSFRow) it.next(); + String value = getCellStringValue(row.getCell(idColPosition)); - String value = getCellStringValue(row.getCell(idColPosition)); + if (value.isEmpty() && isRowEmpty(row)) { + continue; // Skip the row if the specific cell is empty and the whole row is empty + } - if (check.containsKey(value)) { - return new HashMap<>(); - } else { - check.put(value, value); + if (!checkUnique(check, value, columnLabel, row.getRowNum())) { + return null; + } } + } catch (Exception e) { + setLastError("Failed to process XLSX file: " + e.getMessage()); + return null; + } finally { + close(); + } + return check; + } - if (value.contains("/") || value.contains("\\")) { - specialCharactersFail = true; + private boolean isRowEmpty(XSSFRow row) { + if (row == null || row.getLastCellNum() <= 0) { + return true; + } + for (Cell cell : row) { + if (cell != null && cell.getCellType() != Cell.CELL_TYPE_BLANK) { + return false; } - } - - return check; + return true; } public void open() { @@ -484,47 +563,28 @@ public void open() { } public String[] readNext() { - - DataFormatter fmt = new DataFormatter(); XSSFSheet sheet = wb.getSheetAt(0); - XSSFFormulaEvaluator evaluator = wb.getCreationHelper().createFormulaEvaluator(); - - if (currentRow >= sheet.getPhysicalNumberOfRows()) { + if (currentRow > sheet.getLastRowNum()) { return null; } XSSFRow row = sheet.getRow(currentRow); - ArrayList data = new ArrayList<>(); - - int maxColumns = sheet.getRow(0).getLastCellNum(); // Get total number of columns from header - - for (int colIdx = 0; colIdx < maxColumns; colIdx++) { - XSSFCell cell = (row == null) ? null : row.getCell(colIdx); - - if (cell != null) { - if (cell.getCellType() == Cell.CELL_TYPE_FORMULA) {//formula - int type = evaluator.evaluateFormulaCell(cell); - switch (type) { - case CELL_TYPE_BOOLEAN: - data.add(String.valueOf(cell.getBooleanCellValue())); - break; - case CELL_TYPE_NUMERIC: - data.add(String.valueOf(cell.getNumericCellValue())); - break; - default: - data.add(cell.getStringCellValue()); - break; - } - } else { - data.add(fmt.formatCellValue(cell)); - } - } else { - data.add(""); // Add empty string for missing/empty cells - } + if (row == null) { + currentRow++; + return new String[0]; + } + + + int numCells = row.getLastCellNum(); + String[] data = new String[numCells]; + + for (int cellIndex = 0; cellIndex < numCells; cellIndex++) { + XSSFCell cell = row.getCell(cellIndex, Row.RETURN_BLANK_AS_NULL); + data[cellIndex] = (cell == null) ? "" : formatter.formatCellValue(cell); } - currentRow += 1; - return data.toArray(new String[] {}); + currentRow++; + return data; } public void close() { @@ -542,38 +602,20 @@ public void close() { } /** - * Helper function that reads the cell value and parses to string from xlsx sheets. - * @param cell the xssf cell object - * @return attempt to parse the string value of the cell + * Helper function that reads the cell value and formats it as a string, handling different cell types including formulas. + * @param cell the XSSFCell object from an Apache POI XSSFWorkbook. + * @return the formatted string value of the cell. */ private static String getCellStringValue(XSSFCell cell) { - if (cell == null) return ""; - FormulaEvaluator evaluator = cell.getSheet().getWorkbook().getCreationHelper().createFormulaEvaluator(); - - switch (cell.getCellType()) { - case 0: { //numeric - return String.valueOf(cell.getNumericCellValue()); - } - case 1: { //text - return cell.getStringCellValue(); - } - case Cell.CELL_TYPE_FORMULA: { //formula - switch (evaluator.evaluateFormulaCell(cell)) { - case CELL_TYPE_BOOLEAN: - return String.valueOf(cell.getBooleanCellValue()); - case CELL_TYPE_NUMERIC: - return String.valueOf(cell.getNumericCellValue()); - case CELL_TYPE_STRING: - return cell.getStringCellValue(); - } - } - case 3: { //boolean - return String.valueOf(cell.getBooleanCellValue()); - } - default: - return ""; + DataFormatter formatter = new DataFormatter(); + try { + // Use the DataFormatter to handle different data types uniformly + return formatter.formatCellValue(cell, cell.getSheet().getWorkbook().getCreationHelper().createFormulaEvaluator()); + } catch (Exception e) { + Log.e("FieldFileBase", "Error parsing cell value: " + e.getMessage()); + return ""; } } @@ -598,7 +640,7 @@ public String[] getColumns() { return new String[0]; } - public HashMap getColumnSet(int idColPosition) { + public HashMap getColumnSet(String columnLabel, int idColPosition) { return new HashMap<>(); } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 6890ffdcc..921f3b329 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -159,6 +159,12 @@ Database failed to switch fields. Failed to create field data for line %d in file. Duplicate columns found, only first instance will be used. + + Unique identifier value %s in column %s is duplicated on row %d. + Unique identifier value %s in column %s on row %d contains special character "%s" which is not allowed. + Missing value for %s %s on row %d. + Please correct the file and try importing it again. + This field already exists. This field is already imported at this observation level. Field Book could not load the file.