Skip to content

Commit

Permalink
#155: ao10: update db from contentprovider
Browse files Browse the repository at this point in the history
  • Loading branch information
@k3b
k3b committed Feb 23, 2021
1 parent 3de6314 commit 754872e
Show file tree
Hide file tree
Showing 7 changed files with 132 additions and 24 deletions.
5 changes: 4 additions & 1 deletion app/src/main/java/de/k3b/android/androFotoFinder/FotoGalleryActivity.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2020 by k3b.
* Copyright (c) 2015-2021 by k3b.
*
* This file is part of AndroFotoFinder / #APhotoManager.
*
Expand Down Expand Up @@ -208,6 +208,9 @@ public boolean onOptionsItemSelected(MenuItem item) {
case R.id.cmd_db_reload:
AndroFotoFinderApp.getMediaContent2DbUpdateService().rebuild(this, null);
return true;
case R.id.cmd_db_update:
AndroFotoFinderApp.getMediaContent2DbUpdateService().update(this, null);
return true;
case R.id.cmd_more:
new Handler().postDelayed(new Runnable() {
public void run() {
Expand Down
25 changes: 22 additions & 3 deletions app/src/main/java/de/k3b/android/androFotoFinder/queries/FotoSql.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,11 @@ public class FotoSql extends FotoSqlBase {
// SQL_COL_LAST_MODIFIED in seconds since 1970; "?" in milli-seconds since 1970
private static final String FILTER_EXPR_DATE_MODIFIED_MAX = SQL_COL_LAST_MODIFIED + " < ?";
private static final String FILTER_EXPR_DATE_MODIFIED_MIN = SQL_COL_LAST_MODIFIED + " >= ?";

// SQL_COL_DATE_ADDED in seconds since 1970; "?" in milli-seconds since 1970
private static final String FILTER_EXPR_DATE_ADDED_MAX = SQL_COL_DATE_ADDED + " < ?";
private static final String FILTER_EXPR_DATE_ADDED_MIN = SQL_COL_DATE_ADDED + " >= ?";

protected static final String FILTER_EXPR_PATH_LIKE = "(" + SQL_COL_PATH + " like ?)";

// same format as dir. i.e. description='/2014/12/24/' or '/mnt/sdcard/pictures/'
Expand Down Expand Up @@ -405,12 +410,26 @@ public static void addWhereDateMinMax(QueryParameter resultQuery, final long dat
public static void addWhereDateModifiedMinMax(QueryParameter resultQuery, final long dateMinInMilliSecs1970, final long dateMaxInMilliSecs1970) {

// SQL_COL_LAST_MODIFIED in seconds since 1970; translate from MilliSecs
if (dateMinInMilliSecs1970 != 0) resultQuery.addWhere(FILTER_EXPR_DATE_MODIFIED_MIN, Long.toString(dateMinInMilliSecs1970 / LAST_MODIFIED_FACTOR));
if (dateMinInMilliSecs1970 != 0)
resultQuery.addWhere(FILTER_EXPR_DATE_MODIFIED_MIN, Long.toString(dateMinInMilliSecs1970 / LAST_MODIFIED_FACTOR));

if (dateMaxInMilliSecs1970 != 0)
resultQuery.addWhere(FILTER_EXPR_DATE_MODIFIED_MAX, Long.toString(dateMaxInMilliSecs1970 / LAST_MODIFIED_FACTOR));
}

if (dateMaxInMilliSecs1970 != 0) resultQuery.addWhere(FILTER_EXPR_DATE_MODIFIED_MAX, Long.toString(dateMaxInMilliSecs1970 / LAST_MODIFIED_FACTOR));
public static void addWhereDateAddedMinMax(QueryParameter resultQuery, final long dateMinInMilliSecs1970, final long dateMaxInMilliSecs1970) {

// SQL_COL_DATE_ADDED in seconds since 1970; translate from MilliSecs
if (dateMinInMilliSecs1970 != 0)
resultQuery.addWhere(FILTER_EXPR_DATE_ADDED_MIN, Long.toString(dateMinInMilliSecs1970 / LAST_MODIFIED_FACTOR));

if (dateMaxInMilliSecs1970 != 0)
resultQuery.addWhere(FILTER_EXPR_DATE_ADDED_MAX, Long.toString(dateMaxInMilliSecs1970 / LAST_MODIFIED_FACTOR));
}

/** translates a query back to filter */
/**
* translates a query back to filter
*/
public static IGalleryFilter parseQuery(QueryParameter query, boolean removeFromSourceQuery) {
if (query != null) {
GalleryFilterParameter filter = new GalleryFilterParameter();
Expand Down
16 changes: 13 additions & 3 deletions app/src/main/java/de/k3b/android/androFotoFinder/queries/MediaContent2DBUpdateService.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2020 by k3b.
* Copyright (c) 2019-2021 by k3b.
*
* This file is part of AndroFotoFinder / #APhotoManager.
*
Expand Down Expand Up @@ -29,7 +29,7 @@
import de.k3b.io.IProgessListener;

/**
* #155: takes care that chages from
* #155: takes care that changes from
* {@link MediaContentproviderRepository} are transfered to {@link MediaDBRepository}
*/
public class MediaContent2DBUpdateService {
Expand Down Expand Up @@ -60,11 +60,21 @@ public void clearMediaCopy() {
public void rebuild(Context context, IProgessListener progessListener) {
long start = new Date().getTime();
clearMediaCopy();
MediaDBRepository.Impl.updateMediaCopy(context, writableDatabase, null, progessListener);
MediaDBRepository.Impl.updateMediaCopy(context, writableDatabase, null, null, progessListener);
start = (new Date().getTime() - start) / 1000;
final String text = "load db " + start + " secs";
Toast.makeText(context, text, Toast.LENGTH_LONG).show();
if (progessListener != null) progessListener.onProgress(0, 0, text);

}

public void update(Context context, IProgessListener progessListener) {
long start = new Date().getTime();
MediaDBRepository.Impl.updateMediaCopy(context, writableDatabase, progessListener);
start = (new Date().getTime() - start) / 1000;
final String text = "update db " + start + " secs";
Toast.makeText(context, text, Toast.LENGTH_LONG).show();
if (progessListener != null) progessListener.onProgress(0, 0, text);
}


Expand Down
102 changes: 85 additions & 17 deletions app/src/main/java/de/k3b/android/androFotoFinder/queries/MediaDBRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,20 @@

import android.content.ContentValues;
import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import android.database.DatabaseUtils;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteStatement;
import android.net.Uri;
import android.os.Build;
import android.os.CancellationSignal;
import android.preference.PreferenceManager;
import android.provider.MediaStore;
import android.util.Log;

import java.sql.Date;
import java.util.Calendar;

import de.k3b.LibGlobal;
import de.k3b.android.androFotoFinder.Global;
Expand Down Expand Up @@ -86,7 +89,9 @@ public MediaDBRepository(SQLiteDatabase db) {
public Cursor createCursorForQuery(StringBuilder out_debugMessage, String dbgContext,
QueryParameter parameters, VISIBILITY visibility,
CancellationSignal cancellationSignal) {
if (visibility != null) FotoSql.setWhereVisibility(parameters, visibility);
if (visibility != null) {
FotoSql.setWhereVisibility(parameters, visibility);
}
return createCursorForQuery(out_debugMessage, dbgContext,
parameters.toWhere(), parameters.toAndroidParameters(),
parameters.toGroupBy(), parameters.toHaving(),
Expand Down Expand Up @@ -475,6 +480,7 @@ public static class Impl {
.addColumn(USED_MEDIA_COLUMNS)
.addFrom(SQL_TABLE_EXTERNAL_CONTENT_URI_FILE_NAME)
.addWhere(FILTER_EXPR_AFFECTED_FILES);
private static long nextMonthTimeInSecs;

private static boolean isLomg(int index) {
return index >= intMin && index <= intMax;
Expand Down Expand Up @@ -517,7 +523,7 @@ private static String getSqlUpdateWithParams() {
return sql.toString();
}

private static int bindAndExecUpdate(Cursor c, SQLiteStatement sql) {
private static int bindAndExecUpdate(Cursor c, SQLiteStatement sql, long dateAdded, long dateUpdated) {
sql.clearBindings();

// sql where
Expand All @@ -538,10 +544,16 @@ private static int bindAndExecUpdate(Cursor c, SQLiteStatement sql) {
sql.bindDouble(i, c.getDouble(i));
}
}
if (dateAdded != 0) {
sql.bindLong(colDATE_ADDED, dateAdded);
}
if (dateUpdated != 0) {
sql.bindLong(colLAST_MODIFIED, dateUpdated);
}
return sql.executeUpdateDelete();
}

private static void bindAndExecInsert(Cursor c, SQLiteStatement sql) {
private static void bindAndExecInsert(Cursor c, SQLiteStatement sql, long dateAdded, long dateUpdated) {
sql.clearBindings();

for (int i = intMin; i <= intMax; i++) {
Expand All @@ -559,6 +571,12 @@ private static void bindAndExecInsert(Cursor c, SQLiteStatement sql) {
sql.bindDouble(i + 1, c.getDouble(i));
}
}
if (dateAdded != 0) {
sql.bindLong(colDATE_ADDED + 1, dateAdded);
}
if (dateUpdated != 0) {
sql.bindLong(colLAST_MODIFIED + 1, dateUpdated);
}
sql.executeInsert();
}

Expand Down Expand Up @@ -590,19 +608,37 @@ public static void clearMedaiCopy(SQLiteDatabase db) {
}
}

public static int updateMediaCopy(
Context context, SQLiteDatabase db,
IProgessListener progessListener) {
SharedPreferences prefsInstance = PreferenceManager
.getDefaultSharedPreferences(context.getApplicationContext());
long maxDateAddedSecs = prefsInstance.getLong("maxDateAddedSecs", 0l);
return updateMediaCopy(context, db, null, new Date(maxDateAddedSecs), progessListener);
}

public static int updateMediaCopy(Context context, SQLiteDatabase db, Date lastUpdate, IProgessListener progessListener) {
public static int updateMediaCopy(
Context context, SQLiteDatabase db,
Date filterLastUpdateMin, Date filterLastAddedMin, IProgessListener progessListener) {
int progress = 0;
java.util.Date startTime = new java.util.Date();

QueryParameter query = queryGetAllColumns;
long _lastUpdate = (lastUpdate != null) ? (lastUpdate.getTime() / 1000L) : 0L;
QueryParameter query = new QueryParameter().getFrom(queryGetAllColumns);

if (_lastUpdate != 0) {
query = new QueryParameter().getFrom(queryGetAllColumns);
FotoSql.addWhereDateModifiedMinMax(query, _lastUpdate, 0);
// FotoSql.createCursorForQuery()
Calendar nextMonth = Calendar.getInstance();
nextMonth.add(Calendar.MONTH, 1);
nextMonthTimeInSecs = nextMonth.getTimeInMillis() / 1000;

long filterLastUpdateMinInMillis = (filterLastUpdateMin != null) ? (filterLastUpdateMin.getTime()) : 0L;
if (filterLastUpdateMinInMillis != 0) {
FotoSql.addWhereDateModifiedMinMax(query, filterLastUpdateMinInMillis, 0);
}

long filterLastAddedMinInMillis = (filterLastAddedMin != null) ? (filterLastAddedMin.getTime()) : 0L;
if (filterLastAddedMinInMillis != 0) {
FotoSql.addWhereDateAddedMinMax(query, filterLastAddedMinInMillis, nextMonth.getTimeInMillis());
}

Cursor c = null;
SQLiteStatement sqlInsert = null;
SQLiteStatement sqlUpdate = null;
Expand All @@ -611,7 +647,7 @@ public static int updateMediaCopy(Context context, SQLiteDatabase db, Date lastU
int itemCount = 0;
int insertCout = 0;
int updateCount = 0;
// ContentValues contentValues = new ContentValues();

try {
db.beginTransaction(); // Performance boost: all db-inserts/updates in one transaction

Expand All @@ -624,35 +660,48 @@ public static int updateMediaCopy(Context context, SQLiteDatabase db, Date lastU

sqlInsert = db.compileStatement(getSqlInsertWithParams());
sqlUpdate = db.compileStatement(getSqlUpdateWithParams());
long maxDateAddedSecs = 0;
long maxDateUpdatedSecs = 0;
while (c.moveToNext()) {
// getContentValues(c, contentValues);

isUpdate = (c.getLong(colDATE_ADDED) <= _lastUpdate);
long curDateAddedSecs = getDateInSecs(c, colDATE_ADDED);
if (curDateAddedSecs > maxDateAddedSecs) {
maxDateAddedSecs = curDateAddedSecs;
}
isUpdate = (curDateAddedSecs <= filterLastUpdateMinInMillis / 1000);

long curDateUpdatedSecs = getDateInSecs(c, colLAST_MODIFIED);
if (curDateUpdatedSecs > maxDateUpdatedSecs) {
maxDateUpdatedSecs = curDateUpdatedSecs;
}

if (isUpdate) {
updateCount++;
lastSql = sqlUpdate;
isUpdate = bindAndExecUpdate(c, sqlUpdate) > 0;
isUpdate = bindAndExecUpdate(c, sqlUpdate, curDateAddedSecs, curDateUpdatedSecs) > 0;
// 0 affected update rows: must insert
}

if (!isUpdate) {
insertCout++;
lastSql = sqlInsert;
bindAndExecInsert(c, sqlInsert);
bindAndExecInsert(c, sqlInsert, curDateAddedSecs, curDateUpdatedSecs);
}

lastSql = null;
// save(db, c, contentValues, _lastUpdate);

if ((progessListener != null) && (progress % 100) == 0) {
if (!progessListener.onProgress(progress, itemCount, context.getString(R.string.scanner_update_result_format, progress))) {
// canceled in gui thread
return -progress;
}
}
progress++;
}
} // while over all old items
db.setTransactionSuccessful(); // This commits the transaction if there were no exceptions

saveStats(context, maxDateAddedSecs, maxDateUpdatedSecs);

if (Global.debugEnabledSql) {
java.util.Date endTime = new java.util.Date();
final String message = "MediaDBRepository.updateMedaiCopy(inserted:" + insertCout +
Expand Down Expand Up @@ -685,6 +734,25 @@ public static int updateMediaCopy(Context context, SQLiteDatabase db, Date lastU
return progress;
}

private static void saveStats(Context context, long maxDateAddedSecs, long maxDateUpdatedSecs) {
SharedPreferences prefsInstance = PreferenceManager
.getDefaultSharedPreferences(context.getApplicationContext());

SharedPreferences.Editor prefs = prefsInstance.edit();
if (maxDateUpdatedSecs > 0) prefs.putLong("maxDateUpdatedSecs", maxDateUpdatedSecs + 1);
if (maxDateAddedSecs > 0) prefs.putLong("maxDateAddedSecs", maxDateAddedSecs + 1);
prefs.apply();
}

protected static long getDateInSecs(Cursor c, int colPosition) {
long curDateAdded = (c.isNull(colPosition)) ? 0 : c.getLong(colPosition);
if (curDateAdded > nextMonthTimeInSecs) {
// colDATE_ADDED: some apps/libs use milliscs instead of secs. Fix this.
curDateAdded = curDateAdded / 1000;
}
return curDateAdded;
}

private static void save(SQLiteDatabase db, Cursor c, ContentValues contentValues, long lastUpdate) {
boolean isNew = (c.getLong(colDATE_ADDED) > lastUpdate);

Expand Down
6 changes: 6 additions & 0 deletions app/src/main/res/menu/menu_ao10.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,10 @@
android:title="@string/load_db_menu_title"
android:visible="true" />

<item
android:id="@+id/cmd_db_update"
android:orderInCategory="9900"
android:showAsAction="never"
android:title="@string/update_db_menu_title"
android:visible="true" />
</menu>
1 change: 1 addition & 0 deletions app/src/main/res/values-de/strings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ Verstecken kann über das \'Media-Scanner\' Gallery-Menü rückgängig gemacht w
<string name="dark_theme_title">Dunkel</string>
<!-- #155: android10 -->
<string name="load_db_menu_title">Mediendatenbank (neu) laden</string>
<string name="update_db_menu_title">Mediendatenbank aktualisieren</string>
<!-- #169: sd-card-saf-write-permission support -->
<string name="permission_error">Fehlende Berechtigung</string>
<string name="select_folder_root_rationale"><![CDATA["<h2>Benötigt Verzeichnisberechtigungen.</h2>
Expand Down
1 change: 1 addition & 0 deletions app/src/main/res/values/strings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ You can undo hiding by calling the mediascanner from gallery-menu."</string>

<!-- #155: android10 -->
<string name="load_db_menu_title">(Re)Load Media Database</string>
<string name="update_db_menu_title">Update Media Database</string>

<!-- #169: sd-card-saf-write-permission support -->
<string name="permission_error">Missing permisions</string>
Expand Down

0 comments on commit 754872e

Please sign in to comment.