From b964f4907314310e7a978407f7556067d2ec75d9 Mon Sep 17 00:00:00 2001 From: Tianhua Ran Date: Sun, 1 Sep 2019 19:43:11 +0800 Subject: [PATCH] refactor: speed up maintaining database --- .../database/DatabaseMaintainer.java | 188 ++++++++++++++++++ .../database/SqliteDatabaseImpl.java | 37 +++- .../download/CustomComponentHolder.java | 139 +------------ 3 files changed, 218 insertions(+), 146 deletions(-) create mode 100644 library/src/main/java/com/liulishuo/filedownloader/database/DatabaseMaintainer.java diff --git a/library/src/main/java/com/liulishuo/filedownloader/database/DatabaseMaintainer.java b/library/src/main/java/com/liulishuo/filedownloader/database/DatabaseMaintainer.java new file mode 100644 index 00000000..95f9ec65 --- /dev/null +++ b/library/src/main/java/com/liulishuo/filedownloader/database/DatabaseMaintainer.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2015 LingoChamp Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.liulishuo.filedownloader.database; + +import com.liulishuo.filedownloader.model.FileDownloadModel; +import com.liulishuo.filedownloader.model.FileDownloadStatus; +import com.liulishuo.filedownloader.util.FileDownloadExecutors; +import com.liulishuo.filedownloader.util.FileDownloadHelper; +import com.liulishuo.filedownloader.util.FileDownloadLog; +import com.liulishuo.filedownloader.util.FileDownloadUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +public class DatabaseMaintainer { + + private final ThreadPoolExecutor maintainThreadPool; + private final FileDownloadDatabase.Maintainer maintainer; + private final FileDownloadHelper.IdGenerator idGenerator; + + public DatabaseMaintainer(FileDownloadDatabase.Maintainer maintainer, + FileDownloadHelper.IdGenerator idGenerator) { + this.maintainer = maintainer; + this.idGenerator = idGenerator; + this.maintainThreadPool = FileDownloadExecutors.newDefaultThreadPool(3, + FileDownloadUtils.getThreadPoolName("MaintainDatabase")); + } + + public void doMaintainAction() { + final Iterator iterator = maintainer.iterator(); + + final AtomicInteger removedDataCount = new AtomicInteger(0); + final AtomicInteger resetIdCount = new AtomicInteger(0); + final AtomicInteger refreshDataCount = new AtomicInteger(0); + + final long startTimestamp = System.currentTimeMillis(); + final List futures = new ArrayList<>(); + try { + while (iterator.hasNext()) { + final FileDownloadModel model = iterator.next(); + final Future modelFuture = maintainThreadPool.submit(new Runnable() { + @Override + public void run() { + boolean isInvalid = false; + do { + if (model.getStatus() == FileDownloadStatus.progress + || model.getStatus() == FileDownloadStatus.connected + || model.getStatus() == FileDownloadStatus.error + || (model.getStatus() == FileDownloadStatus.pending && model + .getSoFar() > 0) + ) { + // Ensure can be covered by RESUME FROM BREAKPOINT. + model.setStatus(FileDownloadStatus.paused); + } + final String targetFilePath = model.getTargetFilePath(); + if (targetFilePath == null) { + // no target file path, can't used to resume from breakpoint. + isInvalid = true; + break; + } + + final File targetFile = new File(targetFilePath); + if (model.getStatus() == FileDownloadStatus.paused + && FileDownloadUtils.isBreakpointAvailable(model.getId(), model, + model.getPath(), null)) { + // can be reused in the old mechanism(no-temp-file). + + final File tempFile = new File(model.getTempFilePath()); + + if (!tempFile.exists() && targetFile.exists()) { + final boolean successRename = targetFile.renameTo(tempFile); + if (FileDownloadLog.NEED_LOG) { + FileDownloadLog.d(DatabaseMaintainer.class, + "resume from the old no-temp-file architecture " + + "[%B], [%s]->[%s]", + successRename, targetFile.getPath(), + tempFile.getPath()); + + } + } + } + + /** + * Remove {@code model} from DB if it can't used for judging whether the + * old-downloaded file is valid for reused & it can't used for + * resuming from BREAKPOINT, In other words, {@code model} is no + * use anymore for FileDownloader. + */ + if (model.getStatus() == FileDownloadStatus.pending + && model.getSoFar() <= 0) { + // This model is redundant. + isInvalid = true; + break; + } + + if (!FileDownloadUtils.isBreakpointAvailable(model.getId(), model)) { + // It can't used to resuming from breakpoint. + isInvalid = true; + break; + } + + if (targetFile.exists()) { + // It has already completed downloading. + isInvalid = true; + break; + } + + } while (false); + + if (isInvalid) { + maintainer.onRemovedInvalidData(model); + removedDataCount.addAndGet(1); + } else { + final int oldId = model.getId(); + final int newId = idGenerator.transOldId(oldId, + model.getUrl(), model.getPath(), + model.isPathAsDirectory()); + if (newId != oldId) { + if (FileDownloadLog.NEED_LOG) { + FileDownloadLog.d(DatabaseMaintainer.class, + "the id is changed on restoring from db:" + + " old[%d] -> new[%d]", + oldId, newId); + } + model.setId(newId); + maintainer.changeFileDownloadModelId(oldId, model); + resetIdCount.addAndGet(1); + } + + maintainer.onRefreshedValidData(model); + refreshDataCount.addAndGet(1); + } + } + }); + futures.add(modelFuture); + } + } finally { + final Future markConvertedFuture = maintainThreadPool.submit(new Runnable() { + @Override + public void run() { + FileDownloadUtils.markConverted(FileDownloadHelper.getAppContext()); + } + }); + futures.add(markConvertedFuture); + final Future finishMaintainFuture = maintainThreadPool.submit(new Runnable() { + @Override + public void run() { + maintainer.onFinishMaintain(); + } + }); + futures.add(finishMaintainFuture); + for (Future future : futures) { + try { + if (!future.isDone()) future.get(); + } catch (Exception e) { + if (FileDownloadLog.NEED_LOG) FileDownloadLog.e(this, e, e.getMessage()); + } + } + futures.clear(); + if (FileDownloadLog.NEED_LOG) { + FileDownloadLog.d(this, + "refreshed data count: %d , delete data count: %d, reset id count:" + + " %d. consume %d", + refreshDataCount.get(), removedDataCount.get(), resetIdCount.get(), + System.currentTimeMillis() - startTimestamp); + } + } + } +} \ No newline at end of file diff --git a/library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java b/library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java index 43b3adeb..3c6c2d4c 100644 --- a/library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java +++ b/library/src/main/java/com/liulishuo/filedownloader/database/SqliteDatabaseImpl.java @@ -35,22 +35,21 @@ /** * Persist data to SQLite database. - * + *

* You can valid this database implementation through: *

* class MyApplication extends Application { * ... * public void onCreate() { - * ... - * FileDownloader.setupOnApplicationOnCreate(this) - * .database(SqliteDatabaseImpl.createMaker()) - * ... - * .commit(); - * ... + * ... + * FileDownloader.setupOnApplicationOnCreate(this) + * .database(SqliteDatabaseImpl.createMaker()) + * ... + * .commit(); + * ... * } * ... * } - * */ public class SqliteDatabaseImpl implements FileDownloadDatabase { @@ -267,6 +266,7 @@ private void update(final int id, final ContentValues cv) { public class Maintainer implements FileDownloadDatabase.Maintainer { private final SparseArray needChangeIdList = new SparseArray<>(); + private final SparseArray needRemoveList = new SparseArray<>(); private MaintainerIterator currentIterator; private final SparseArray downloaderModelMap; @@ -317,6 +317,14 @@ public void onFinishMaintain() { } } + // remove invalid + final int removeSize = needRemoveList.size(); + for (int i = 0; i < removeSize; i++) { + final int modelId = needRemoveList.keyAt(i); + db.delete(TABLE_NAME, FileDownloadModel.ID + " = ?", + new String[]{String.valueOf(modelId)}); + } + // initial cache of connection model if (downloaderModelMap != null && connectionModelListMap != null) { final int size = downloaderModelMap.size(); @@ -339,16 +347,25 @@ public void onFinishMaintain() { @Override public void onRemovedInvalidData(FileDownloadModel model) { + synchronized (needRemoveList) { + needRemoveList.put(model.getId(), model); + } } @Override public void onRefreshedValidData(FileDownloadModel model) { - if (downloaderModelMap != null) downloaderModelMap.put(model.getId(), model); + if (downloaderModelMap != null) { + synchronized (downloaderModelMap) { + downloaderModelMap.put(model.getId(), model); + } + } } @Override public void changeFileDownloadModelId(int oldId, FileDownloadModel modelWithNewId) { - needChangeIdList.put(oldId, modelWithNewId); + synchronized (needChangeIdList) { + needChangeIdList.put(oldId, modelWithNewId); + } } } diff --git a/library/src/main/java/com/liulishuo/filedownloader/download/CustomComponentHolder.java b/library/src/main/java/com/liulishuo/filedownloader/download/CustomComponentHolder.java index 0bab9b37..7dee35b7 100644 --- a/library/src/main/java/com/liulishuo/filedownloader/download/CustomComponentHolder.java +++ b/library/src/main/java/com/liulishuo/filedownloader/download/CustomComponentHolder.java @@ -17,19 +17,15 @@ package com.liulishuo.filedownloader.download; import com.liulishuo.filedownloader.connection.FileDownloadConnection; +import com.liulishuo.filedownloader.database.DatabaseMaintainer; import com.liulishuo.filedownloader.database.FileDownloadDatabase; -import com.liulishuo.filedownloader.model.FileDownloadModel; -import com.liulishuo.filedownloader.model.FileDownloadStatus; import com.liulishuo.filedownloader.services.DownloadMgrInitialParams; import com.liulishuo.filedownloader.services.ForegroundServiceConfig; import com.liulishuo.filedownloader.stream.FileDownloadOutputStream; import com.liulishuo.filedownloader.util.FileDownloadHelper; -import com.liulishuo.filedownloader.util.FileDownloadLog; -import com.liulishuo.filedownloader.util.FileDownloadUtils; import java.io.File; import java.io.IOException; -import java.util.Iterator; /** * The holder for supported custom components. @@ -88,17 +84,8 @@ public FileDownloadDatabase getDatabaseInstance() { synchronized (this) { if (database == null) { database = getDownloadMgrInitialParams().createDatabase(); - // There is no reusable thread for this action and this action has - // a very low frequency, so, a new thread is simplest way to make this action - // run on background thread. - final String maintainThreadName = - FileDownloadUtils.getThreadPoolName("MaintainDatabase"); - new Thread(new Runnable() { - @Override - public void run() { - maintainDatabase(database.maintainer()); - } - }, maintainThreadName).start(); + new DatabaseMaintainer(database.maintainer(), getImpl().getIdGeneratorInstance()) + .doMaintainAction(); } } @@ -177,124 +164,4 @@ private DownloadMgrInitialParams getDownloadMgrInitialParams() { return initialParams; } - - private static void maintainDatabase(FileDownloadDatabase.Maintainer maintainer) { - final Iterator iterator = maintainer.iterator(); - long refreshDataCount = 0; - long removedDataCount = 0; - long resetIdCount = 0; - final FileDownloadHelper.IdGenerator idGenerator = getImpl().getIdGeneratorInstance(); - - final long startTimestamp = System.currentTimeMillis(); - try { - while (iterator.hasNext()) { - boolean isInvalid = false; - final FileDownloadModel model = iterator.next(); - do { - if (model.getStatus() == FileDownloadStatus.progress - || model.getStatus() == FileDownloadStatus.connected - || model.getStatus() == FileDownloadStatus.error - || (model.getStatus() == FileDownloadStatus.pending && model - .getSoFar() > 0) - ) { - // Ensure can be covered by RESUME FROM BREAKPOINT. - model.setStatus(FileDownloadStatus.paused); - } - final String targetFilePath = model.getTargetFilePath(); - if (targetFilePath == null) { - // no target file path, can't used to resume from breakpoint. - isInvalid = true; - break; - } - - final File targetFile = new File(targetFilePath); - // consider check in new thread, but SQLite lock | file lock aways effect, so - // sync - if (model.getStatus() == FileDownloadStatus.paused - && FileDownloadUtils.isBreakpointAvailable(model.getId(), model, - model.getPath(), null)) { - // can be reused in the old mechanism(no-temp-file). - - final File tempFile = new File(model.getTempFilePath()); - - if (!tempFile.exists() && targetFile.exists()) { - final boolean successRename = targetFile.renameTo(tempFile); - if (FileDownloadLog.NEED_LOG) { - FileDownloadLog.d(FileDownloadDatabase.class, - "resume from the old no-temp-file architecture " - + "[%B], [%s]->[%s]", - successRename, targetFile.getPath(), tempFile.getPath()); - - } - } - } - - /** - * Remove {@code model} from DB if it can't used for judging whether the - * old-downloaded file is valid for reused & it can't used for resuming from - * BREAKPOINT, In other words, {@code model} is no use anymore for - * FileDownloader. - */ - if (model.getStatus() == FileDownloadStatus.pending && model.getSoFar() <= 0) { - // This model is redundant. - isInvalid = true; - break; - } - - if (!FileDownloadUtils.isBreakpointAvailable(model.getId(), model)) { - // It can't used to resuming from breakpoint. - isInvalid = true; - break; - } - - if (targetFile.exists()) { - // It has already completed downloading. - isInvalid = true; - break; - } - - } while (false); - - - if (isInvalid) { - iterator.remove(); - maintainer.onRemovedInvalidData(model); - removedDataCount++; - } else { - final int oldId = model.getId(); - final int newId = idGenerator.transOldId(oldId, model.getUrl(), model.getPath(), - model.isPathAsDirectory()); - if (newId != oldId) { - if (FileDownloadLog.NEED_LOG) { - FileDownloadLog.d(FileDownloadDatabase.class, - "the id is changed on restoring from db:" - + " old[%d] -> new[%d]", - oldId, newId); - } - model.setId(newId); - maintainer.changeFileDownloadModelId(oldId, model); - resetIdCount++; - } - - maintainer.onRefreshedValidData(model); - refreshDataCount++; - } - } - - } finally { - FileDownloadUtils.markConverted(FileDownloadHelper.getAppContext()); - maintainer.onFinishMaintain(); - // 566 data consumes about 140ms - // update by rth: different devices have very large disparity, such as, - // in my HuaWei Android 8.0, 67 data consumes about 355ms. - // so, it's better do this action in background thread. - if (FileDownloadLog.NEED_LOG) { - FileDownloadLog.d(FileDownloadDatabase.class, - "refreshed data count: %d , delete data count: %d, reset id count:" - + " %d. consume %d", - refreshDataCount, removedDataCount, resetIdCount, - System.currentTimeMillis() - startTimestamp); - } - } - } }