Skip to content

Commit

Permalink
Added support for using multiple stores simultaneously in the `FMTCTi…
Browse files Browse the repository at this point in the history 
…leProvider`

Replaced `FMTCTileProviderSettings.maxStoreLength` with `maxLength` on each store individually
Exposed direct constructor for `FMTCTileProvider`
Refactored and exposed tile provider logic into seperate `getBytes` method
Added more `CacheBehavior` options
Added toggle for hit/miss stat recording
Added tests
Improved documentation
  • Loading branch information
@JaffaKetchup
JaffaKetchup committed May 24, 2024
1 parent dfee86c commit 15a82c5
Show file tree
Hide file tree
Showing 21 changed files with 776 additions and 296 deletions.
4 changes: 2 additions & 2 deletions example/lib/screens/main/pages/map/components/map_view.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ class MapView extends StatelessWidget {
metadata.data!['validDuration']!,
),
),
maxStoreLength:
int.parse(metadata.data!['maxLength']!),
/*maxStoreLength:
int.parse(metadata.data!['maxLength']!),*/
),
)
: NetworkTileProvider(),
Expand Down
9 changes: 5 additions & 4 deletions lib/flutter_map_tile_caching.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ import 'dart:collection';
import 'dart:io';
import 'dart:isolate';
import 'dart:math' as math;
import 'dart:math';
import 'dart:typed_data';
import 'dart:ui';

import 'package:async/async.dart';
import 'package:collection/collection.dart';
Expand All @@ -36,7 +37,6 @@ import 'src/bulk_download/tile_loops/shared.dart';
import 'src/misc/int_extremes.dart';
import 'src/misc/obscure_query_params.dart';
import 'src/providers/browsing_errors.dart';
import 'src/providers/image_provider.dart';

export 'src/backend/export_external.dart';
export 'src/providers/browsing_errors.dart';
Expand All @@ -46,6 +46,7 @@ part 'src/bulk_download/manager.dart';
part 'src/bulk_download/thread.dart';
part 'src/bulk_download/tile_event.dart';
part 'src/misc/deprecations.dart';
part 'src/providers/image_provider.dart';
part 'src/providers/tile_provider.dart';
part 'src/providers/tile_provider_settings.dart';
part 'src/regions/base_region.dart';
Expand All @@ -55,12 +56,12 @@ part 'src/regions/downloadable_region.dart';
part 'src/regions/line.dart';
part 'src/regions/recovered_region.dart';
part 'src/regions/rectangle.dart';
part 'src/root/root.dart';
part 'src/root/external.dart';
part 'src/root/recovery.dart';
part 'src/root/root.dart';
part 'src/root/statistics.dart';
part 'src/store/store.dart';
part 'src/store/download.dart';
part 'src/store/manage.dart';
part 'src/store/metadata.dart';
part 'src/store/statistics.dart';
part 'src/store/store.dart';
4 changes: 3 additions & 1 deletion lib/src/backend/impls/objectbox/backend/backend.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// A full license can be found at .\LICENSE

import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io';
import 'dart:isolate';
Expand Down Expand Up @@ -38,7 +39,8 @@ final class FMTCObjectBoxBackend implements FMTCBackend {
/// ---
///
/// [maxDatabaseSize] is the maximum size the database file can grow
/// to, in KB. Exceeding it throws [DbFullException]. Defaults to 10 GB.
/// to, in KB. Exceeding it throws [DbFullException] on write operations.
/// Defaults to 10 GB (10000000 KB).
///
/// [macosApplicationGroup] should be set when creating a sandboxed macOS app,
/// specify the application group (of less than 20 chars). See
Expand Down
77 changes: 56 additions & 21 deletions lib/src/backend/impls/objectbox/backend/internal.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {
// `removeOldestTilesAboveLimit` tracking & debouncing

Timer? _rotalDebouncer;
String? _rotalStore;
Completer<int>? _rotalResultCompleter;
int? _rotalStoresHash;
Completer<Map<String, int>>? _rotalResultCompleter;

// Define communicators

Expand Down Expand Up @@ -268,7 +268,7 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {
_workerResStreamed.clear();
_rotalDebouncer?.cancel();
_rotalDebouncer = null;
_rotalStore = null;
_rotalStoresHash = null;
_rotalResultCompleter?.completeError(RootUnavailable());
_rotalResultCompleter = null;

Expand All @@ -294,6 +294,25 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {
Future<List<String>> listStores() async =>
(await _sendCmdOneShot(type: _CmdType.listStores))!['stores'];

@override
Future<int?> storeGetMaxLength({
required String storeName,
}) async =>
(await _sendCmdOneShot(
type: _CmdType.storeGetMaxLength,
args: {'storeName': storeName},
))!['maxLength'];

@override
Future<void> storeSetMaxLength({
required String storeName,
required int? newMaxLength,
}) =>
_sendCmdOneShot(
type: _CmdType.storeSetMaxLength,
args: {'storeName': storeName, 'newMaxLength': newMaxLength},
);

@override
Future<bool> storeExists({
required String storeName,
Expand All @@ -306,10 +325,11 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {
@override
Future<void> createStore({
required String storeName,
required int? maxLength,
}) =>
_sendCmdOneShot(
type: _CmdType.createStore,
args: {'storeName': storeName},
args: {'storeName': storeName, 'maxLength': maxLength},
);

@override
Expand Down Expand Up @@ -353,25 +373,41 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {
))!['stats'];

@override
Future<bool> tileExistsInStore({
required String storeName,
Future<bool> tileExists({
required String url,
List<String>? storeNames,
}) async =>
(await _sendCmdOneShot(
type: _CmdType.tileExistsInStore,
args: {'storeName': storeName, 'url': url},
type: _CmdType.tileExists,
args: {'url': url, 'storeNames': storeNames},
))!['exists'];

@override
Future<ObjectBoxTile?> readTile({
required String url,
String? storeName,
List<String>? storeNames,
}) async =>
(await _sendCmdOneShot(
type: _CmdType.readTile,
args: {'url': url, 'storeName': storeName},
args: {'url': url, 'storeNames': storeNames},
))!['tile'];

@override
Future<({BackendTile? tile, List<String> storeNames})>
readTileWithStoreNames({
required String url,
List<String>? storeNames,
}) async {
final res = (await _sendCmdOneShot(
type: _CmdType.readTile,
args: {'url': url, 'storeNames': storeNames},
))!;
return (
tile: res['tile'] as BackendTile?,
storeNames: res['stores'] as List<String>,
);
}

@override
Future<ObjectBoxTile?> readLatestTile({
required String storeName,
Expand All @@ -383,13 +419,13 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {

@override
Future<void> writeTile({
required String storeName,
required String url,
required Uint8List bytes,
required List<String> storeNames,
}) =>
_sendCmdOneShot(
type: _CmdType.writeTile,
args: {'storeName': storeName, 'url': url, 'bytes': bytes},
args: {'storeNames': storeNames, 'url': url, 'bytes': bytes},
);

@override
Expand All @@ -404,35 +440,34 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal {

@override
Future<void> registerHitOrMiss({
required String storeName,
required List<String>? storeNames,
required bool hit,
}) =>
_sendCmdOneShot(
type: _CmdType.registerHitOrMiss,
args: {'storeName': storeName, 'hit': hit},
args: {'storeNames': storeNames, 'hit': hit},
);

@override
Future<int> removeOldestTilesAboveLimit({
required String storeName,
required int tilesLimit,
Future<Map<String, int>> removeOldestTilesAboveLimit({
required List<String> storeNames,
}) async {
// By sharing a single completer, all invocations of this method during the
// debounce period will return the same result at the same time
if (_rotalResultCompleter?.isCompleted ?? true) {
_rotalResultCompleter = Completer<int>();
_rotalResultCompleter = Completer<Map<String, int>>();
}
void sendCmdAndComplete() => _rotalResultCompleter!.complete(
_sendCmdOneShot(
type: _CmdType.removeOldestTilesAboveLimit,
args: {'storeName': storeName, 'tilesLimit': tilesLimit},
args: {'storeNames': storeNames},
).then((v) => v!['numOrphans']),
);

// If the store has changed, failing to reset the batch/queue will mean
// tiles are removed from the wrong store
if (_rotalStore != storeName) {
_rotalStore = storeName;
if (_rotalStoresHash != storeNames.hashCode) {
_rotalStoresHash = storeNames.hashCode;
if (_rotalDebouncer?.isActive ?? false) {
_rotalDebouncer!.cancel();
sendCmdAndComplete();
Expand Down
51 changes: 34 additions & 17 deletions lib/src/backend/impls/objectbox/backend/internal_workers/shared.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,34 +5,35 @@ part of '../backend.dart';

void _sharedWriteSingleTile({
required Store root,
required String storeName,
required List<String> storeNames,
required String url,
required Uint8List bytes,
}) {
final tiles = root.box<ObjectBoxTile>();
final stores = root.box<ObjectBoxStore>();
final storesBox = root.box<ObjectBoxStore>();
final rootBox = root.box<ObjectBoxRoot>();

final tilesQuery = tiles.query(ObjectBoxTile_.url.equals(url)).build();
final storeQuery =
stores.query(ObjectBoxStore_.name.equals(storeName)).build();
storesBox.query(ObjectBoxStore_.name.oneOf(storeNames)).build();

final storesToUpdate = <String, ObjectBoxStore>{};
// If tile exists in this store, just update size, otherwise
// length and size
// Also update size of all related stores
bool didContainAlready = false;

root.runInTransaction(
TxMode.write,
() {
final existingTile = tilesQuery.findUnique();
final store = storeQuery.findUnique() ??
(throw StoreNotExists(storeName: storeName));
final stores = storeQuery.find(); // Assumed not empty

if (existingTile != null) {
// If tile exists in this store, just update size, otherwise
// length and size
// Also update size of all related stores
final didContainAlready = <String>{};

for (final relatedStore in existingTile.stores) {
if (relatedStore.name == storeName) didContainAlready = true;
didContainAlready
.addAll(storeNames.where((s) => s == relatedStore.name));

storesToUpdate[relatedStore.name] =
(storesToUpdate[relatedStore.name] ?? relatedStore)
Expand All @@ -45,29 +46,45 @@ void _sharedWriteSingleTile({
..size += -existingTile.bytes.lengthInBytes + bytes.lengthInBytes,
mode: PutMode.update,
);

storesToUpdate.addEntries(
stores.whereNot((s) => didContainAlready.contains(s.name)).map(
(s) => MapEntry(
s.name,
s
..length += 1
..size += bytes.lengthInBytes,
),
),
);
} else {
rootBox.put(
rootBox.get(1)!
..length += 1
..size += bytes.lengthInBytes,
mode: PutMode.update,
);
}

if (!didContainAlready || existingTile == null) {
storesToUpdate[storeName] = store
..length += 1
..size += bytes.lengthInBytes;
storesToUpdate.addEntries(
stores.map(
(s) => MapEntry(
s.name,
s
..length += 1
..size += bytes.lengthInBytes,
),
),
);
}

tiles.put(
ObjectBoxTile(
url: url,
lastModified: DateTime.timestamp(),
bytes: bytes,
)..stores.addAll({store, ...?existingTile?.stores}),
)..stores.addAll({...stores, ...?existingTile?.stores}),
);
stores.putMany(storesToUpdate.values.toList(), mode: PutMode.update);
storesBox.putMany(storesToUpdate.values.toList(), mode: PutMode.update);
},
);

Expand Down
4 changes: 3 additions & 1 deletion lib/src/backend/impls/objectbox/backend/internal_workers/standard/cmd_type.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ enum _CmdType {
rootSize,
rootLength,
listStores,
storeGetMaxLength,
storeSetMaxLength,
storeExists,
createStore,
resetStore,
renameStore,
deleteStore,
getStoreStats,
tileExistsInStore,
tileExists,
readTile,
readLatestTile,
writeTile,
Expand Down
Loading

0 comments on commit 15a82c5

Please sign in to comment.