diff --git a/commitlint.yaml b/commitlint.yaml index d25d7a104ee..1ab287dd96f 100644 --- a/commitlint.yaml +++ b/commitlint.yaml @@ -25,6 +25,7 @@ rules: - neon_http_client - neon_lints - neon_lints_test + - neon_storage - news_app - nextcloud - nextcloud_test diff --git a/packages/neon_framework/packages/neon_storage/LICENSE b/packages/neon_framework/packages/neon_storage/LICENSE new file mode 120000 index 00000000000..f0b83dad961 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/LICENSE @@ -0,0 +1 @@ +../../../../assets/AGPL-3.0.txt \ No newline at end of file diff --git a/packages/neon_framework/packages/neon_storage/analysis_options.yaml b/packages/neon_framework/packages/neon_storage/analysis_options.yaml new file mode 100644 index 00000000000..bff1b129f3c --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/analysis_options.yaml @@ -0,0 +1,5 @@ +include: package:neon_lints/dart.yaml + +custom_lint: + rules: + - avoid_exports: false diff --git a/packages/neon_framework/packages/neon_storage/lib/neon_sqlite.dart b/packages/neon_framework/packages/neon_storage/lib/neon_sqlite.dart new file mode 100644 index 00000000000..de049019321 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/neon_sqlite.dart @@ -0,0 +1,4 @@ +/// SQLite wrappers for working with multiple tables independently. +library; + +export 'src/sqlite/sqlite.dart' hide setupDatabase; diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_browser_database_factory.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_browser_database_factory.dart new file mode 100644 index 00000000000..c778f1833fc --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_browser_database_factory.dart @@ -0,0 +1,7 @@ +import 'package:sqflite_common/sqflite.dart'; +import 'package:sqflite_common_ffi_web/sqflite_ffi_web.dart'; + +/// Initializes the sqlite database factory with the [databaseFactoryFfiWeb]. +void setupDatabase() { + databaseFactory = databaseFactoryFfiWeb; +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_io_database_factory.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_io_database_factory.dart new file mode 100644 index 00000000000..f09fefb2938 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/_io_database_factory.dart @@ -0,0 +1,8 @@ +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +/// Initializes the sqlite database factory with the [databaseFactoryFfi]. + +void setupDatabase() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_factory.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_factory.dart new file mode 100644 index 00000000000..959326c6e69 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_factory.dart @@ -0,0 +1,7 @@ +import 'package:meta/meta.dart'; + +/// Initializes the sqlite database factory. +@internal +void setupDatabase() { + throw UnsupportedError('Cannot create a database factory without dart:js_interop or dart:io.'); +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_path_utils.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_path_utils.dart new file mode 100644 index 00000000000..b7e128d4ccc --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/database_path_utils.dart @@ -0,0 +1,19 @@ +import 'package:path/path.dart' as p; + +/// The default database file extension. +const String databaseExtension = '.db'; + +/// Builds the database storage path from a directory and file name. +String buildDatabasePath(String? directory, String name) { + if (p.basenameWithoutExtension(name) != name) { + throw ArgumentError.value(name, 'name', 'MUST NOT contain any directory or file extension.'); + } + + final nameWithExtension = p.setExtension(name, databaseExtension); + + if (directory == null) { + return nameWithExtension; + } + + return p.join(directory, nameWithExtension); +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/multi_table_database.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/multi_table_database.dart new file mode 100644 index 00000000000..4d0e1c8993b --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/multi_table_database.dart @@ -0,0 +1,170 @@ +import 'dart:async'; + +import 'package:built_collection/built_collection.dart'; +import 'package:meta/meta.dart'; +import 'package:neon_storage/src/sqlite/sqlite.dart'; +import 'package:sqflite_common/sqflite.dart'; + +/// A wrapper for the SQLite [Database] managing multiple tables. +abstract class MultiTableDatabase { + /// Creates a new database for the given [tables]. + /// + /// The provided tables must have distinct names or an `ArgumentError` is thrown. + MultiTableDatabase({ + required Iterable tables, + }) : _tables = BuiltList.from(tables) { + final names = _tables.map((t) => t.name).toSet(); + + if (names.length != _tables.length || names.contains(_metaTable)) { + throw ArgumentError.value(_tables, 'tables', 'contains conflicting table names'); + } + } + + /// The full storage path of the database. + @protected + FutureOr get path; + + /// The name of the database without any file extension. + @protected + String get name; + + /// When `true` all open parameters are ignored and the database is opened as-is. + @protected + bool get readOnly => false; + + /// When `true` (the default), a single database instance is opened for a + /// given path. Subsequent calls to [init] with the same path will + /// return the same instance, and will discard all other parameters such as + /// callbacks for that invocation. + @protected + bool get singleInstance => true; + + static bool _sqfliteInitialized = false; + + late final _metaTable = '_${name}_meta'; + + final BuiltList
_tables; + + Database? _database; + + /// Throws a `StateError` if [init] has not been called or completed before. + @visibleForTesting + @protected + void assertInitialized() { + if (_database == null) { + throw StateError( + 'The database "$name" has not been set up. Make sure init() has been called before and completed.', + ); + } + } + + /// The database instance. + /// + /// Throws a `StateError` if [init] has not been called or completed before. + Database get database { + assertInitialized(); + + return _database!; + } + + /// Closes the database. + /// + /// A closed database must be initialized again by calling [init]. + @visibleForTesting + Future close() async { + await _database?.close(); + _database = null; + } + + /// Initializes the database and all tables. + /// + /// This must called and completed before accessing other methods. + Future init() async { + if (_database != null) { + return; + } + + if (!_sqfliteInitialized) { + setupDatabase(); + _sqfliteInitialized = true; + } + + final database = await openDatabase( + await path, + version: 1, + onCreate: _createMetaTable, + readOnly: readOnly, + singleInstance: singleInstance, + ); + + await _createTables(database); + + for (final table in _tables) { + table.controller = this; + } + await Future.wait( + _tables.map((t) => t.onOpen()), + ); + + _database = database; + } + + Future _createMetaTable(Database db, int version) async { + await db.execute(''' +CREATE TABLE IF NOT EXISTS "$_metaTable" ( + "name" TEXT NOT NULL, + "version" INTEGER NOT NULL, + PRIMARY KEY("name") +); +'''); + } + + Future _createTables(Database db) async { + final rows = await db.query(_metaTable); + + final versions = {}; + for (final row in rows) { + final name = row['name']! as String; + final version = row['version']! as int; + + versions[name] = version; + } + + final batch = db.batch(); + + for (final table in _tables) { + final oldVersion = versions[table.name]; + final newVersion = table.version; + + if (oldVersion == null) { + table.onCreate(batch, newVersion); + batch.insert( + _metaTable, + { + 'name': table.name, + 'version': newVersion, + }, + ); + continue; + } else if (oldVersion == newVersion) { + continue; + } else if (oldVersion > newVersion) { + table.onDowngrade(batch, oldVersion, newVersion); + } else if (oldVersion < newVersion) { + table.onUpgrade(batch, oldVersion, newVersion); + } + + batch.update( + _metaTable, + { + 'name': table.name, + 'version': newVersion, + }, + where: 'name = ?', + whereArgs: [table.name], + ); + } + + await batch.commit(noResult: true); + } +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/sqlite.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/sqlite.dart new file mode 100644 index 00000000000..bce8c4a1341 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/sqlite.dart @@ -0,0 +1,7 @@ +export 'database_factory.dart' + if (dart.library.js_interop) '_browser_database_factory.dart' + if (dart.library.io) '_io_database_factory.dart'; + +export 'database_path_utils.dart'; +export 'multi_table_database.dart'; +export 'table.dart'; diff --git a/packages/neon_framework/packages/neon_storage/lib/src/sqlite/table.dart b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/table.dart new file mode 100644 index 00000000000..a93a9398f1f --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/sqlite/table.dart @@ -0,0 +1,42 @@ +import 'dart:async'; + +import 'package:neon_storage/src/sqlite/sqlite.dart'; +import 'package:sqflite_common/sqlite_api.dart'; + +/// A SQLite table abstraction that mimics the `openDatabase` function from the sqflite package. +/// +/// Depending on the current [version], [onCreate], [onUpgrade], and [onDowngrade] can +/// be called. These functions are mutually exclusive — only one of them can be +/// called depending on the context. +abstract mixin class Table { + /// The name of the table. + String get name; + + /// The value must be a 32-bit integer greater than `0`. + int get version; + + /// The owning multi table database. + late MultiTableDatabase controller; + + /// A callback for creating the required schema for the table. + /// + /// It is only called called if the table did not exist prior. + void onCreate(Batch db, int version) {} + + /// A callback for upgrading the schema of a table. + /// + /// It is only called called if the database already exists and [version] is + /// higher than the last database version. + void onUpgrade(Batch db, int oldVersion, int newVersion) {} + + /// A callback for downgrading the required schema for the table. + /// + /// It is only called called when [version] is lower than the last database + /// version. This is a rare case and should only come up if a newer version of + /// your code has created a database that is then interacted with by an older + /// version of your code. You should try to avoid this scenario. + void onDowngrade(Batch db, int oldVersion, int newVersion) {} + + /// A callback invoked after all the table versions are set. + Future onOpen() async {} +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/testing/test_table_database.dart b/packages/neon_framework/packages/neon_storage/lib/src/testing/test_table_database.dart new file mode 100644 index 00000000000..43062de7ab4 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/testing/test_table_database.dart @@ -0,0 +1,20 @@ +import 'package:meta/meta.dart'; +import 'package:neon_storage/neon_sqlite.dart'; +import 'package:sqflite_common/sqlite_api.dart'; + +/// A test implementation for testing a [Table]. +@visibleForTesting +final class TestTableDatabase extends MultiTableDatabase { + /// Creates a test database for the given [table]. + TestTableDatabase(Table table) + : _table = table, + super(tables: [table]); + + final Table _table; + + @override + String get name => '${_table.name}_database'; + + @override + String get path => inMemoryDatabasePath; +} diff --git a/packages/neon_framework/packages/neon_storage/lib/src/testing/testing.dart b/packages/neon_framework/packages/neon_storage/lib/src/testing/testing.dart new file mode 100644 index 00000000000..aeb6798c93c --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/src/testing/testing.dart @@ -0,0 +1 @@ +export 'test_table_database.dart'; diff --git a/packages/neon_framework/packages/neon_storage/lib/testing.dart b/packages/neon_framework/packages/neon_storage/lib/testing.dart new file mode 100644 index 00000000000..c94f72f5339 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/lib/testing.dart @@ -0,0 +1,7 @@ +/// This library contains testing helpers for the neon storage. +@visibleForTesting +library; + +import 'package:meta/meta.dart'; + +export 'src/testing/testing.dart'; diff --git a/packages/neon_framework/packages/neon_storage/pubspec.yaml b/packages/neon_framework/packages/neon_storage/pubspec.yaml new file mode 100644 index 00000000000..d3d4c7c3a2c --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/pubspec.yaml @@ -0,0 +1,34 @@ +name: neon_storage +description: Storage management and abstractions for the neon framework. +version: 0.1.0 +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + built_collection: ^5.0.0 + cookie_store: + git: + url: https://github.com/nextcloud/neon + path: packages/cookie_store + http: ^1.0.0 + logging: ^1.0.0 + meta: ^1.0.0 + nextcloud: ^7.0.0 + path: ^1.9.0 + sqflite_common: ^2.0.0 + sqflite_common_ffi: ^2.3.3 + sqflite_common_ffi_web: ^0.4.3+1 + +dev_dependencies: + cookie_store_conformance_tests: + git: + url: https://github.com/nextcloud/neon + path: packages/cookie_store_conformance_tests + mocktail: ^1.0.4 + neon_lints: + git: + url: https://github.com/nextcloud/neon + path: packages/neon_lints + test: ^1.25.8 diff --git a/packages/neon_framework/packages/neon_storage/pubspec_overrides.yaml b/packages/neon_framework/packages/neon_storage/pubspec_overrides.yaml new file mode 100644 index 00000000000..d4b2d2ff99a --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/pubspec_overrides.yaml @@ -0,0 +1,12 @@ +# melos_managed_dependency_overrides: neon_lints,cookie_store,cookie_store_conformance_tests,dynamite_runtime,nextcloud +dependency_overrides: + cookie_store: + path: ../../../cookie_store + cookie_store_conformance_tests: + path: ../../../cookie_store/packages/cookie_store_conformance_tests + dynamite_runtime: + path: ../../../dynamite/packages/dynamite_runtime + neon_lints: + path: ../../../neon_lints + nextcloud: + path: ../../../nextcloud diff --git a/packages/neon_framework/packages/neon_storage/test/sqlite/database_path_utils_test.dart b/packages/neon_framework/packages/neon_storage/test/sqlite/database_path_utils_test.dart new file mode 100644 index 00000000000..deb20cfe4ac --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/test/sqlite/database_path_utils_test.dart @@ -0,0 +1,26 @@ +import 'package:neon_storage/neon_sqlite.dart'; +import 'package:test/test.dart'; + +void main() { + test('buildDatabasePath', () { + expect( + () => buildDatabasePath(null, 'database.db'), + throwsArgumentError, + ); + + expect( + () => buildDatabasePath(null, 'tmp/database.db'), + throwsArgumentError, + ); + + expect( + buildDatabasePath(null, 'database'), + equals('database.db'), + ); + + expect( + buildDatabasePath('/tmp', 'database'), + equals('/tmp/database.db'), + ); + }); +} diff --git a/packages/neon_framework/packages/neon_storage/test/sqlite/multi_table_database_test.dart b/packages/neon_framework/packages/neon_storage/test/sqlite/multi_table_database_test.dart new file mode 100644 index 00000000000..e8b1c2d5cdf --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/test/sqlite/multi_table_database_test.dart @@ -0,0 +1,184 @@ +import 'dart:async'; + +import 'package:mocktail/mocktail.dart'; +import 'package:neon_storage/neon_sqlite.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:test/test.dart'; + +abstract class _DatabaseController { + String get name; + + FutureOr get path; + + Iterable
get tables; +} + +class _DatabaseControllerMock extends Mock implements _DatabaseController {} + +class _TestDatabase extends MultiTableDatabase { + _TestDatabase(this.controller) : super(tables: controller.tables); + + final _DatabaseController controller; + + @override + String get name => controller.name; + + @override + FutureOr get path => controller.path; +} + +class _FakeMultiTableDatabase extends Fake implements MultiTableDatabase {} + +class _TableMock extends Mock implements Table {} + +class _FakeBatch extends Fake implements Batch {} + +void main() { + group(MultiTableDatabase, () { + late _DatabaseController controller; + late MultiTableDatabase database; + + setUpAll(() { + registerFallbackValue(_FakeBatch()); + registerFallbackValue(_FakeMultiTableDatabase()); + }); + + setUp(() { + controller = _DatabaseControllerMock(); + when(() => controller.tables).thenReturn([]); + when(() => controller.path).thenReturn(inMemoryDatabasePath); + when(() => controller.name).thenReturn('database'); + + database = _TestDatabase(controller); + resetMocktailState(); + }); + + tearDown(() async { + await database.close(); + }); + + test('init initializes the database', () async { + await database.init(); + + expect( + database.database, + isA(), + ); + }); + + test('throws if not initialized', () async { + expect( + () => database.database, + throwsStateError, + ); + }); + + test('returns the same instance once initialized', () async { + await database.init(); + + expect( + identical(database.database, database.database), + isTrue, + ); + }); + + group('conflicting table names', () { + setUp(() { + controller = _DatabaseControllerMock(); + when(() => controller.name).thenReturn('database'); + }); + + test('throws an ArgumentError for duplicate table names', () { + final table = _TableMock(); + when(() => table.name).thenReturn('cache_table'); + + final table2 = _TableMock(); + when(() => table2.name).thenReturn('cache_table'); + + when(() => controller.tables).thenReturn([table, table2]); + + expect( + () => _TestDatabase(controller), + throwsArgumentError, + ); + }); + + test('throws a TableNameError for conflicting table names', () { + final table = _TableMock(); + when(() => table.name).thenReturn('_database_meta'); + + when(() => controller.tables).thenReturn([table]); + + expect( + () => _TestDatabase(controller), + throwsArgumentError, + ); + }); + }); + + group('table versioning', () { + late Table table; + setUp(() async { + table = _TableMock(); + when(() => table.name).thenReturn('table'); + when(() => table.version).thenReturn(2); + when(() => table.onOpen()).thenAnswer((_) async {}); + + controller = _DatabaseControllerMock(); + when(() => controller.path).thenReturn(inMemoryDatabasePath); + when(() => controller.name).thenReturn('database'); + when(() => controller.tables).thenReturn([table]); + + database = _TestDatabase(controller); + + await database.init(); + clearInteractions(table); + }); + + tearDown(() { + verify(table.onOpen).called(1); + + verify(() => table.name); + verify(() => table.version); + verify(() => table.controller = any()).called(1); + + verifyNoMoreInteractions(table); + }); + + test('calls onCreate when table is not present', () async { + when(() => table.name).thenReturn('table2'); + when(() => table.version).thenReturn(1); + + database = _TestDatabase(controller); + await database.init(); + + verify(() => table.onCreate(any(), 1)).called(1); + }); + + test('does not call onCreate when table version did not change', () async { + database = _TestDatabase(controller); + await database.init(); + + verifyNever(() => table.onCreate(any(), any())); + }); + + test('calls onUpgrade when table version is higher', () async { + when(() => table.version).thenReturn(3); + + database = _TestDatabase(controller); + await database.init(); + + verify(() => table.onUpgrade(any(), 2, 3)).called(1); + }); + + test('calls onDowngrade when table version is lower', () async { + when(() => table.version).thenReturn(1); + + database = _TestDatabase(controller); + await database.init(); + + verify(() => table.onDowngrade(any(), 2, 1)).called(1); + }); + }); + }); +} diff --git a/packages/neon_framework/packages/neon_storage/test/sqlite/table_test.dart b/packages/neon_framework/packages/neon_storage/test/sqlite/table_test.dart new file mode 100644 index 00000000000..a19b0ce9047 --- /dev/null +++ b/packages/neon_framework/packages/neon_storage/test/sqlite/table_test.dart @@ -0,0 +1,18 @@ +import 'package:neon_storage/neon_sqlite.dart'; +import 'package:test/test.dart'; + +class _TestTable extends Table { + @override + String get name => throw UnimplementedError(); + + @override + int get version => throw UnimplementedError(); +} + +void main() { + group('Table', () { + test('can be implemented', () { + expect(_TestTable(), isNotNull); + }); + }); +}