From 8e503afd2bebe7a528ad3d82ffe2c7145af62ea9 Mon Sep 17 00:00:00 2001 From: provokateurin Date: Mon, 22 Jul 2024 23:42:21 +0200 Subject: [PATCH] feat(nextcloud): Implement chunked upload Signed-off-by: provokateurin --- .../src/api/webdav/chunked_upload_client.dart | 107 ++++++++++++++++++ .../lib/src/api/webdav/webdav_client.dart | 16 ++- packages/nextcloud/pubspec.yaml | 1 + .../fixtures/webdav/chunked_upload.regexp | 37 ++++++ packages/nextcloud/test/webdav_test.dart | 40 +++++++ 5 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 packages/nextcloud/lib/src/api/webdav/chunked_upload_client.dart create mode 100644 packages/nextcloud/test/fixtures/webdav/chunked_upload.regexp diff --git a/packages/nextcloud/lib/src/api/webdav/chunked_upload_client.dart b/packages/nextcloud/lib/src/api/webdav/chunked_upload_client.dart new file mode 100644 index 00000000000..c005f80e812 --- /dev/null +++ b/packages/nextcloud/lib/src/api/webdav/chunked_upload_client.dart @@ -0,0 +1,107 @@ +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:http/http.dart' as http; +import 'package:nextcloud/nextcloud.dart'; +import 'package:nextcloud/src/api/webdav/webdav.dart'; +import 'package:nextcloud/src/utils/utils.dart'; +import 'package:uuid/uuid.dart'; + +const _uuid = Uuid(); + +typedef ChunkedUpload = ({String token, PathUri destination}); + +class ChunkedUploadClient { + ChunkedUploadClient( + NextcloudClient rootClient, + String username, + ) : _username = username, + _webDavClient = WebDavClient( + rootClient, + endpoint: 'remote.php/dav/uploads/$username', + ); + + final WebDavClient _webDavClient; + final String _username; + + Future start(PathUri path) async { + final destination = PathUri.parse('/remote.php/dav/files/$_username/').join(path); + final token = _uuid.v7(); + + final streamedResponse = await _webDavClient.mkcol( + PathUri.parse(token), + headers: { + 'Destination': destination.toString(), + }, + ); + if (streamedResponse.statusCode != 201) { + final response = await http.Response.fromStream(streamedResponse); + + throw DynamiteStatusCodeException(response); + } + + return ( + token: token, + destination: destination, + ); + } + + Future uploadChunk( + ChunkedUpload chunkedUpload, + int index, + Uint8List chunk, + int totalLength, + ) async { + assert( + index >= 1 && index <= 10000, + 'Index must be a number between 1 and 10000', + ); + assert( + chunk.lengthInBytes >= 5 * pow(1024, 2) && chunk.lengthInBytes <= 5 * pow(1024, 3), + 'Chunk size must be between 5MB and 5GB', + ); + + final streamedResponse = await _webDavClient.put( + chunk, + PathUri.parse('${chunkedUpload.token}/$index'), + headers: { + 'Destination': chunkedUpload.destination.toString(), + 'OC-Total-Length': totalLength.toString(), + }, + ); + if (streamedResponse.statusCode != 201) { + final response = await http.Response.fromStream(streamedResponse); + + throw DynamiteStatusCodeException(response); + } + } + + Future assembleChunks( + ChunkedUpload chunkedUpload, + int totalLength, { + DateTime? lastModified, + DateTime? created, + }) async { + final request = http.Request( + 'MOVE', + constructUri( + _webDavClient.rootClient.baseURL, + 'remote.php/dav/uploads/$_username', + PathUri.parse('${chunkedUpload.token}/.file'), + ), + ); + request.headers.addAll({ + 'Destination': chunkedUpload.destination.toString(), + 'OC-Total-Length': totalLength.toString(), + if (lastModified != null) 'X-OC-Mtime': lastModified.secondsSinceEpoch.toString(), + if (created != null) 'X-OC-CTime': created.secondsSinceEpoch.toString(), + }); + + final streamedResponse = await _webDavClient.csrfClient.send(request); + if (streamedResponse.statusCode != 201) { + final response = await http.Response.fromStream(streamedResponse); + + throw DynamiteStatusCodeException(response); + } + } +} diff --git a/packages/nextcloud/lib/src/api/webdav/webdav_client.dart b/packages/nextcloud/lib/src/api/webdav/webdav_client.dart index 1089ca6a59a..b89eae8b584 100644 --- a/packages/nextcloud/lib/src/api/webdav/webdav_client.dart +++ b/packages/nextcloud/lib/src/api/webdav/webdav_client.dart @@ -70,8 +70,14 @@ class WebDavClient { /// See: /// * http://www.webdav.org/specs/rfc2518.html#METHOD_MKCOL for more information. /// * [mkcol_Request] for the request sent by this method. - Future mkcol(PathUri path) { + Future mkcol( + PathUri path, { + Map? headers, + }) { final request = mkcol_Request(path); + if (headers != null) { + request.headers.addAll(headers); + } return csrfClient.send(request); } @@ -132,6 +138,7 @@ class WebDavClient { PathUri path, { DateTime? lastModified, DateTime? created, + Map? headers, }) { final request = put_Request( localData, @@ -139,6 +146,9 @@ class WebDavClient { lastModified: lastModified, created: created, ); + if (headers != null) { + request.headers.addAll(headers); + } return csrfClient.send(request); } @@ -513,12 +523,16 @@ class WebDavClient { PathUri sourcePath, PathUri destinationPath, { bool overwrite = false, + Map? headers, }) { final request = move_Request( sourcePath, destinationPath, overwrite: overwrite, ); + if (headers != null) { + request.headers.addAll(headers); + } return csrfClient.send(request); } diff --git a/packages/nextcloud/pubspec.yaml b/packages/nextcloud/pubspec.yaml index 3b77771eaa3..fd2791d166e 100644 --- a/packages/nextcloud/pubspec.yaml +++ b/packages/nextcloud/pubspec.yaml @@ -26,6 +26,7 @@ dependencies: timezone: ^0.9.4 universal_io: ^2.0.0 uri: ^1.0.0 + uuid: ^4.0.0 version: ^3.0.0 xml: ^6.0.0 xml_annotation: ^2.1.0 diff --git a/packages/nextcloud/test/fixtures/webdav/chunked_upload.regexp b/packages/nextcloud/test/fixtures/webdav/chunked_upload.regexp new file mode 100644 index 00000000000..0181b1eb111 --- /dev/null +++ b/packages/nextcloud/test/fixtures/webdav/chunked_upload.regexp @@ -0,0 +1,37 @@ +MKCOL http://localhost/remote\.php/dav/uploads/user1/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} +authorization: Bearer mock +content-type: application/xml +destination: //:80/remote\.php/dav/files/user1/test/test\.bin +ocs-apirequest: true +requesttoken: token +PUT http://localhost/remote\.php/dav/uploads/user1/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/1 +authorization: Bearer mock +content-length: 10485760 +content-type: application/xml +destination: //:80/remote\.php/dav/files/user1/test/test\.bin +oc-total-length: 15728640 +ocs-apirequest: true +requesttoken: token +.+ +PUT http://localhost/remote\.php/dav/uploads/user1/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/2 +authorization: Bearer mock +content-length: 5242880 +content-type: application/xml +destination: //:80/remote\.php/dav/files/user1/test/test\.bin +oc-total-length: 15728640 +ocs-apirequest: true +requesttoken: token +.+ +MOVE http://localhost/remote\.php/dav/uploads/user1/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/\.file +destination: //:80/remote\.php/dav/files/user1/test/test\.bin +oc-total-length: 15728640 +ocs-apirequest: true +requesttoken: token +x-oc-ctime: 0 +x-oc-mtime: 31536000 +PROPFIND http://localhost/remote\.php/webdav/test/test\.bin +authorization: Bearer mock +content-type: application/xml +ocs-apirequest: true +requesttoken: token + \ No newline at end of file diff --git a/packages/nextcloud/test/webdav_test.dart b/packages/nextcloud/test/webdav_test.dart index ff9181ae13c..5cdb789bf81 100644 --- a/packages/nextcloud/test/webdav_test.dart +++ b/packages/nextcloud/test/webdav_test.dart @@ -4,11 +4,13 @@ import 'dart:typed_data'; import 'package:mocktail/mocktail.dart'; import 'package:nextcloud/nextcloud.dart'; +import 'package:nextcloud/src/api/webdav/chunked_upload_client.dart'; import 'package:nextcloud/src/api/webdav/webdav.dart'; import 'package:nextcloud/src/utils/date_time.dart'; import 'package:nextcloud/webdav.dart'; import 'package:nextcloud_test/nextcloud_test.dart'; import 'package:test/test.dart'; +import 'package:timezone/timezone.dart' as tz; import 'package:universal_io/io.dart'; class MockCallbackFunction extends Mock { @@ -215,6 +217,44 @@ void main() { await container.destroy(); }); + test('Chunked upload', () async { + final chunkedUploadClient = ChunkedUploadClient(client, 'user1'); + + final chunkedUpload = await chunkedUploadClient.start(PathUri.parse('test/test.bin')); + + final chunk1 = Uint8List.fromList(List.generate(10 * pow(1024, 2).toInt(), (i) => 1)); + final chunk2 = Uint8List.fromList(List.generate(5 * pow(1024, 2).toInt(), (i) => 2)); + final combined = Uint8List.fromList(chunk1 + chunk2); + + await chunkedUploadClient.uploadChunk(chunkedUpload, 1, chunk1, combined.lengthInBytes); + await chunkedUploadClient.uploadChunk(chunkedUpload, 2, chunk2, combined.lengthInBytes); + + final lastModified = tz.TZDateTime(tz.UTC, 1971); + final created = tz.TZDateTime(tz.UTC, 1970); + + await chunkedUploadClient.assembleChunks( + chunkedUpload, + combined.lengthInBytes, + lastModified: lastModified, + created: created, + ); + + final result = await client.webdav().propfind( + PathUri.parse('test/test.bin'), + prop: const WebDavPropWithoutValues.fromBools( + davGetlastmodified: true, + davGetcontentlength: true, + ncCreationTime: true, + ), + ); + final response = result.toWebDavFiles().single; + + expect(response.path, PathUri.parse('test/test.bin')); + expect(response.lastModified, lastModified); + expect(response.createdDate, created); + expect(response.size, combined.lengthInBytes); + }); + test('List directory', () async { final responses = (await client.webdav().propfind( PathUri.parse('test'),