Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(nextcloud): Implement chunked upload #2308

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions packages/neon/neon_files/lib/src/blocs/browser.dart
Original file line number Diff line number Diff line change
Expand Up @@ -91,19 +91,19 @@ class _FilesBrowserBloc extends InteractiveBloc implements FilesBrowserBloc {
await RequestManager.instance.wrap(
account: account,
subject: files,
getRequest: () => account.client.webdav.propfind_Request(
uri.value,
prop: const WebDavPropWithoutValues.fromBools(
davGetcontenttype: true,
davGetetag: true,
davGetlastmodified: true,
ncHasPreview: true,
ncMetadataBlurhash: true,
ocSize: true,
ocFavorite: true,
),
depth: WebDavDepth.one,
),
getRequest: () => account.client.webdav().propfind_Request(
uri.value,
prop: const WebDavPropWithoutValues.fromBools(
davGetcontenttype: true,
davGetetag: true,
davGetlastmodified: true,
ncHasPreview: true,
ncMetadataBlurhash: true,
ocSize: true,
ocFavorite: true,
),
depth: WebDavDepth.one,
),
converter: const WebDavResponseConverter(),
unwrap: (response) => BuiltList<WebDavFile>.build((b) {
for (final file in response.toWebDavFiles()) {
Expand Down Expand Up @@ -144,6 +144,6 @@ class _FilesBrowserBloc extends InteractiveBloc implements FilesBrowserBloc {

@override
Future<void> createFolder(PathUri uri) async {
await wrapAction(() async => account.client.webdav.mkcol(uri));
await wrapAction(() async => account.client.webdav().mkcol(uri));
}
}
30 changes: 15 additions & 15 deletions packages/neon/neon_files/lib/src/blocs/files.dart
Original file line number Diff line number Diff line change
Expand Up @@ -164,46 +164,46 @@ class _FilesBloc extends InteractiveBloc implements FilesBloc {

@override
Future<void> delete(PathUri uri) async {
await wrapAction(() async => account.client.webdav.delete(uri));
await wrapAction(() async => account.client.webdav().delete(uri));
}

@override
Future<void> rename(PathUri uri, String name) async {
await wrapAction(
() async => account.client.webdav.move(
uri,
uri.rename(name),
),
() async => account.client.webdav().move(
uri,
uri.rename(name),
),
);
}

@override
Future<void> move(PathUri uri, PathUri destination) async {
await wrapAction(() async => account.client.webdav.move(uri, destination));
await wrapAction(() async => account.client.webdav().move(uri, destination));
}

@override
Future<void> copy(PathUri uri, PathUri destination) async {
await wrapAction(() async => account.client.webdav.copy(uri, destination));
await wrapAction(() async => account.client.webdav().copy(uri, destination));
}

@override
Future<void> addFavorite(PathUri uri) async {
await wrapAction(
() async => account.client.webdav.proppatch(
uri,
set: const WebDavProp(ocFavorite: true),
),
() async => account.client.webdav().proppatch(
uri,
set: const WebDavProp(ocFavorite: true),
),
);
}

@override
Future<void> removeFavorite(PathUri uri) async {
await wrapAction(
() async => account.client.webdav.proppatch(
uri,
set: const WebDavProp(ocFavorite: false),
),
() async => account.client.webdav().proppatch(
uri,
set: const WebDavProp(ocFavorite: false),
),
);
}

Expand Down
46 changes: 23 additions & 23 deletions packages/neon/neon_files/lib/src/utils/task.dart
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ class FilesDownloadTaskIO extends FilesTaskIO implements FilesDownloadTask {
});

Future<void> execute(NextcloudClient client) async {
await client.webdav.getFile(
uri,
file,
onProgress: progressController.add,
);
await client.webdav().getFile(
uri,
file,
onProgress: progressController.add,
);
await progressController.close();
}
}
Expand All @@ -91,13 +91,13 @@ class FilesUploadTaskIO extends FilesTaskIO implements FilesUploadTask {
late tz.TZDateTime lastModified = tz.TZDateTime.from(_stat.modified, tz.UTC);

Future<void> execute(NextcloudClient client) async {
await client.webdav.putFile(
file,
_stat,
uri,
lastModified: _stat.modified,
onProgress: progressController.add,
);
await client.webdav().putFile(
file,
_stat,
uri,
lastModified: _stat.modified,
onProgress: progressController.add,
);
await progressController.close();
}
}
Expand All @@ -108,10 +108,10 @@ class FilesDownloadTaskMemory extends FilesTaskMemory implements FilesDownloadTa
});

Future<void> execute(NextcloudClient client) async {
final stream = client.webdav.getStream(
uri,
onProgress: progressController.add,
);
final stream = client.webdav().getStream(
uri,
onProgress: progressController.add,
);
await stream.pipe(_stream);
await progressController.close();
}
Expand All @@ -134,13 +134,13 @@ class FilesUploadTaskMemory extends FilesTaskMemory implements FilesUploadTask {
final tz.TZDateTime? lastModified;

Future<void> execute(NextcloudClient client) async {
await client.webdav.putStream(
_stream.stream,
uri,
lastModified: lastModified,
contentLength: size,
onProgress: progressController.add,
);
await client.webdav().putStream(
_stream.stream,
uri,
lastModified: lastModified,
contentLength: size,
onProgress: progressController.add,
);
await progressController.close();
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import 'dart:convert';

import 'package:dynamite_runtime/http_client.dart';
import 'package:http/http.dart' as http;
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'package:neon_http_client/src/interceptors/http_interceptor.dart';
import 'package:nextcloud/nextcloud.dart';
import 'package:nextcloud/webdav.dart';
import 'package:universal_io/io.dart';

/// A HttpInterceptor that works around a Nextcloud CSRF bug when cookies are sent.
Expand Down Expand Up @@ -44,7 +43,7 @@ final class CSRFInterceptor implements HttpInterceptor {

@override
bool shouldInterceptRequest(http.BaseRequest request) {
if (request.url.host != _baseURL.host || !request.url.path.startsWith('${_baseURL.path}$webdavBase')) {
if (request.url.host != _baseURL.host || !request.url.path.startsWith('${_baseURL.path}/remote.php')) {
return false;
}

Expand Down
5 changes: 4 additions & 1 deletion packages/neon_http_client/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ dependencies:
git:
url: https://github.com/nextcloud/neon
path: packages/cookie_store
dynamite_runtime:
git:
url: https://github.com/nextcloud/neon
path: packages/dynamite/dynamite_runtime
http: ^1.0.0
logging: ^1.0.0
meta: ^1.0.0
nextcloud: ^6.1.0
universal_io: ^2.0.0

dev_dependencies:
Expand Down
107 changes: 107 additions & 0 deletions packages/nextcloud/lib/src/api/webdav/chunked_upload_client.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import 'dart:math';
import 'dart:typed_data';

import 'package:http/http.dart' as http;
import 'package:nextcloud/nextcloud.dart';
import 'package:nextcloud/src/api/webdav/webdav.dart';
import 'package:nextcloud/src/utils/utils.dart';
import 'package:uuid/uuid.dart';

const _uuid = Uuid();

typedef ChunkedUpload = ({String token, PathUri destination});

class ChunkedUploadClient {
ChunkedUploadClient(
NextcloudClient rootClient,
String username,
) : _username = username,
_webDavClient = WebDavClient(
rootClient,
endpoint: 'remote.php/dav/uploads/$username',
);

final WebDavClient _webDavClient;
final String _username;

Future<ChunkedUpload> start(PathUri path) async {
final destination = PathUri.parse('/remote.php/dav/files/$_username/').join(path);
final token = _uuid.v7();

final streamedResponse = await _webDavClient.mkcol(
PathUri.parse(token),
headers: {
'Destination': destination.toString(),
},
);
if (streamedResponse.statusCode != 201) {
final response = await http.Response.fromStream(streamedResponse);

throw DynamiteStatusCodeException(response);
}

return (
token: token,
destination: destination,
);
}

Future<void> uploadChunk(
ChunkedUpload chunkedUpload,
int index,
Uint8List chunk,
int totalLength,
) async {
assert(
index >= 1 && index <= 10000,
'Index must be a number between 1 and 10000',
);
assert(
chunk.lengthInBytes >= 5 * pow(1024, 2) && chunk.lengthInBytes <= 5 * pow(1024, 3),
'Chunk size must be between 5MB and 5GB',
);

final streamedResponse = await _webDavClient.put(
chunk,
PathUri.parse('${chunkedUpload.token}/$index'),
headers: {
'Destination': chunkedUpload.destination.toString(),
'OC-Total-Length': totalLength.toString(),
},
);
if (streamedResponse.statusCode != 201) {
final response = await http.Response.fromStream(streamedResponse);

throw DynamiteStatusCodeException(response);
}
}

Future<void> assembleChunks(
ChunkedUpload chunkedUpload,
int totalLength, {
DateTime? lastModified,
DateTime? created,
}) async {
final request = http.Request(
'MOVE',
constructUri(
_webDavClient.rootClient.baseURL,
'remote.php/dav/uploads/$_username',
PathUri.parse('${chunkedUpload.token}/.file'),
),
);
request.headers.addAll({
'Destination': chunkedUpload.destination.toString(),
'OC-Total-Length': totalLength.toString(),
if (lastModified != null) 'X-OC-Mtime': lastModified.secondsSinceEpoch.toString(),
if (created != null) 'X-OC-CTime': created.secondsSinceEpoch.toString(),
});

final streamedResponse = await _webDavClient.csrfClient.send(request);
if (streamedResponse.statusCode != 201) {
final response = await http.Response.fromStream(streamedResponse);

throw DynamiteStatusCodeException(response);
}
}
}
18 changes: 14 additions & 4 deletions packages/nextcloud/lib/src/api/webdav/models/webdav_file.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,28 @@ import 'package:timezone/timezone.dart' as tz;
// ignore: public_member_api_docs
extension WebDavMultistatusFile on WebDavMultistatus {
/// Convert the [WebDavMultistatus] into a [WebDavFile] for easier handling
List<WebDavFile> toWebDavFiles() =>
responses.where((response) => response.href != null).map((response) => WebDavFile(response: response)).toList();
List<WebDavFile> toWebDavFiles({String endpoint = 'remote.php/webdav'}) => responses
.where((response) => response.href != null)
.map(
(response) => WebDavFile(
response: response,
endpoint: endpoint,
),
)
.toList();
}

/// WebDavFile class
class WebDavFile {
/// Creates a new WebDavFile object with the given path
WebDavFile({
required WebDavResponse response,
}) : _response = response;
required String endpoint,
}) : _response = response,
_endpoint = PathUri.parse(endpoint);

final WebDavResponse _response;
final PathUri _endpoint;

/// Get the props of the file
late final WebDavProp props = _response.propstats.singleWhere((propstat) => propstat.status.contains('200')).prop;
Expand All @@ -26,7 +36,7 @@ class WebDavFile {
return PathUri(
isAbsolute: false,
isDirectory: href.isDirectory,
pathSegments: href.pathSegments.sublist(webdavBase.pathSegments.length),
pathSegments: href.pathSegments.sublist(_endpoint.pathSegments.length),
);
}();

Expand Down
7 changes: 2 additions & 5 deletions packages/nextcloud/lib/src/api/webdav/utils/webdav_uri.dart
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import 'package:meta/meta.dart';
import 'package:nextcloud/src/api/webdav/webdav.dart';

/// Base path used on the server
final webdavBase = PathUri.parse('/remote.php/webdav');

/// Constructs the uri for a webdav request for a given server [baseURL] and file [path].
@internal
Uri constructUri(Uri baseURL, [PathUri? path]) {
final segments = baseURL.pathSegments.toList()..addAll(webdavBase.pathSegments);
Uri constructUri(Uri baseURL, String endpoint, [PathUri? path]) {
final segments = baseURL.pathSegments.toList()..addAll(PathUri.parse(endpoint).pathSegments);
if (path != null) {
segments.addAll(path.pathSegments);
}
Expand Down
Loading
Loading