Skip to content

Commit

Permalink
fix: upload big file
Browse files Browse the repository at this point in the history
  • Loading branch information
leancloud-bot committed Mar 9, 2022
1 parent 27f916c commit bd1d2e0
Show file tree
Hide file tree
Showing 7 changed files with 216 additions and 67 deletions.
4 changes: 2 additions & 2 deletions Common/Common/LCCore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,10 @@ public static void Initialize(string appId,
string server = null,
string masterKey = null) {
if (string.IsNullOrEmpty(appId)) {
throw new ArgumentException(nameof(appId));
throw new ArgumentNullException(nameof(appId));
}
if (string.IsNullOrEmpty(appKey)) {
throw new ArgumentException(nameof(appKey));
throw new ArgumentNullException(nameof(appKey));
}

AppId = appId;
Expand Down
57 changes: 44 additions & 13 deletions Storage/Storage.Test/FileTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ namespace Storage.Test {
public class FileTest : BaseTest {
static readonly string AvatarFilePath = "../../../../../assets/hello.png";
static readonly string APKFilePath = "../../../../../assets/test.apk";
static readonly string VideoFilePath = "../../../../../assets/video.mp4";

private LCFile avatar;
private LCFile video;

[Test]
[Order(0)]
public async Task SaveFromPath() {
avatar = new LCFile("avatar", AvatarFilePath);
LCFile avatar = new LCFile("avatar", AvatarFilePath);
await avatar.Save((count, total) => {
TestContext.WriteLine($"progress: {count}/{total}");
});
Expand All @@ -25,15 +26,27 @@ await avatar.Save((count, total) => {

[Test]
[Order(1)]
public async Task SaveBigFileFromPath() {
video = new LCFile("video", VideoFilePath);
await video.Save((count, total) => {
TestContext.WriteLine($"progress: {count}/{total}");
});
TestContext.WriteLine(video.ObjectId);
Assert.NotNull(video.ObjectId);
}

[Test]
[Order(2)]
public async Task QueryFile() {
LCQuery<LCFile> query = LCFile.GetQuery();
LCFile file = await query.Get(avatar.ObjectId);
LCFile file = await query.Get(video.ObjectId);
Assert.NotNull(file.Url);
TestContext.WriteLine(file.Url);
TestContext.WriteLine(file.GetThumbnailUrl(32, 32));
}

[Test]
[Order(3)]
public async Task SaveFromMemory() {
string text = "hello, world";
byte[] data = Encoding.UTF8.GetBytes(text);
Expand All @@ -46,6 +59,7 @@ public async Task SaveFromMemory() {
}

[Test]
[Order(4)]
public async Task SaveFromUrl() {
LCFile file = new LCFile("scene", new Uri("http://img95.699pic.com/photo/50015/9034.jpg_wh300.jpg"));
file.AddMetaData("size", 1024);
Expand All @@ -58,6 +72,7 @@ public async Task SaveFromUrl() {
}

[Test]
[Order(5)]
public async Task Qiniu() {
LCFile file = new LCFile("avatar", APKFilePath);
await file.Save();
Expand All @@ -66,6 +81,7 @@ public async Task Qiniu() {
}

[Test]
[Order(6)]
public async Task FileACL() {
LCUser user = await LCUser.LoginAnonymously();

Expand All @@ -87,15 +103,30 @@ public async Task FileACL() {
}
}

//[Test]
//public async Task AWS() {
// LCApplication.Initialize("UlCpyvLm8aMzQsW6KnP6W3Wt-MdYXbMMI", "PyCTYoNoxCVoKKg394PBeS4r");
// LCFile file = new LCFile("avatar", AvatarFilePath);
// await file.Save((count, total) => {
// TestContext.WriteLine($"progress: {count}/{total}");
// });
// TestContext.WriteLine(file.ObjectId);
// Assert.NotNull(file.ObjectId);
//}
[Test]
[Order(10)]
public async Task AWS() {
LCApplication.Initialize("HudJvWWmAuGMifwxByDVLmQi-MdYXbMMI", "YjoQr1X8wHoFIfsSGXzeJaAM",
"https://hudjvwwm.api.lncldglobal.com");
LCFile file = new LCFile("avatar", AvatarFilePath);
await file.Save((count, total) => {
TestContext.WriteLine($"progress: {count}/{total}");
});
TestContext.WriteLine(file.ObjectId);
Assert.NotNull(file.ObjectId);
}

[Test]
[Order(11)]
public async Task AWSBigFile() {
LCApplication.Initialize("HudJvWWmAuGMifwxByDVLmQi-MdYXbMMI", "YjoQr1X8wHoFIfsSGXzeJaAM",
"https://hudjvwwm.api.lncldglobal.com");
LCFile file = new LCFile("video", VideoFilePath);
await file.Save((count, total) => {
TestContext.WriteLine($"progress: {count}/{total}");
});
TestContext.WriteLine(file.ObjectId);
Assert.NotNull(file.ObjectId);
}
}
}
7 changes: 4 additions & 3 deletions Storage/Storage/Internal/File/LCAWSUploader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ internal LCAWSUploader(string uploadUrl, string mimeType, Stream stream) {
}

internal async Task Upload(Action<long, long> onProgress) {
LCProgressableStreamContent content = new LCProgressableStreamContent(new StreamContent(stream), onProgress);
LCProgressableStreamContent content = new LCProgressableStreamContent(stream, onProgress);

HttpRequestMessage request = new HttpRequestMessage {
RequestUri = new Uri(uploadUrl),
Expand All @@ -42,8 +42,9 @@ internal async Task Upload(Action<long, long> onProgress) {
response.Dispose();
LCHttpUtils.PrintResponse(response, resultString);

HttpStatusCode statusCode = response.StatusCode;

if (!response.IsSuccessStatusCode) {
throw new Exception(resultString);
}
}
}
}
53 changes: 16 additions & 37 deletions Storage/Storage/Internal/File/LCProgressableStreamContent.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,58 +6,37 @@

namespace LeanCloud.Storage.Internal.File {
internal class LCProgressableStreamContent : HttpContent {
const int defaultBufferSize = 5 * 4096;
const int DEFAULT_BUFFER_SIZE = 100 * 1024;

readonly HttpContent content;

readonly int bufferSize;
readonly Stream content;

readonly Action<long, long> progress;

internal LCProgressableStreamContent(HttpContent content, Action<long, long> progress) : this(content, defaultBufferSize, progress) { }

internal LCProgressableStreamContent(HttpContent content, int bufferSize, Action<long, long> progress) {
internal LCProgressableStreamContent(Stream content, Action<long, long> progress) {
if (content == null) {
throw new ArgumentNullException("content");
}
if (bufferSize <= 0) {
throw new ArgumentOutOfRangeException("bufferSize");
throw new ArgumentNullException(nameof(content));
}

this.content = content;
this.bufferSize = bufferSize;
this.progress = progress;

foreach (var h in content.Headers) {
Headers.Add(h.Key, h.Value);
}
}

protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) {

return Task.Run(async () => {
var buffer = new byte[bufferSize];
TryComputeLength(out long size);
var uploaded = 0;

using (var sinput = await content.ReadAsStreamAsync()) {
while (true) {
var length = sinput.Read(buffer, 0, buffer.Length);
if (length <= 0) break;

uploaded += length;
progress?.Invoke(uploaded, size);

stream.Write(buffer, 0, length);
stream.Flush();
}
protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) {
byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
while (true) {
int length = await content.ReadAsync(buffer, 0, buffer.Length);
if (length <= 0) {
break;
}
stream.Flush();
});

await stream.WriteAsync(buffer, 0, length);
progress?.Invoke(content.Position, content.Length);
}
await stream.FlushAsync();
}

protected override bool TryComputeLength(out long length) {
length = content.Headers.ContentLength.GetValueOrDefault();
length = content.Length;
return true;
}

Expand Down
Loading

0 comments on commit bd1d2e0

Please sign in to comment.