diff --git a/Common/Common/LCCore.cs b/Common/Common/LCCore.cs index 2e391641..3b0b987d 100644 --- a/Common/Common/LCCore.cs +++ b/Common/Common/LCCore.cs @@ -52,10 +52,10 @@ public static void Initialize(string appId, string server = null, string masterKey = null) { if (string.IsNullOrEmpty(appId)) { - throw new ArgumentException(nameof(appId)); + throw new ArgumentNullException(nameof(appId)); } if (string.IsNullOrEmpty(appKey)) { - throw new ArgumentException(nameof(appKey)); + throw new ArgumentNullException(nameof(appKey)); } AppId = appId; diff --git a/Storage/Storage.Test/FileTest.cs b/Storage/Storage.Test/FileTest.cs index b2aec705..b8e34a34 100644 --- a/Storage/Storage.Test/FileTest.cs +++ b/Storage/Storage.Test/FileTest.cs @@ -9,13 +9,14 @@ namespace Storage.Test { public class FileTest : BaseTest { static readonly string AvatarFilePath = "../../../../../assets/hello.png"; static readonly string APKFilePath = "../../../../../assets/test.apk"; + static readonly string VideoFilePath = "../../../../../assets/video.mp4"; - private LCFile avatar; + private LCFile video; [Test] [Order(0)] public async Task SaveFromPath() { - avatar = new LCFile("avatar", AvatarFilePath); + LCFile avatar = new LCFile("avatar", AvatarFilePath); await avatar.Save((count, total) => { TestContext.WriteLine($"progress: {count}/{total}"); }); @@ -25,15 +26,27 @@ await avatar.Save((count, total) => { [Test] [Order(1)] + public async Task SaveBigFileFromPath() { + video = new LCFile("video", VideoFilePath); + await video.Save((count, total) => { + TestContext.WriteLine($"progress: {count}/{total}"); + }); + TestContext.WriteLine(video.ObjectId); + Assert.NotNull(video.ObjectId); + } + + [Test] + [Order(2)] public async Task QueryFile() { LCQuery query = LCFile.GetQuery(); - LCFile file = await query.Get(avatar.ObjectId); + LCFile file = await query.Get(video.ObjectId); Assert.NotNull(file.Url); TestContext.WriteLine(file.Url); TestContext.WriteLine(file.GetThumbnailUrl(32, 32)); } [Test] + [Order(3)] public async Task SaveFromMemory() { string text = "hello, world"; byte[] data = Encoding.UTF8.GetBytes(text); @@ -46,6 +59,7 @@ public async Task SaveFromMemory() { } [Test] + [Order(4)] public async Task SaveFromUrl() { LCFile file = new LCFile("scene", new Uri("http://img95.699pic.com/photo/50015/9034.jpg_wh300.jpg")); file.AddMetaData("size", 1024); @@ -58,6 +72,7 @@ public async Task SaveFromUrl() { } [Test] + [Order(5)] public async Task Qiniu() { LCFile file = new LCFile("avatar", APKFilePath); await file.Save(); @@ -66,6 +81,7 @@ public async Task Qiniu() { } [Test] + [Order(6)] public async Task FileACL() { LCUser user = await LCUser.LoginAnonymously(); @@ -87,15 +103,30 @@ public async Task FileACL() { } } - //[Test] - //public async Task AWS() { - // LCApplication.Initialize("UlCpyvLm8aMzQsW6KnP6W3Wt-MdYXbMMI", "PyCTYoNoxCVoKKg394PBeS4r"); - // LCFile file = new LCFile("avatar", AvatarFilePath); - // await file.Save((count, total) => { - // TestContext.WriteLine($"progress: {count}/{total}"); - // }); - // TestContext.WriteLine(file.ObjectId); - // Assert.NotNull(file.ObjectId); - //} + [Test] + [Order(10)] + public async Task AWS() { + LCApplication.Initialize("HudJvWWmAuGMifwxByDVLmQi-MdYXbMMI", "YjoQr1X8wHoFIfsSGXzeJaAM", + "https://hudjvwwm.api.lncldglobal.com"); + LCFile file = new LCFile("avatar", AvatarFilePath); + await file.Save((count, total) => { + TestContext.WriteLine($"progress: {count}/{total}"); + }); + TestContext.WriteLine(file.ObjectId); + Assert.NotNull(file.ObjectId); + } + + [Test] + [Order(11)] + public async Task AWSBigFile() { + LCApplication.Initialize("HudJvWWmAuGMifwxByDVLmQi-MdYXbMMI", "YjoQr1X8wHoFIfsSGXzeJaAM", + "https://hudjvwwm.api.lncldglobal.com"); + LCFile file = new LCFile("video", VideoFilePath); + await file.Save((count, total) => { + TestContext.WriteLine($"progress: {count}/{total}"); + }); + TestContext.WriteLine(file.ObjectId); + Assert.NotNull(file.ObjectId); + } } } diff --git a/Storage/Storage/Internal/File/LCAWSUploader.cs b/Storage/Storage/Internal/File/LCAWSUploader.cs index 1c9a731b..307dbf0a 100644 --- a/Storage/Storage/Internal/File/LCAWSUploader.cs +++ b/Storage/Storage/Internal/File/LCAWSUploader.cs @@ -21,7 +21,7 @@ internal LCAWSUploader(string uploadUrl, string mimeType, Stream stream) { } internal async Task Upload(Action onProgress) { - LCProgressableStreamContent content = new LCProgressableStreamContent(new StreamContent(stream), onProgress); + LCProgressableStreamContent content = new LCProgressableStreamContent(stream, onProgress); HttpRequestMessage request = new HttpRequestMessage { RequestUri = new Uri(uploadUrl), @@ -42,8 +42,9 @@ internal async Task Upload(Action onProgress) { response.Dispose(); LCHttpUtils.PrintResponse(response, resultString); - HttpStatusCode statusCode = response.StatusCode; - + if (!response.IsSuccessStatusCode) { + throw new Exception(resultString); + } } } } diff --git a/Storage/Storage/Internal/File/LCProgressableStreamContent.cs b/Storage/Storage/Internal/File/LCProgressableStreamContent.cs index 161c273e..2c5025af 100644 --- a/Storage/Storage/Internal/File/LCProgressableStreamContent.cs +++ b/Storage/Storage/Internal/File/LCProgressableStreamContent.cs @@ -6,58 +6,37 @@ namespace LeanCloud.Storage.Internal.File { internal class LCProgressableStreamContent : HttpContent { - const int defaultBufferSize = 5 * 4096; + const int DEFAULT_BUFFER_SIZE = 100 * 1024; - readonly HttpContent content; - - readonly int bufferSize; + readonly Stream content; readonly Action progress; - internal LCProgressableStreamContent(HttpContent content, Action progress) : this(content, defaultBufferSize, progress) { } - - internal LCProgressableStreamContent(HttpContent content, int bufferSize, Action progress) { + internal LCProgressableStreamContent(Stream content, Action progress) { if (content == null) { - throw new ArgumentNullException("content"); - } - if (bufferSize <= 0) { - throw new ArgumentOutOfRangeException("bufferSize"); + throw new ArgumentNullException(nameof(content)); } this.content = content; - this.bufferSize = bufferSize; this.progress = progress; - - foreach (var h in content.Headers) { - Headers.Add(h.Key, h.Value); - } } - protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) { - - return Task.Run(async () => { - var buffer = new byte[bufferSize]; - TryComputeLength(out long size); - var uploaded = 0; - - using (var sinput = await content.ReadAsStreamAsync()) { - while (true) { - var length = sinput.Read(buffer, 0, buffer.Length); - if (length <= 0) break; - - uploaded += length; - progress?.Invoke(uploaded, size); - - stream.Write(buffer, 0, length); - stream.Flush(); - } + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) { + byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; + while (true) { + int length = await content.ReadAsync(buffer, 0, buffer.Length); + if (length <= 0) { + break; } - stream.Flush(); - }); + + await stream.WriteAsync(buffer, 0, length); + progress?.Invoke(content.Position, content.Length); + } + await stream.FlushAsync(); } protected override bool TryComputeLength(out long length) { - length = content.Headers.ContentLength.GetValueOrDefault(); + length = content.Length; return true; } diff --git a/Storage/Storage/Internal/File/LCQiniuUploader.cs b/Storage/Storage/Internal/File/LCQiniuUploader.cs index 29ea743c..1a033ef0 100644 --- a/Storage/Storage/Internal/File/LCQiniuUploader.cs +++ b/Storage/Storage/Internal/File/LCQiniuUploader.cs @@ -1,42 +1,168 @@ using System; +using System.Text; using System.IO; using System.Threading.Tasks; -using System.Net; +using System.Web; using System.Net.Http; +using System.Net.Http.Headers; +using System.Collections.Generic; +using System.Security.Cryptography; using LeanCloud.Common; +using LC.Newtonsoft.Json; namespace LeanCloud.Storage.Internal.File { + class QiniuBlock { + [JsonProperty("etag")] + public string ETag { + get; set; + } + + [JsonProperty("md5")] + public string MD5 { + get; set; + } + } + + class QiniuPart { + [JsonProperty("partNumber")] + public int PartNumber { + get; set; + } + + [JsonProperty("etag")] + public string ETag { + get; set; + } + } + internal class LCQiniuUploader { private string uploadUrl; private string token; + private string bucket; + private string key; private Stream stream; - internal LCQiniuUploader(string uploadUrl, string token, string key, Stream stream) { + internal LCQiniuUploader(string uploadUrl, string token, string bucket, string key, Stream stream) { this.uploadUrl = uploadUrl; this.token = token; + this.bucket = bucket; this.key = key; this.stream = stream; } internal async Task Upload(Action onProgress) { - MultipartFormDataContent dataContent = new MultipartFormDataContent(); - dataContent.Add(new StringContent(key), "key"); - dataContent.Add(new StringContent(token), "token"); - dataContent.Add(new StreamContent(stream), "file"); + string encodedObjectName = HttpUtility.UrlEncode(Convert.ToBase64String(Encoding.UTF8.GetBytes(key))); + + HttpClient client = new HttpClient(); + + Uri uri = new Uri(uploadUrl); + client.DefaultRequestHeaders.Host = uri.Host; + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("UpToken", token); + + // 1. 初始化任务,请求 upload id + string uploadId = await RequestUploadId(client, encodedObjectName); - LCProgressableStreamContent content = new LCProgressableStreamContent(dataContent, onProgress); + // 2. 分块上传数据 + List parts = await UploadBlocks(client, encodedObjectName, uploadId, onProgress); + // 3. 完成文件上传 + await FinishUpload(client, encodedObjectName, uploadId, parts); + } + + async Task RequestUploadId(HttpClient client, string encodedObjectName) { + string endpoint = $"buckets/{bucket}/objects/{encodedObjectName}/uploads"; HttpRequestMessage request = new HttpRequestMessage { - RequestUri = new Uri(uploadUrl), + RequestUri = new Uri($"{uploadUrl}/{endpoint}"), Method = HttpMethod.Post, - Content = content }; - HttpClient client = new HttpClient(); + LCHttpUtils.PrintRequest(client, request); + + HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + request.Dispose(); + + string resultString = await response.Content.ReadAsStringAsync(); + response.Dispose(); + LCHttpUtils.PrintResponse(response, resultString); + + if (!response.IsSuccessStatusCode) { + throw new Exception(resultString); + } + + Dictionary res = JsonConvert.DeserializeObject>(resultString, + LCJsonConverter.Default); + string uploadId = res["uploadId"] as string; + + return uploadId; + } + + async Task> UploadBlocks(HttpClient client, string encodedObjectName, string uploadId, Action onProgress) { + int size = 4 * 1024 * 1024; + byte[] buffer = new byte[size]; + long blockCount = (stream.Length + buffer.Length - 1) / buffer.Length; + List parts = new List(); + for (int i = 1; i <= blockCount; i++) { + int count = await stream.ReadAsync(buffer, 0, Math.Min((int)(stream.Length - (i - 1) * size), size)); + string endpoint = $"buckets/{bucket}/objects/{encodedObjectName}/uploads/{uploadId}/{i}"; + + MemoryStream memoryStream = new MemoryStream(buffer, 0, count); + + LCProgressableStreamContent content = new LCProgressableStreamContent(memoryStream, (uploaded, _) => { + long totalUploaded = size * (i - 1) + uploaded; + onProgress?.Invoke(totalUploaded, stream.Length); + }); + HttpRequestMessage request = new HttpRequestMessage { + RequestUri = new Uri($"{uploadUrl}/{endpoint}"), + Method = HttpMethod.Put, + Content = content + }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + request.Content.Headers.ContentMD5 = CalcMD5(buffer, 0, count); + request.Content.Headers.ContentLength = count; + LCHttpUtils.PrintRequest(client, request); + + HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + request.Dispose(); + + string resultString = await response.Content.ReadAsStringAsync(); + response.Dispose(); + LCHttpUtils.PrintResponse(response, resultString); + + if (!response.IsSuccessStatusCode) { + throw new Exception(resultString); + } + + QiniuBlock block = JsonConvert.DeserializeObject(resultString, + LCJsonConverter.Default); + + QiniuPart part = new QiniuPart(); + part.PartNumber = i; + part.ETag = block.ETag; + parts.Add(part); + } + + return parts; + } + + async Task FinishUpload(HttpClient client, string encodedObjectName, string uploadId, List parts) { + string endpoint = $"buckets/{bucket}/objects/{encodedObjectName}/uploads/{uploadId}"; + Dictionary data = new Dictionary { + { "parts", parts } + }; + + string body = JsonConvert.SerializeObject(data); + HttpRequestMessage request = new HttpRequestMessage { + RequestUri = new Uri($"{uploadUrl}/{endpoint}"), + Method = HttpMethod.Post, + Content = new StringContent(body) + }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + LCHttpUtils.PrintRequest(client, request, body); + HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); request.Dispose(); @@ -44,7 +170,15 @@ internal async Task Upload(Action onProgress) { response.Dispose(); LCHttpUtils.PrintResponse(response, resultString); - HttpStatusCode statusCode = response.StatusCode; + if (!response.IsSuccessStatusCode) { + throw new Exception(resultString); + } + } + + static byte[] CalcMD5(byte[] buffer, int index, int count) { + MD5 md5 = MD5.Create(); + byte[] data = md5.ComputeHash(buffer, index, count); + return data; } } } diff --git a/Storage/Storage/Public/LCFile.cs b/Storage/Storage/Public/LCFile.cs index d758dbd6..8b06b1dd 100644 --- a/Storage/Storage/Public/LCFile.cs +++ b/Storage/Storage/Public/LCFile.cs @@ -140,7 +140,8 @@ public async Task Save(Action onProgress = null) { await uploader.Upload(onProgress); } else if (provider == "qiniu") { // Qiniu - LCQiniuUploader uploader = new LCQiniuUploader(uploadUrl, token, key, stream); + string bucket = uploadToken["bucket"] as string; + LCQiniuUploader uploader = new LCQiniuUploader(uploadUrl, token, bucket, key, stream); await uploader.Upload(onProgress); } else { throw new Exception($"{provider} is not support."); @@ -157,6 +158,9 @@ public async Task Save(Action onProgress = null) { { "token", token } }); throw e; + } finally { + stream?.Close(); + stream?.Dispose(); } } return this; diff --git a/assets/video.mp4 b/assets/video.mp4 new file mode 100644 index 00000000..b28a17bd Binary files /dev/null and b/assets/video.mp4 differ