From b7e4e5e0272c8c51a2a8770b382ef88c43c4683f Mon Sep 17 00:00:00 2001 From: Ali Naqvi Date: Wed, 9 Oct 2024 14:47:17 +0800 Subject: [PATCH] feat: PPT-1517 Add Azure Storage support --- OPENAPI_DOC.yml | 17 +++- shard.lock | 20 +++-- spec/controllers/uploads_spec.cr | 92 +++++++++++++++++++++ src/placeos-rest-api/controllers/uploads.cr | 43 ++++++++-- 4 files changed, 151 insertions(+), 21 deletions(-) diff --git a/OPENAPI_DOC.yml b/OPENAPI_DOC.yml index 454a6ba2..06848b06 100644 --- a/OPENAPI_DOC.yml +++ b/OPENAPI_DOC.yml @@ -17998,7 +17998,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil_' + $ref: '#/components/schemas/NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil_' 409: description: Conflict content: @@ -18227,7 +18227,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/_NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__' + $ref: '#/components/schemas/_NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil__' 409: description: Conflict content: @@ -26586,7 +26586,7 @@ components: - type - signature - residence - ? NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil_ + ? NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil_ : type: object properties: type: @@ -26609,6 +26609,9 @@ components: upload_id: type: string nullable: true + body: + type: string + nullable: true required: - type - signature @@ -26635,6 +26638,9 @@ components: part: type: integer format: Int32 + block_id: + type: string + nullable: true required: - md5 - part @@ -26648,7 +26654,7 @@ components: part_update: type: boolean nullable: true - ? _NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__ + ? _NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil__ : anyOf: - type: object properties: @@ -26678,6 +26684,9 @@ components: upload_id: type: string nullable: true + body: + type: string + nullable: true required: - type - signature diff --git a/shard.lock b/shard.lock index 2b176ed5..547455f5 100644 --- a/shard.lock +++ b/shard.lock @@ -29,6 +29,10 @@ shards: git: https://github.com/taylorfinnell/awscr-signer.git version: 0.8.2 + azblob: + git: https://github.com/spider-gazelle/azblob.cr.git + version: 0.1.0+git.commit.16b822289fca1703cafc9dc9206733e86656e01c + backtracer: git: https://github.com/sija/backtracer.cr.git version: 1.2.2 @@ -63,7 +67,7 @@ shards: csuuid: git: https://github.com/wyhaines/csuuid.cr.git - version: 1.0.1+git.commit.4cb8656a9214aede9c1840cad4acf8e55e658f2f + version: 1.0.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -163,7 +167,7 @@ shards: nbchannel: git: https://github.com/wyhaines/nbchannel.cr.git - version: 0.1.0+git.commit.a8f5be6aa198abfa9f1893e1156640b8ea526094 + version: 0.1.0 neuroplastic: git: https://github.com/spider-gazelle/neuroplastic.git @@ -183,7 +187,7 @@ shards: opentelemetry-api: git: https://github.com/wyhaines/opentelemetry-api.cr.git - version: 0.5.0 + version: 0.5.1 opentelemetry-instrumentation: git: https://github.com/wyhaines/opentelemetry-instrumentation.cr.git @@ -191,7 +195,7 @@ shards: opentelemetry-sdk: # Overridden git: https://github.com/wyhaines/opentelemetry-sdk.cr.git - version: 0.6.1+git.commit.addc3c740d5ea8e61ffd9500fe32ebf21210d66c + version: 0.6.3+git.commit.470e34105727b039aee2bb3650e907bf6eefc971 pars: # Overridden git: https://github.com/spider-gazelle/pars.git @@ -235,7 +239,7 @@ shards: placeos-driver: git: https://github.com/placeos/driver.git - version: 7.2.4 + version: 7.2.14 placeos-frontend-loader: git: https://github.com/placeos/frontend-loader.git @@ -335,7 +339,7 @@ shards: time-ext: git: https://github.com/wyhaines/time-ext.cr.git - version: 0.1.0+git.commit.175f658235fb6cdc9c804cb96da510fec27f4cd6 + version: 1.0.1 timecop: git: https://github.com/crystal-community/timecop.cr.git @@ -347,7 +351,7 @@ shards: tracer: git: https://github.com/wyhaines/tracer.cr.git - version: 0.3.1 + version: 0.3.2 ulid: # Overridden git: https://github.com/place-labs/ulid.git @@ -355,7 +359,7 @@ shards: upload-signer: git: https://github.com/spider-gazelle/upload-signer.git - version: 0.1.0+git.commit.4c7baf3fc72ca15035c827e16f2c8a15b5f39246 + version: 0.2.0+git.commit.fdf8a1b2886006777efd01d69450c2028e0cf21e webmock: git: https://github.com/manastech/webmock.cr.git diff --git a/spec/controllers/uploads_spec.cr b/spec/controllers/uploads_spec.cr index a5741747..84860676 100644 --- a/spec/controllers/uploads_spec.cr +++ b/spec/controllers/uploads_spec.cr @@ -172,5 +172,97 @@ module PlaceOS::Api headers: Spec::Authentication.headers(sys_admin: false)) resp.status_code.should eq(403) end + + it "should properly handle azure storage for direct uploads" do + storage = Model::Generator.storage(type: PlaceOS::Model::Storage::Type::Azure) + storage.access_key = "myteststorage" + storage.access_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + storage.save! + params = { + "file_name" => "some_file_name.jpg", + "file_size" => "500", + "file_id" => "some_file_md5_hash", + "file_mime" => "image/jpeg", + "public" => false, + "permissions" => "admin", + } + + resp = client.post(Uploads.base_route, + body: params.to_json, + headers: Spec::Authentication.headers) + + resp.status_code.should eq(200) + info = JSON.parse(resp.body).as_h + info["type"].should eq("direct_upload") + sig = info["signature"].as_h + sig["verb"].as_s.should eq("PUT") + sig["url"].as_s.should_not be_nil + upload = Model::Upload.find?(info["upload_id"].as_s) + upload.should_not be_nil + end + + it "should properly handle azure storage for chunked uploads" do + storage = Model::Generator.storage(type: PlaceOS::Model::Storage::Type::Azure) + storage.access_key = "myteststorage" + storage.access_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + storage.save! + params = { + "file_name" => "some_file_name.jpg", + "file_size" => (258 * 1024 * 1024).to_s, + "file_id" => "some_file_md5_hash", + "file_mime" => "image/jpeg", + "public" => false, + "permissions" => "admin", + } + + resp = client.post(Uploads.base_route, + body: params.to_json, + headers: Spec::Authentication.headers) + + resp.status_code.should eq(200) + info = JSON.parse(resp.body).as_h + info["type"].should eq("chunked_upload") + info["residence"].should eq("AzureStorage") + sig = info["signature"].as_h + sig["verb"].as_s.should eq("PUT") + sig["url"].as_s.size.should eq(0) + upload = Model::Upload.find!(info["upload_id"].as_s) + + params = { + "part" => Base64.strict_encode(UUID.random.to_s), + "file_id" => "some_file_md5_hash", + } + + pinfo = Uploads::PartInfo.new(params["file_id"], 1, params["part"]) + uinfo = Uploads::UpdateInfo.new(params["file_id"], 1, "some-random-resumable-id", [pinfo], [1], false) + + resp = client.patch( + path: "#{Uploads.base_route}/#{upload.id}?#{HTTP::Params.encode(params)}", + body: uinfo.to_json, + headers: Spec::Authentication.headers) + + resp.status_code.should eq(200) + info = JSON.parse(resp.body).as_h + info["type"].should eq("part_upload") + sig = info["signature"].as_h + sig["verb"].as_s.should eq("PUT") + sig["url"].as_s.size.should be > 0 + uri = URI.parse(sig["url"].as_s) + uri.host.should eq(sprintf("%s.blob.core.windows.net", "myteststorage")) + qparams = URI::Params.parse(uri.query || "") + qparams["blockid"].should eq(params["part"]) + + params = { + "part" => "finish", + } + resp = client.get( + path: "#{Uploads.base_route}/#{upload.id}/edit?#{HTTP::Params.encode(params)}", + headers: Spec::Authentication.headers) + + resp.status_code.should eq(200) + info = JSON.parse(resp.body).as_h + info["type"].should eq("finish") + info["body"].should_not be_nil + end end end diff --git a/src/placeos-rest-api/controllers/uploads.cr b/src/placeos-rest-api/controllers/uploads.cr index 75045f03..acff71d4 100644 --- a/src/placeos-rest-api/controllers/uploads.cr +++ b/src/placeos-rest-api/controllers/uploads.cr @@ -2,6 +2,7 @@ require "mime" require "upload-signer" require "placeos-models/storage" require "placeos-models/upload" +require "xml" require "./application" module PlaceOS::Api @@ -40,12 +41,12 @@ module PlaceOS::Api raise Error::NotFound.new(ex.message || "Authority storage configuration not found") end end - @signer = UploadSigner::AmazonS3.new(storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint) + @signer = UploadSigner.signer(UploadSigner::StorageType.from_value(storage.storage_type.value), storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint) end getter! authority : ::PlaceOS::Model::Authority? getter! storage : ::PlaceOS::Model::Storage? - getter! signer : UploadSigner::AmazonS3? + getter! signer : UploadSigner::Storage? getter! current_upload : ::PlaceOS::Model::Upload? # returns the list of uploads for current domain authority @@ -192,8 +193,9 @@ module PlaceOS::Api end end - s3 = UploadSigner::AmazonS3.new(storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint) - object_url = s3.get_object(storage.bucket_name, current_upload.object_key, expiry * 60) + us = UploadSigner.signer(UploadSigner::StorageType.from_value(storage.storage_type.value), storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint) + + object_url = us.get_object(storage.bucket_name, current_upload.object_key, expiry * 60) redirect_to object_url, status: :see_other end @@ -211,24 +213,37 @@ module PlaceOS::Api ) : NamedTuple( type: Symbol, signature: NamedTuple(verb: String, url: String, headers: Hash(String, String)), - upload_id: String | Nil) + upload_id: String | Nil, body: String | Nil) if (resumable_id = current_upload.resumable_id) && current_upload.resumable if part.strip == "finish" s3 = signer.commit_file(storage.bucket_name, current_upload.object_key, resumable_id, get_headers(current_upload)) - {type: :finish, signature: s3, upload_id: current_upload.id} + finish_body = nil + if storage.storage_type == PlaceOS::Model::Storage::Type::Azure + if part_data = current_upload.part_data + block_ids = [] of String + parts = part_data.keys.sort! + parts.each do |ppart| + block_ids << part_data[ppart].as_h["block_id"].as_s + end + finish_body = block_list_xml(block_ids) + else + raise AC::Route::Param::ValueError.new("missing part_data information. Required for AzureStorage") + end + end + {type: :finish, signature: s3, upload_id: current_upload.id, body: finish_body} else unless md5 = file_id raise AC::Route::Param::ValueError.new("Missing MD5 hash of file part", "file_id", "required except for the `finish` part") end s3 = signer.set_part(storage.bucket_name, current_upload.object_key, current_upload.file_size, md5, part, resumable_id, get_headers(current_upload)) - {type: :part_upload, signature: s3, upload_id: current_upload.id} + {type: :part_upload, signature: s3, upload_id: current_upload.id, body: nil} end else raise AC::Route::Param::ValueError.new("upload is not resumable, no part available") end end - record PartInfo, md5 : String, part : Int32 do + record PartInfo, md5 : String, part : Int32, block_id : String? do include JSON::Serializable end @@ -248,7 +263,7 @@ module PlaceOS::Api ) : NamedTuple( type: Symbol, signature: NamedTuple(verb: String, url: String, headers: Hash(String, String)), - upload_id: String | Nil) | NamedTuple(ok: Bool) + upload_id: String | Nil, body: String | Nil) | NamedTuple(ok: Bool) raise AC::Route::Param::ValueError.new("upload is not resumable") unless current_upload.resumable if part_list = info.part_list @@ -361,5 +376,15 @@ module PlaceOS::Api end end end + + private def block_list_xml(block_ids) + XML.build(encoding: "UTF-8") do |xml| + xml.element("BlockList") do + block_ids.each do |tag| + xml.element("Latest") { xml.text(tag) } + end + end + end + end end end