feat: PPT-1517 Add Azure Storage support

PlaceOS · Oct 9, 2024 · b7e4e5e · b7e4e5e
1 parent 3bb8d8a
commit b7e4e5e
Show file tree

Hide file tree

Showing 4 changed files with 151 additions and 21 deletions.
diff --git a/OPENAPI_DOC.yml b/OPENAPI_DOC.yml
@@ -17998,7 +17998,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil_'
+                $ref: '#/components/schemas/NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil_'
         409:
           description: Conflict
           content:
@@ -18227,7 +18227,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/_NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__'
+                $ref: '#/components/schemas/_NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil__'
         409:
           description: Conflict
           content:
@@ -26586,7 +26586,7 @@ components:
         - type
         - signature
         - residence
-    ? NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil_
+    ? NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil_
     : type: object
       properties:
         type:
@@ -26609,6 +26609,9 @@ components:
         upload_id:
           type: string
           nullable: true
+        body:
+          type: string
+          nullable: true
       required:
       - type
       - signature
@@ -26635,6 +26638,9 @@ components:
               part:
                 type: integer
                 format: Int32
+              block_id:
+                type: string
+                nullable: true
             required:
             - md5
             - part
@@ -26648,7 +26654,7 @@ components:
         part_update:
           type: boolean
           nullable: true
-    ? _NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__
+    ? _NamedTuple_ok__Bool____NamedTuple_type__Symbol__signature__NamedTuple_verb__String__url__String__headers__Hash_String__String____upload_id__String___Nil__body__String___Nil__
     : anyOf:
       - type: object
         properties:
@@ -26678,6 +26684,9 @@ components:
           upload_id:
             type: string
             nullable: true
+          body:
+            type: string
+            nullable: true
         required:
         - type
         - signature

diff --git a/shard.lock b/shard.lock
@@ -29,6 +29,10 @@ shards:
     git: https://github.com/taylorfinnell/awscr-signer.git
     version: 0.8.2
 
+  azblob:
+    git: https://github.com/spider-gazelle/azblob.cr.git
+    version: 0.1.0+git.commit.16b822289fca1703cafc9dc9206733e86656e01c
+
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
     version: 1.2.2
@@ -63,7 +67,7 @@ shards:
 
   csuuid:
     git: https://github.com/wyhaines/csuuid.cr.git
-    version: 1.0.1+git.commit.4cb8656a9214aede9c1840cad4acf8e55e658f2f
+    version: 1.0.2
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -163,7 +167,7 @@ shards:
 
   nbchannel:
     git: https://github.com/wyhaines/nbchannel.cr.git
-    version: 0.1.0+git.commit.a8f5be6aa198abfa9f1893e1156640b8ea526094
+    version: 0.1.0
 
   neuroplastic:
     git: https://github.com/spider-gazelle/neuroplastic.git
@@ -183,15 +187,15 @@ shards:
 
   opentelemetry-api:
     git: https://github.com/wyhaines/opentelemetry-api.cr.git
-    version: 0.5.0
+    version: 0.5.1
 
   opentelemetry-instrumentation:
     git: https://github.com/wyhaines/opentelemetry-instrumentation.cr.git
     version: 0.5.4+git.commit.9b26aabef208b9eb0bf4612788362fa41768790b
 
   opentelemetry-sdk: # Overridden
     git: https://github.com/wyhaines/opentelemetry-sdk.cr.git
-    version: 0.6.1+git.commit.addc3c740d5ea8e61ffd9500fe32ebf21210d66c
+    version: 0.6.3+git.commit.470e34105727b039aee2bb3650e907bf6eefc971
 
   pars: # Overridden
     git: https://github.com/spider-gazelle/pars.git
@@ -235,7 +239,7 @@ shards:
 
   placeos-driver:
     git: https://github.com/placeos/driver.git
-    version: 7.2.4
+    version: 7.2.14
 
   placeos-frontend-loader:
     git: https://github.com/placeos/frontend-loader.git
@@ -335,7 +339,7 @@ shards:
 
   time-ext:
     git: https://github.com/wyhaines/time-ext.cr.git
-    version: 0.1.0+git.commit.175f658235fb6cdc9c804cb96da510fec27f4cd6
+    version: 1.0.1
 
   timecop:
     git: https://github.com/crystal-community/timecop.cr.git
@@ -347,15 +351,15 @@ shards:
 
   tracer:
     git: https://github.com/wyhaines/tracer.cr.git
-    version: 0.3.1
+    version: 0.3.2
 
   ulid: # Overridden
     git: https://github.com/place-labs/ulid.git
     version: 0.1.3+git.commit.0bb8d5d4bee4168acfac2630ce51ce4706253688
 
   upload-signer:
     git: https://github.com/spider-gazelle/upload-signer.git
-    version: 0.1.0+git.commit.4c7baf3fc72ca15035c827e16f2c8a15b5f39246
+    version: 0.2.0+git.commit.fdf8a1b2886006777efd01d69450c2028e0cf21e
 
   webmock:
     git: https://github.com/manastech/webmock.cr.git

diff --git a/spec/controllers/uploads_spec.cr b/spec/controllers/uploads_spec.cr
@@ -172,5 +172,97 @@ module PlaceOS::Api
         headers: Spec::Authentication.headers(sys_admin: false))
       resp.status_code.should eq(403)
     end
+
+    it "should properly handle azure storage for direct uploads" do
+      storage = Model::Generator.storage(type: PlaceOS::Model::Storage::Type::Azure)
+      storage.access_key = "myteststorage"
+      storage.access_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+      storage.save!
+      params = {
+        "file_name"   => "some_file_name.jpg",
+        "file_size"   => "500",
+        "file_id"     => "some_file_md5_hash",
+        "file_mime"   => "image/jpeg",
+        "public"      => false,
+        "permissions" => "admin",
+      }
+
+      resp = client.post(Uploads.base_route,
+        body: params.to_json,
+        headers: Spec::Authentication.headers)
+
+      resp.status_code.should eq(200)
+      info = JSON.parse(resp.body).as_h
+      info["type"].should eq("direct_upload")
+      sig = info["signature"].as_h
+      sig["verb"].as_s.should eq("PUT")
+      sig["url"].as_s.should_not be_nil
+      upload = Model::Upload.find?(info["upload_id"].as_s)
+      upload.should_not be_nil
+    end
+
+    it "should properly handle azure storage for chunked uploads" do
+      storage = Model::Generator.storage(type: PlaceOS::Model::Storage::Type::Azure)
+      storage.access_key = "myteststorage"
+      storage.access_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+      storage.save!
+      params = {
+        "file_name"   => "some_file_name.jpg",
+        "file_size"   => (258 * 1024 * 1024).to_s,
+        "file_id"     => "some_file_md5_hash",
+        "file_mime"   => "image/jpeg",
+        "public"      => false,
+        "permissions" => "admin",
+      }
+
+      resp = client.post(Uploads.base_route,
+        body: params.to_json,
+        headers: Spec::Authentication.headers)
+
+      resp.status_code.should eq(200)
+      info = JSON.parse(resp.body).as_h
+      info["type"].should eq("chunked_upload")
+      info["residence"].should eq("AzureStorage")
+      sig = info["signature"].as_h
+      sig["verb"].as_s.should eq("PUT")
+      sig["url"].as_s.size.should eq(0)
+      upload = Model::Upload.find!(info["upload_id"].as_s)
+
+      params = {
+        "part"    => Base64.strict_encode(UUID.random.to_s),
+        "file_id" => "some_file_md5_hash",
+      }
+
+      pinfo = Uploads::PartInfo.new(params["file_id"], 1, params["part"])
+      uinfo = Uploads::UpdateInfo.new(params["file_id"], 1, "some-random-resumable-id", [pinfo], [1], false)
+
+      resp = client.patch(
+        path: "#{Uploads.base_route}/#{upload.id}?#{HTTP::Params.encode(params)}",
+        body: uinfo.to_json,
+        headers: Spec::Authentication.headers)
+
+      resp.status_code.should eq(200)
+      info = JSON.parse(resp.body).as_h
+      info["type"].should eq("part_upload")
+      sig = info["signature"].as_h
+      sig["verb"].as_s.should eq("PUT")
+      sig["url"].as_s.size.should be > 0
+      uri = URI.parse(sig["url"].as_s)
+      uri.host.should eq(sprintf("%s.blob.core.windows.net", "myteststorage"))
+      qparams = URI::Params.parse(uri.query || "")
+      qparams["blockid"].should eq(params["part"])
+
+      params = {
+        "part" => "finish",
+      }
+      resp = client.get(
+        path: "#{Uploads.base_route}/#{upload.id}/edit?#{HTTP::Params.encode(params)}",
+        headers: Spec::Authentication.headers)
+
+      resp.status_code.should eq(200)
+      info = JSON.parse(resp.body).as_h
+      info["type"].should eq("finish")
+      info["body"].should_not be_nil
+    end
   end
 end
diff --git a/src/placeos-rest-api/controllers/uploads.cr b/src/placeos-rest-api/controllers/uploads.cr
@@ -2,6 +2,7 @@ require "mime"
 require "upload-signer"
 require "placeos-models/storage"
 require "placeos-models/upload"
+require "xml"
 require "./application"
 
 module PlaceOS::Api
@@ -40,12 +41,12 @@ module PlaceOS::Api
                raise Error::NotFound.new(ex.message || "Authority storage configuration not found")
              end
       end
-      @signer = UploadSigner::AmazonS3.new(storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint)
+      @signer = UploadSigner.signer(UploadSigner::StorageType.from_value(storage.storage_type.value), storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint)
     end
 
     getter! authority : ::PlaceOS::Model::Authority?
     getter! storage : ::PlaceOS::Model::Storage?
-    getter! signer : UploadSigner::AmazonS3?
+    getter! signer : UploadSigner::Storage?
     getter! current_upload : ::PlaceOS::Model::Upload?
 
     # returns the list of uploads for current domain authority
@@ -192,8 +193,9 @@ module PlaceOS::Api
         end
       end
 
-      s3 = UploadSigner::AmazonS3.new(storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint)
-      object_url = s3.get_object(storage.bucket_name, current_upload.object_key, expiry * 60)
+      us = UploadSigner.signer(UploadSigner::StorageType.from_value(storage.storage_type.value), storage.access_key, storage.decrypt_secret, storage.region, endpoint: storage.endpoint)
+
+      object_url = us.get_object(storage.bucket_name, current_upload.object_key, expiry * 60)
 
       redirect_to object_url, status: :see_other
     end
@@ -211,24 +213,37 @@ module PlaceOS::Api
     ) : NamedTuple(
       type: Symbol,
       signature: NamedTuple(verb: String, url: String, headers: Hash(String, String)),
-      upload_id: String | Nil)
+      upload_id: String | Nil, body: String | Nil)
       if (resumable_id = current_upload.resumable_id) && current_upload.resumable
         if part.strip == "finish"
           s3 = signer.commit_file(storage.bucket_name, current_upload.object_key, resumable_id, get_headers(current_upload))
-          {type: :finish, signature: s3, upload_id: current_upload.id}
+          finish_body = nil
+          if storage.storage_type == PlaceOS::Model::Storage::Type::Azure
+            if part_data = current_upload.part_data
+              block_ids = [] of String
+              parts = part_data.keys.sort!
+              parts.each do |ppart|
+                block_ids << part_data[ppart].as_h["block_id"].as_s
+              end
+              finish_body = block_list_xml(block_ids)
+            else
+              raise AC::Route::Param::ValueError.new("missing part_data information. Required for AzureStorage")
+            end
+          end
+          {type: :finish, signature: s3, upload_id: current_upload.id, body: finish_body}
         else
           unless md5 = file_id
             raise AC::Route::Param::ValueError.new("Missing MD5 hash of file part", "file_id", "required except for the `finish` part")
           end
           s3 = signer.set_part(storage.bucket_name, current_upload.object_key, current_upload.file_size, md5, part, resumable_id, get_headers(current_upload))
-          {type: :part_upload, signature: s3, upload_id: current_upload.id}
+          {type: :part_upload, signature: s3, upload_id: current_upload.id, body: nil}
         end
       else
         raise AC::Route::Param::ValueError.new("upload is not resumable, no part available")
       end
     end
 
-    record PartInfo, md5 : String, part : Int32 do
+    record PartInfo, md5 : String, part : Int32, block_id : String? do
       include JSON::Serializable
     end
 
@@ -248,7 +263,7 @@ module PlaceOS::Api
     ) : NamedTuple(
       type: Symbol,
       signature: NamedTuple(verb: String, url: String, headers: Hash(String, String)),
-      upload_id: String | Nil) | NamedTuple(ok: Bool)
+      upload_id: String | Nil, body: String | Nil) | NamedTuple(ok: Bool)
       raise AC::Route::Param::ValueError.new("upload is not resumable") unless current_upload.resumable
 
       if part_list = info.part_list
@@ -361,5 +376,15 @@ module PlaceOS::Api
         end
       end
     end
+
+    private def block_list_xml(block_ids)
+      XML.build(encoding: "UTF-8") do |xml|
+        xml.element("BlockList") do
+          block_ids.each do |tag|
+            xml.element("Latest") { xml.text(tag) }
+          end
+        end
+      end
+    end
   end
 end