feat: add support for pfop workflow template (#265)

CHANGELOG.md

@@ -1,5 +1,9 @@
 **2024-09-05**
 
+v8.7.0
+
+新增：持久化处理，支持工作流模版
+
 v8.6.0
 
 新增：验证 Qbox, Qiniu 签名的辅助方法

src/Qiniu/QiniuCSharpSDK.cs

@@ -37,6 +37,6 @@ public class QiniuCSharpSDK
     /// <summary>
     /// SDK版本号
     /// </summary>
-    public const string VERSION = "8.6.0";
+    public const string VERSION = "8.7.0";
 
 }

src/Qiniu/Storage/OperationManager.cs

@@ -37,31 +37,46 @@ public OperationManager(Mac mac, Config config)
         /// </summary>
         /// <param name="bucket">空间</param>
         /// <param name="key">空间文件的key</param>
-        /// <param name="fops">操作(命令参数)</param>
+        /// <param name="fops">操作(命令参数)，与 workflowTemplateId 二选一</param>
         /// <param name="pipeline">私有队列</param>
         /// <param name="notifyUrl">通知url</param>
         /// <param name="force">forece参数</param>
-        /// <param name="persistentType">为 1 时开启闲时任务</param>
+        /// <param name="type">为 1 时开启闲时任务</param>
+        /// <param name="workflowTemplateId">模版 ID，与 fops 二选一</param>
         /// <returns>pfop操作返回结果，正确返回结果包含persistentId</returns>
         public PfopResult Pfop(
             string bucket,
             string key,
-            string fops,
-            string pipeline,
-            string notifyUrl,
-            bool force,
-            int type = 0
+            string fops=null,
+            string pipeline=null,
+            string notifyUrl=null,
+            bool force=false,
+            int type = 0,
+            string workflowTemplateId = null
         )
         {
             PfopResult result = new PfopResult();
 
+            if (string.IsNullOrEmpty(fops) && string.IsNullOrEmpty(workflowTemplateId))
+            {
+                throw new ArgumentException("Must provide one of fops or workflowTemplateId");
+            }
+
             try
             {
                 string pfopUrl = string.Format("{0}/pfop/", this.config.ApiHost(this.mac.AccessKey, bucket));
 
                 StringBuilder sb = new StringBuilder();
-                sb.AppendFormat("bucket={0}&key={1}&fops={2}", StringHelper.UrlEncode(bucket), StringHelper.UrlEncode(key),
-                    StringHelper.UrlEncode(fops));
+                sb.AppendFormat("bucket={0}&key={1}", StringHelper.UrlEncode(bucket), StringHelper.UrlEncode(key));
+                if (!string.IsNullOrEmpty(fops))
+                {
+                    sb.AppendFormat("&fops={0}", StringHelper.UrlEncode(fops));
+                }
+
+                if (!string.IsNullOrEmpty(workflowTemplateId))
+                {
+                    sb.AppendFormat("&workflowTemplateID={0}", workflowTemplateId);
+                }
                 if (!string.IsNullOrEmpty(notifyUrl))
                 {
                     sb.AppendFormat("&notifyURL={0}", StringHelper.UrlEncode(notifyUrl));

src/Qiniu/Storage/PfopInfo.cs

@@ -53,6 +53,11 @@ public class PfopInfo
         [JsonProperty("reqid")]
         public string Reqid;
         /// <summary>
+        /// 任务来源
+        /// </summary>
+        [JsonProperty("taskFrom")]
+        public string TaskFrom;
+        /// <summary>
         /// 数据处理的命令集合
         /// </summary>
         [JsonProperty("items")]

src/Qiniu/Storage/PutPolicy.cs

@@ -94,7 +94,7 @@ public class PutPolicy
         public int? CallbackFetchKey { get; set; }
 
         /// <summary>
-        /// [可选]上传预转持久化
+        /// [可选]上传预转持久化，与 PersistentWorkflowTemplateId 二选一
         /// </summary>
         [JsonProperty("persistentOps", NullValueHandling = NullValueHandling.Ignore)]
         public string PersistentOps { get; set; }
@@ -116,6 +116,12 @@ public class PutPolicy
         /// </summary>
         [JsonProperty("persistentType", NullValueHandling = NullValueHandling.Ignore)]
         public int? PersistentType { get; set; }
+
+        /// <summary>
+        /// [可选]任务模版，与 PersistentOps 二选一
+        /// </summary>
+        [JsonProperty("persistentWorkflowTemplateID", NullValueHandling = NullValueHandling.Ignore)]
+        public string PersistentWorkflowTemplateId { get; set; }
 
 
         /// <summary>

src/QiniuTests/Storage/FormUploaderTests.cs

@@ -2,6 +2,7 @@
 using Qiniu.Http;
 using System;
 using System.Collections.Generic;
+using System.Text;
 using Newtonsoft.Json;
 using Qiniu.Util;
 using Qiniu.Tests;
@@ -82,13 +83,14 @@ public void UploadFileV2Test()
             System.IO.File.Delete(filePath);
         }
 
-        [Test]
-        public void UploadFileWithPersistTypeTest()
+        [TestCaseSource(typeof(OperationManagerTests), nameof(OperationManagerTests.PfopOptionsTestCases))]
+        public void UploadFileWithPersistOptionsTest(int type, string workflowId)
         {
             Mac mac = new Mac(AccessKey, SecretKey);
-            Random rand = new Random();
-            string key = string.Format("UploadFileTest_{0}.dat", rand.Next());
+            string bucketName = Bucket;
+            string key = "test-pfop/upload-file";
 
+            // generate file to upload
             string tempPath = System.IO.Path.GetTempPath();
             int rnd = new Random().Next(1, 100000);
             string filePath = tempPath + "resumeFile" + rnd.ToString();
@@ -99,37 +101,70 @@ public void UploadFileWithPersistTypeTest()
             sw.Close();
             stream.Close();
 
+            // generate put policy
             PutPolicy putPolicy = new PutPolicy();
-            putPolicy.Scope = Bucket + ":" + key;
+            putPolicy.Scope = string.Join(":", bucketName, key);
             putPolicy.SetExpires(3600);
             putPolicy.DeleteAfterDays = 1;
-            string saveEntry = Base64.UrlSafeBase64Encode(Bucket + ":pfop-test_avinfo");
-            putPolicy.PersistentOps = "avinfo|saveas/" + saveEntry;
-            putPolicy.PersistentType = 1;
-            string token = Auth.CreateUploadToken(mac, putPolicy.ToJsonString());
+
+            StringBuilder persistentKeyBuilder = new StringBuilder("test-pfop/test-pfop-by-upload");
+            if (type > 0)
+            {
+                persistentKeyBuilder.Append("type_" + type);
+                putPolicy.PersistentType = type;
+            }
+
+            if (!string.IsNullOrEmpty(workflowId))
+            {
+                putPolicy.PersistentWorkflowTemplateId = workflowId;
+            }
+            else
+            {
+                string saveEntry = Base64.UrlSafeBase64Encode(String.Join(
+                    ":",
+                    bucketName,
+                    persistentKeyBuilder.ToString()
+                ));
+                putPolicy.PersistentOps = "avinfo|saveas/" + saveEntry;
+            }
+
+            // upload
             Config config = new Config();
-            config.Zone = Zone.ZONE_CN_East;
             config.UseHttps = true;
             config.UseCdnDomains = true;
-            FormUploader target = new FormUploader(config);
-            PutExtra extra = new PutExtra();
-            extra.Version = "v2";
-            HttpResult result = target.UploadFile(filePath, key, token, extra);
+            string token = Auth.CreateUploadToken(mac, putPolicy.ToJsonString());
+            FormUploader uploader = new FormUploader(config);
+            HttpResult result = uploader.UploadFile(filePath, key, token, null);
             Console.WriteLine("form upload result: " + result.ToString());
             Assert.AreEqual((int)HttpCode.OK, result.Code);
             System.IO.File.Delete(filePath);
 
+            // get persist task info
             Dictionary<string, object> dict = JsonConvert.DeserializeObject<Dictionary<string, object>>(result.Text.ToString());
             Assert.IsTrue(dict.ContainsKey("persistentId"));
             OperationManager manager = new OperationManager(mac, config);
             PrefopResult prefopRet = manager.Prefop(dict["persistentId"].ToString());
+
+            // assert the result
             if (prefopRet.Code != (int)HttpCode.OK)
             {
                 Assert.Fail("prefop error: " + prefopRet.ToString());
             }
-            Assert.AreEqual(1, prefopRet.Result.Type);
+
             Assert.IsNotNull(prefopRet.Result.CreationDate);
             Assert.IsNotEmpty(prefopRet.Result.CreationDate);
+
+            if (type == 1)
+            {
+                Assert.AreEqual(1, prefopRet.Result.Type);
+            }
+
+            if (!string.IsNullOrEmpty(workflowId))
+            {
+                Assert.IsNotNull(prefopRet.Result.TaskFrom);
+                Assert.IsNotEmpty(prefopRet.Result.TaskFrom);
+                Assert.IsTrue(prefopRet.Result.TaskFrom.Contains(workflowId));
+            }
         }
     }
 }

src/QiniuTests/Storage/OperationManagerTests.cs

@@ -1,21 +1,21 @@
 using NUnit.Framework;
 using System;
+using System.Collections;
 using System.Text;
 using Qiniu.Util;
 using Qiniu.Http;
-using Qiniu.Storage;
 using Qiniu.Tests;
 
 namespace Qiniu.Storage.Tests
 {
     [TestFixture]
-    public class OperationManagerTests :TestEnv
+    public class OperationManagerTests : TestEnv
     {
         private OperationManager getOperationManager()
         {
             Mac mac = new Mac(AccessKey, SecretKey);
             Config config = new Config();
-            // config.UseHttps = true;
+            config.UseHttps = true;
 
             OperationManager manager = new OperationManager(mac, config);
             return manager;
@@ -50,31 +50,88 @@ public void PfopAndPrefopTest()
             Console.WriteLine(ret.ToString());
         }
 
-        [Test]
-        public void PfopWithIdleTimeTest()
+        public static IEnumerable PfopOptionsTestCases
         {
+            get
+            {
+                yield return new TestCaseData(
+                    0, // type
+                    null // workflow template id
+                );
+                yield return new TestCaseData(
+                    1,
+                    null
+                );
+                yield return new TestCaseData(
+                    0,
+                    "test-workflow"
+                );
+            }
+        }
+
+        [TestCaseSource(typeof(OperationManagerTests), nameof(PfopOptionsTestCases))]
+        public void PfopWithOptionsTest(int type, string workflowId)
+        {
+            string bucketName = Bucket;
             string key = "qiniu.mp4";
-            bool force = true;
-            int type = 1;
-            string pipeline = null;
-            string saveJpgEntry = Base64.UrlSafeBase64Encode(Bucket + ":vframe_test_target.jpg");
-            string vframeJpgFop = "vframe/jpg/offset/1|saveas/" + saveJpgEntry;
+
+            StringBuilder persistentKeyBuilder = new StringBuilder("test-pfop/test-pfop-by-api");
+            if (type > 0)
+            {
+                persistentKeyBuilder.Append("type_" + type);
+            }
+
+            string fops;
+            if (!string.IsNullOrEmpty(workflowId))
+            {
+                fops = null;
+            }
+            else
+            {
+                string saveEntry = Base64.UrlSafeBase64Encode(String.Join(
+                    ":",
+                    bucketName,
+                    persistentKeyBuilder.ToString()
+                ));
+                fops = "avinfo|saveas/" + saveEntry;
+            }
 
             OperationManager manager = getOperationManager();
-            PfopResult pfopRet = manager.Pfop(Bucket, key, vframeJpgFop, pipeline, null, force, type);
+            PfopResult pfopRet = manager.Pfop(
+                Bucket,
+                key,
+                fops,
+                null,
+                null,
+                true,
+                type,
+                workflowId
+            );
             if (pfopRet.Code != (int)HttpCode.OK)
             {
-                Assert.Fail("pfop error: " + pfopRet.ToString());
+                Assert.Fail("pfop error: " + pfopRet);
             }
 
             PrefopResult prefopRet = manager.Prefop(pfopRet.PersistentId);
             if (prefopRet.Code != (int)HttpCode.OK)
             {
-                Assert.Fail("prefop error: " + prefopRet.ToString());
+                Assert.Fail("prefop error: " + prefopRet);
             }
-            Assert.AreEqual(1, prefopRet.Result.Type);
+
             Assert.IsNotNull(prefopRet.Result.CreationDate);
             Assert.IsNotEmpty(prefopRet.Result.CreationDate);
+
+            if (type == 1)
+            {
+                Assert.AreEqual(1, prefopRet.Result.Type);
+            }
+
+            if (!string.IsNullOrEmpty(workflowId))
+            {
+                Assert.IsNotNull(prefopRet.Result.TaskFrom);
+                Assert.IsNotEmpty(prefopRet.Result.TaskFrom);
+                Assert.IsTrue(prefopRet.Result.TaskFrom.Contains(workflowId));
+            }
         }
     }
 }