Merge pull request #17 from dcSpark/nico/add-file-uploaded

add file uploader manager n stuff

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shinkai_protocol/shinkai-typescript-lib",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Typescript library to build and handle Shinkai Messages",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -40,6 +40,7 @@
     "@noble/hashes": "^1.3.2",
     "@peculiar/webcrypto": "^1.4.3",
     "curve25519-js": "^0.0.4",
+    "form-data": "^4.0.0",
     "libsodium-wrappers-sumo": "^0.7.13",
     "noble-ed25519": "^1.2.6",
     "tweetnacl": "^1.0.3"

src/index.ts

@@ -18,3 +18,4 @@ export * from './shinkai_message/shinkai_message';
 export * from './shinkai_message/shinkai_version';
 
 export * from './shinkai_message_builder/shinkai_message_builder';
+export * from './utils/FileUploaderManager';

src/schemas/inbox_name.ts

@@ -136,6 +136,22 @@ export class InboxName {
   getValue(): string {
     return this.value;
   }
+
+  getUniqueId(): string {
+    const parts: string[] = this.value.split("::");
+    // Ensure there are at least 3 parts for a valid inbox name
+    if (parts.length < 3) {
+      throw new InboxNameError(`Invalid inbox name format: ${this.value}`);
+    }
+
+    // Remove the first part (inbox type) and the last part (boolean value)
+    const idParts = parts.slice(1, parts.length - 1);
+
+    // Rejoin the remaining parts to form SOME_ID, which may contain '::'
+    const someId = idParts.join("::");
+
+    return someId;
+  }
 }
 
 export class RegularInbox extends InboxName {

src/shinkai_message_builder/shinkai_message_builder.ts

@@ -796,10 +796,9 @@ export class ShinkaiMessageBuilder {
     my_subidentity_encryption_sk: EncryptionStaticKey,
     my_subidentity_signature_sk: SignatureStaticKey,
     receiver_public_key: EncryptionPublicKey,
-    inbox: string,
     symmetric_key_sk: string,
-    sender_subidentity: string,
     sender: ProfileName,
+    sender_subidentity: string,
     receiver: ProfileName
   ): Promise<ShinkaiMessage> {
     return new ShinkaiMessageBuilder(
@@ -812,7 +811,7 @@ export class ShinkaiMessageBuilder {
       .set_internal_metadata_with_schema(
         sender_subidentity,
         "",
-        inbox,
+        "",
         MessageSchemaType.SymmetricKeyExchange,
         TSEncryptionMethod.None
       )
@@ -1110,7 +1109,7 @@ export class ShinkaiMessageBuilder {
     receiver: ProfileName,
     receiver_subidentity: string
   ): Promise<ShinkaiMessage> {
-    const createItemsInfo = { destination_path, file_inbox };
+    const createItemsInfo = { path: destination_path, file_inbox };
     return ShinkaiMessageBuilder.createCustomShinkaiMessageToNode(
       my_encryption_secret_key,
       my_signature_secret_key,

src/utils/FileUploaderInterfaces.ts

@@ -0,0 +1,24 @@
+// HTTP Service Interface
+export interface IHttpService {
+    fetch(url: string, options: any): Promise<Response>;
+  }
+
+  // Crypto Service Interface
+  export interface ICryptoService {
+    getRandomValues(buffer: Uint8Array): Uint8Array;
+    subtle: {
+      exportKey(format: string, key: CryptoKey): Promise<ArrayBuffer>;
+      importKey(
+        format: string,
+        keyData: BufferSource,
+        algorithm: string | Algorithm,
+        extractable: boolean,
+        keyUsages: KeyUsage[]
+      ): Promise<CryptoKey>;
+      encrypt(
+        algorithm: AlgorithmIdentifier,
+        key: CryptoKey,
+        data: BufferSource
+      ): Promise<ArrayBuffer>;
+    };
+  }

src/utils/FileUploaderManager.ts

@@ -0,0 +1,229 @@
+import { blake3 } from "@noble/hashes/blake3";
+import { urlJoin } from "../utils/url-join";
+import { InboxName } from "../schemas/inbox_name";
+import { hexToBytes } from "../cryptography/crypto_utils";
+import { ShinkaiMessageBuilder } from "../shinkai_message_builder/shinkai_message_builder";
+import { ICryptoService, IHttpService } from "./FileUploaderInterfaces";
+
+// Conditional FormData implementation
+interface IFormData {
+  append(name: string, value: any, fileName?: string): void;
+}
+
+// Conditional FormData implementation
+let FormDataImplementation: { new (): IFormData };
+if (typeof window === "undefined") {
+  // We are in a Node.js environment
+  FormDataImplementation = require("form-data") as unknown as {
+    new (): IFormData;
+  };
+} else {
+  // We are in a web browser environment
+  FormDataImplementation = FormData as unknown as { new (): IFormData };
+}
+
+function createFormData(): IFormData {
+  return new FormDataImplementation();
+}
+
+function appendFile(
+  formData: IFormData,
+  fieldName: string,
+  file: any,
+  fileName: string
+) {
+  // Convert ArrayBuffer to Buffer in Node.js environment
+  if (typeof window === "undefined" && file instanceof ArrayBuffer) {
+    file = Buffer.from(file);
+  }
+  formData.append(fieldName, file, fileName);
+}
+
+export class FileUploader {
+  private httpService: IHttpService;
+  private cryptoService: ICryptoService;
+
+  private base_url: string;
+  private my_encryption_secret_key: string;
+  private my_signature_secret_key: string;
+  private receiver_public_key: string;
+  private sender: string;
+  private sender_subidentity: string;
+  private receiver: string;
+  private symmetric_key: CryptoKey | null;
+  private folder_id: string | null;
+
+  constructor(
+    httpService: IHttpService,
+    cryptoService: ICryptoService,
+    base_url: string,
+    my_encryption_secret_key: string,
+    my_signature_secret_key: string,
+    receiver_public_key: string,
+    sender: string,
+    sender_subidentity: string,
+    receiver: string
+  ) {
+    this.httpService = httpService;
+    this.cryptoService = cryptoService;
+    this.base_url = base_url;
+    this.my_encryption_secret_key = my_encryption_secret_key;
+    this.my_signature_secret_key = my_signature_secret_key;
+    this.receiver_public_key = receiver_public_key;
+
+    this.sender = sender;
+    this.sender_subidentity = sender_subidentity;
+    this.receiver = receiver;
+    this.symmetric_key = null;
+    this.folder_id = null;
+  }
+
+  async calculateHashFromSymmetricKey(): Promise<string> {
+    if (!this.symmetric_key) {
+      throw new Error("Symmetric key is not set");
+    }
+
+    const rawKey = await this.cryptoService.subtle.exportKey(
+      "raw",
+      this.symmetric_key
+    );
+    const rawKeyArray = new Uint8Array(rawKey);
+    const keyHexString = Array.from(rawKeyArray)
+      .map((b) => b.toString(16).padStart(2, "0"))
+      .join("");
+
+    const hash = blake3(keyHexString);
+    const hashHex = Array.from(hash)
+      .map((b) => b.toString(16).padStart(2, "0"))
+      .join("");
+
+    return hashHex;
+  }
+
+  async generateAndUpdateSymmetricKey(): Promise<void> {
+    const keyData = this.cryptoService.getRandomValues(new Uint8Array(32));
+    this.symmetric_key = await this.cryptoService.subtle.importKey(
+      "raw",
+      keyData,
+      "AES-GCM",
+      true,
+      ["encrypt", "decrypt"]
+    );
+  }
+
+  async createFolder(): Promise<string> {
+    try {
+      const keyData = this.cryptoService.getRandomValues(new Uint8Array(32));
+      this.symmetric_key = await this.cryptoService.subtle.importKey(
+        "raw",
+        keyData,
+        "AES-GCM",
+        true,
+        ["encrypt", "decrypt"]
+      );
+
+      const exportedKey = await this.cryptoService.subtle.exportKey(
+        "raw",
+        this.symmetric_key
+      );
+      const exportedKeyArray = new Uint8Array(exportedKey);
+      const exportedKeyString = Array.from(exportedKeyArray)
+        .map((b) => b.toString(16).padStart(2, "0"))
+        .join("");
+
+      const hash = await this.calculateHashFromSymmetricKey();
+      const message = await ShinkaiMessageBuilder.createFilesInboxWithSymKey(
+        hexToBytes(this.my_encryption_secret_key),
+        hexToBytes(this.my_signature_secret_key),
+        hexToBytes(this.receiver_public_key),
+        exportedKeyString,
+        this.sender,
+        this.sender_subidentity,
+        this.receiver
+      );
+
+      const response = await this.httpService.fetch(
+        urlJoin(this.base_url, "/v1/create_files_inbox_with_symmetric_key"),
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(message),
+        }
+      );
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+      this.folder_id = hash;
+      return this.folder_id;
+    } catch (error) {
+      console.error("Error creating folder:", error);
+      throw error;
+    }
+  }
+
+  async uploadEncryptedFileWeb(file: File, filename?: string): Promise<void> {
+    const fileData = await file.arrayBuffer();
+    return this.uploadEncryptedData(fileData, filename || file.name);
+  }
+
+  // Method for uploading files in a Node.js environment
+  async uploadEncryptedFileNode(
+    buffer: Buffer,
+    filename: string
+  ): Promise<void> {
+    return this.uploadEncryptedData(buffer, filename);
+  }
+
+  private async uploadEncryptedData(
+    data: ArrayBuffer | Buffer,
+    filename: string
+  ): Promise<void> {
+    if (!this.symmetric_key) {
+      throw new Error("Symmetric key is not set");
+    }
+
+    // Generate the initialization vector (iv) here
+    const iv = this.cryptoService.getRandomValues(new Uint8Array(12));
+    const algorithm = { name: "AES-GCM", iv };
+
+    // Perform encryption
+    const encryptedFileData = await this.cryptoService.subtle.encrypt(
+      algorithm,
+      this.symmetric_key, // symmetric_key is guaranteed to be non-null here
+      data
+    );
+
+    const hash = await this.calculateHashFromSymmetricKey();
+    const nonce = Array.from(iv)
+      .map((b) => b.toString(16).padStart(2, "0"))
+      .join("");
+
+    const formData = createFormData();
+    // Adjust for environment differences
+    let fileData;
+    if (typeof window === "undefined") {
+      // In Node.js, directly use Buffer
+      fileData = Buffer.from(encryptedFileData);
+    } else {
+      // In the browser, use Blob
+      fileData = new Blob([encryptedFileData]);
+    }
+    appendFile(formData, "file", fileData, filename);
+
+    await this.httpService.fetch(
+      urlJoin(
+        this.base_url,
+        "/v1/add_file_to_inbox_with_symmetric_key",
+        hash,
+        nonce
+      ),
+      {
+        method: "POST",
+        body: formData,
+      }
+    );
+  }
+}

src/utils/url-join.ts

@@ -0,0 +1,6 @@
+// It safe join url chunks avoiding double '/' between paths
+// Warning: It doesn't supports all cases but it's enough for join shinkai-node api urls
+export const urlJoin = (...chunks: string[]): string => {
+    return chunks.map(chunk => chunk.replace(/(^\/+|\/+$)/mg, '')).filter(chunk => !!chunk).join('/')
+  };
+

tests/inbox_name.test.ts

@@ -1,4 +1,4 @@
-import { InboxName } from "../src/schemas/inbox_name";
+import { InboxName, InboxNameError } from "../src/schemas/inbox_name";
 
 describe("InboxName", () => {
   test("valid_inbox_names", () => {
@@ -36,5 +36,30 @@ describe("InboxName", () => {
     }
   });
 
-  // Add other tests here...
+  describe("InboxName getId method", () => {
+    it("extracts SOME_ID correctly for simple inbox format", () => {
+      const inbox = new InboxName("inbox::simpleId::true", false);
+      expect(inbox.getUniqueId()).toBe("simpleId");
+    });
+
+    it("extracts SOME_ID correctly for inbox format with separator in ID", () => {
+      const inbox = new InboxName("inbox::complex::Id::true", false);
+      expect(inbox.getUniqueId()).toBe("complex::Id");
+    });
+
+    it("extracts SOME_ID correctly for simple job_inbox format", () => {
+      const jobInbox = new InboxName("job_inbox::uniqueId::false", false);
+      expect(jobInbox.getUniqueId()).toBe("uniqueId");
+    });
+
+    it("extracts SOME_ID correctly for job_inbox format with separator in ID", () => {
+      const jobInbox = new InboxName("job_inbox::unique::Id::false", false);
+      expect(jobInbox.getUniqueId()).toBe("unique::Id");
+    });
+
+    it("throws an error for invalid inbox name format", () => {
+      const invalidInbox = new InboxName("invalidFormat", false);
+      expect(() => invalidInbox.getUniqueId()).toThrow(InboxNameError);
+    });
+  });
 });

tests/shinkai_messsage_builder.test.ts

@@ -760,17 +760,15 @@ describe("ShinkaiMessageBuilder pre-made methods", () => {
     const sender = "@@sender.shinkai";
     const receiver = "@@receiver.shinkai";
     const sender_subidentity = "sender_subidentity";
-    const inbox = "inbox_name";
     const symmetric_key_sk = "symmetric_key";
 
     const message = await ShinkaiMessageBuilder.createFilesInboxWithSymKey(
       my_subidentity_encryption_sk,
       my_subidentity_signature_sk,
       receiver_public_key,
-      inbox,
       symmetric_key_sk,
-      sender_subidentity,
       sender,
+      sender_subidentity,
       receiver
     );