add metadata.title to file data connector

mendableai · Apr 3, 2024 · 607698e · 607698e
1 parent eeaca38
commit 607698e
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 10 deletions.
diff --git a/src/__tests__/providers/File/index.test.ts b/src/__tests__/providers/File/index.test.ts
@@ -25,33 +25,33 @@ describe("FileDataProvider", () => {
       {
         content:
           "id, column1, column2, column3\n1, test, 11111, test test\n2, test2 test2, 22222, test\n3, test3, 33333, test test test",
-        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/) },
+        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/), title: "test.csv" },
         provider: "file",
         type: "csv",
       },
       {
         content:
           "# This is a test markdown file\n\nThis file is used for testing purposes. Below is a list of items:\n\n- Item 1\n- Item 2\n- Item 3\n\nEnd of file.\n",
-        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/) },
+        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/), title: "test.md" },
         provider: "file",
         type: "md",
       },
       {
         content: expect.stringContaining("Dummy PDF file"),
-        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/) },
+        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/), title: "test.pdf" },
         provider: "file",
         type: "pdf",
       },
       {
         content: "This is a test file.\n",
-        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/) },
+        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/), title: "test.txt" },
         provider: "file",
         type: "txt",
       },
       {
         content:
           '<?xml version="1.0" encoding="UTF-8"?>\n<tests>\n  <test>\n    <id>1</id>\n    <column1>test</column1>\n    <column2>11111</column2>\n    <column3>test test</column3>\n  </test>\n  <test>\n    <id>2</id>\n    <column1>test2 test2</column1>\n    <column2>22222</column2>\n    <column3>test</column3>\n  </test>\n  <test>\n    <id>3</id>\n    <column1>test3</column1>\n    <column2>33333</column2>\n    <column3>test test test</column3>\n  </test>\n</tests>\n',
-        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/) },
+        metadata: { sourceURL: expect.stringMatching(/^#FILE_\d+$/), title: "test.xml" },
         provider: "file",
         type: "xml",
       },
@@ -83,35 +83,35 @@ describe("FileDataProvider", () => {
     expect(documentsByURL).toContainEqual({
       content:
         "id, column1, column2, column3\n1, test, 11111, test test\n2, test2 test2, 22222, test\n3, test3, 33333, test test test\n",
-      metadata: { sourceURL: optionsURLs.urls[0] },
+      metadata: { sourceURL: optionsURLs.urls[0], title: "test.csv" },
       provider: "file",
       type: "csv",
     });
     expect(documentsByURL).toContainEqual({
       content: expect.stringContaining(
         "# This is a test markdown file\n\nThis file is used for testing purposes. Below is a list of items:\n\n- Item 1\n- Item 2\n- Item 3\n\nEnd of file.\n"
       ),
-      metadata: { sourceURL: optionsURLs.urls[1] },
+      metadata: { sourceURL: optionsURLs.urls[1], title: "test.md" },
       provider: "file",
       type: "md",
     });
     expect(documentsByURL).toContainEqual({
       content: expect.stringContaining("Dummy PDF file"),
-      metadata: { sourceURL: optionsURLs.urls[2] },
+      metadata: { sourceURL: optionsURLs.urls[2], title: "test%20%281%29.pdf" },
       provider: "file",
       type: "pdf",
     });
     expect(documentsByURL).toContainEqual({
       content: expect.stringContaining("This is a test file."),
-      metadata: { sourceURL: optionsURLs.urls[3] },
+      metadata: { sourceURL: optionsURLs.urls[3], title: "test.txt" },
       provider: "file",
       type: "txt",
     });
     expect(documentsByURL).toContainEqual({
       content: expect.stringContaining(
         '<?xml version="1.0" encoding="UTF-8"?>\n<tests>\n  <test>\n    <id>1</id>\n    <column1>test</column1>\n    <column2>11111</column2>\n    <column3>test test</column3>\n  </test>\n  <test>\n    <id>2</id>\n    <column1>test2 test2</column1>\n    <column2>22222</column2>\n    <column3>test</column3>\n  </test>\n  <test>\n    <id>3</id>\n    <column1>test3</column1>\n    <column2>33333</column2>\n    <column3>test test test</column3>\n  </test>\n</tests>'
       ),
-      metadata: { sourceURL: optionsURLs.urls[4] },
+      metadata: { sourceURL: optionsURLs.urls[4], title: "test.xml" },
       provider: "file",
       type: "xml",
     });

diff --git a/src/providers/File/index.ts b/src/providers/File/index.ts
@@ -65,6 +65,7 @@ export class FileDataProvider implements DataProvider<FileInputOptions> {
           content,
           metadata: {
             sourceURL: "#FILE_" + randomNumber.toString(),
+            title: this.files[i].includes('/') ? this.files[i].split('/').pop() : this.files[i],
           },
           provider: "file",
           type: fileType,
@@ -108,6 +109,7 @@ export class FileDataProvider implements DataProvider<FileInputOptions> {
           content,
           metadata: {
             sourceURL: this.urls[i],
+            title: this.urls[i].includes('/') ? this.urls[i].split('/').pop() : this.urls[i],
           },
           provider: "file",
           type: fileType,