feat: directory uploading (#208)

* feat: directory uploading

* feat: added manual tests for directory uploading

* refactor: general improvements

* refactor: general improvements

.babelrc.js

@@ -1,7 +1,7 @@
 'use strict'
 
 module.exports = function (api) {
-  const targets = '>1% or node >=10 and not ie 11 and not dead'
+  const targets = '>1% or node >=14 and not ie 11 and not dead'
   api.cache(true)
   api.cacheDirectory = true
 

README.md

@@ -182,6 +182,30 @@ Deleting a directory
 await fdp.directory.delete('my-new-pod', '/my-dir')
 ```
 
+Upload an entire directory with files in Node.js
+
+```js
+// `recursively: false` will upload only files in passed directory
+// `recursively: true` will find all files recursively in nested directories and upload them to the network
+await fdp.directory.upload('my-new-pod', '/Users/fdp/MY_LOCAL_DIRECTORY', { isRecursive: true })
+```
+
+Upload an entire directory with files in browser.
+
+Create input element with `webkitdirectory` property. With this property entire directory can be chosen instead of a file.
+
+```html
+<input type="file" id="upload-directory" webkitdirectory/>
+```
+
+```js
+// getting list of files in a directory
+const files = document.getElementById('upload-directory').files
+// `recursively: false` will upload only files in passed directory
+// `recursively: true` will find all files recursively in nested directories and upload them to the network
+await fdp.directory.upload('my-new-pod', files, { isRecursive: true })
+```
+
 Uploading data as a file into a pod
 
 ```js

package.json

@@ -87,6 +87,7 @@
     "axios": "^0.27.2",
     "babel-jest": "^27.0.5",
     "babel-loader": "^8.2.2",
+    "bootstrap": "^5.2.3",
     "debug": "^4.3.1",
     "depcheck": "^1.4.0",
     "eslint": "^7.29.0",

src/content-items/handler.ts

@@ -5,12 +5,18 @@ import { getUnixTimestamp } from '../utils/time'
 import { writeFeedData } from '../feed/api'
 import { getRawDirectoryMetadataBytes } from '../directory/adapter'
 import { DIRECTORY_TOKEN, FILE_TOKEN } from '../file/handler'
-import { assertRawDirectoryMetadata, combine } from '../directory/utils'
+import { assertRawDirectoryMetadata, combine, splitPath } from '../directory/utils'
 import { RawDirectoryMetadata } from '../pod/types'
 import { assertItemIsNotExists, getRawMetadata } from './utils'
 import { RawMetadataWithEpoch } from './types'
 import { prepareEthAddress } from '../utils/wallet'
 import { PodPasswordBytes } from '../utils/encryption'
+import { DataUploadOptions } from '../file/types'
+
+export const DEFAULT_UPLOAD_OPTIONS: DataUploadOptions = {
+  blockSize: 1000000,
+  contentType: '',
+}
 
 /**
  * Add child file or directory to a defined parent directory
@@ -42,7 +48,7 @@ export async function addEntryToDirectory(
 
   const address = prepareEthAddress(wallet.address)
   const itemText = isFile ? 'File' : 'Directory'
-  const fullPath = combine(parentPath, entryPath)
+  const fullPath = combine(...splitPath(parentPath), entryPath)
   await assertItemIsNotExists(itemText, connection.bee, fullPath, address, downloadOptions)
 
   let parentData: RawDirectoryMetadata | undefined

src/directory/directory.ts

@@ -1,10 +1,25 @@
 import { AccountData } from '../account/account-data'
-import { createDirectory, readDirectory } from './handler'
+import { createDirectory, readDirectory, DEFAULT_UPLOAD_DIRECTORY_OPTIONS, UploadDirectoryOptions } from './handler'
 import { assertAccount } from '../account/utils'
 import { DirectoryItem } from '../content-items/directory-item'
 import { removeEntryFromDirectory } from '../content-items/handler'
-import { extractPathInfo } from '../file/utils'
+import { extractPathInfo, readBrowserFileAsBytes } from '../file/utils'
 import { assertPodName, getExtendedPodsListByAccountData } from '../pod/utils'
+import { isNode } from '../shim/utils'
+import {
+  assertBrowserFilesWithPath,
+  filterBrowserRecursiveFiles,
+  filterDotFiles,
+  browserFileListToFileInfoList,
+  getDirectoriesToCreate,
+  getNodeFileContent,
+  getNodeFileInfoList,
+  getUploadPath,
+  BrowserFileInfo,
+  NodeFileInfo,
+} from './utils'
+import { uploadData } from '../file/handler'
+import { assertNodeFileInfo, isBrowserFileInfo } from './types'
 
 /**
  * Directory related class
@@ -73,4 +88,73 @@ export class Directory {
       downloadOptions,
     )
   }
+
+  /**
+   * Uploads a directory with files
+   *
+   * @param podName pod where to upload a directory
+   * @param filesSource files source. path for Node.js, `FileList` for browser
+   * @param options upload directory options
+   */
+  async upload(podName: string, filesSource: string | FileList, options?: UploadDirectoryOptions): Promise<void> {
+    assertAccount(this.accountData)
+    assertPodName(podName)
+    const downloadOptions = this.accountData.connection.options?.downloadOptions
+    const { podWallet, pod } = await getExtendedPodsListByAccountData(this.accountData, podName)
+    options = { ...DEFAULT_UPLOAD_DIRECTORY_OPTIONS, ...options }
+
+    const isNodePath = typeof filesSource === 'string'
+    const isNodeEnv = isNode()
+
+    if (!isNodeEnv && isNodePath) {
+      throw new Error('Directory uploading with path as string is available in Node.js only')
+    }
+
+    let files: (BrowserFileInfo | NodeFileInfo)[]
+
+    if (isNodePath) {
+      files = await getNodeFileInfoList(filesSource, Boolean(options.isRecursive))
+    } else {
+      assertBrowserFilesWithPath(filesSource)
+      files = browserFileListToFileInfoList(filesSource)
+
+      if (!options.isRecursive) {
+        files = filterBrowserRecursiveFiles(files as BrowserFileInfo[])
+      }
+    }
+
+    if (options.excludeDotFiles) {
+      files = filterDotFiles(files)
+    }
+    const directoriesToCreate = getDirectoriesToCreate(
+      files.map(item => (options?.isIncludeDirectoryName ? item.relativePathWithBase : item.relativePath)),
+    )
+    for (const directory of directoriesToCreate) {
+      try {
+        await createDirectory(this.accountData.connection, directory, podWallet, pod.password, downloadOptions)
+      } catch (e) {
+        const error = e as Error
+
+        if (!error.message.includes('already listed in the parent directory list')) {
+          throw new Error(error.message)
+        }
+      }
+    }
+
+    for (const file of files) {
+      let bytes
+
+      if (isNodePath) {
+        assertNodeFileInfo(file)
+        bytes = getNodeFileContent(file.fullPath)
+      } else if (!isNodePath && isBrowserFileInfo(file)) {
+        bytes = await readBrowserFileAsBytes(file.browserFile)
+      } else {
+        throw new Error('Directory uploading: one of the files is not correct')
+      }
+
+      const uploadPath = getUploadPath(file, options.isIncludeDirectoryName!)
+      await uploadData(podName, uploadPath, bytes, this.accountData, options.uploadOptions!)
+    }
+  }
 }