feat: data uploading progress (#266)

* feat: data uploading progress

* fix: moved testpage

* fix: path for testpage

* fix: path for test data

README.md

@@ -214,6 +214,14 @@ Uploading data as a file into a pod
 
 ```js
 await fdp.file.uploadData('my-new-pod', '/my-dir/myfile.txt', 'Hello world!')
+
+// you can also track the progress of data upload
+// using the callback, you can track not only the progress of uploaded blocks but also other time-consuming operations required for data upload
+await fdp.file.uploadData('my-new-pod', '/my-dir/myfile.txt', 'Hello world!', {
+  progressCallback: event => {
+    console.log(event)
+  }
+})
 ```
 
 Deleting a file from a pod
@@ -380,7 +388,7 @@ npm run test:unit
 npm run test:browser
 ```
 
-The test HTML file which Puppeteer uses is the [test/testpage/testpage.html](test/testpage/testpage.html).
+The test HTML file which Puppeteer uses is the [test/integration/testpage/testpage.html](test/integration/testpage/testpage.html).
 To open and manually test FDP with developer console, it is necessary to build the library first with `npm run compile:browser` (running the browser tests `npm run test:browser` also builds the library).
 
 ### Compile code

src/file/handler.ts

@@ -3,21 +3,23 @@ import { Bee, Data, BeeRequestOptions } from '@ethersphere/bee-js'
 import { EthAddress } from '@ethersphere/bee-js/dist/types/utils/eth'
 import {
   assertFullPathWithName,
+  calcUploadBlockPercentage,
   DEFAULT_FILE_PERMISSIONS,
   downloadBlocksManifest,
   extractPathInfo,
   getFileMode,
+  updateUploadProgress,
   uploadBytes,
 } from './utils'
 import { FileMetadata } from '../pod/types'
 import { blocksToManifest, getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter'
 import { assertRawFileMetadata } from '../directory/utils'
 import { getCreationPathInfo, getRawMetadata } from '../content-items/utils'
 import { PodPasswordBytes } from '../utils/encryption'
-import { Blocks, DataUploadOptions } from './types'
+import { Blocks, DataUploadOptions, UploadProgressType } from './types'
 import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils'
 import { getUnixTimestamp } from '../utils/time'
-import { addEntryToDirectory } from '../content-items/handler'
+import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS } from '../content-items/handler'
 import { writeFeedData } from '../feed/api'
 import { AccountData } from '../account/account-data'
 import { prepareEthAddress } from '../utils/wallet'
@@ -124,10 +126,15 @@ export async function uploadData(
   assertPodName(podName)
   assertWallet(accountData.wallet)
 
+  const blockSize = options.blockSize ?? Number(DEFAULT_UPLOAD_OPTIONS!.blockSize)
+  const contentType = options.contentType ?? String(DEFAULT_UPLOAD_OPTIONS!.contentType)
+
   data = typeof data === 'string' ? stringToBytes(data) : data
   const connection = accountData.connection
+  updateUploadProgress(options, UploadProgressType.GetPodInfo)
   const { podWallet, pod } = await getExtendedPodsListByAccountData(accountData, podName)
 
+  updateUploadProgress(options, UploadProgressType.GetPathInfo)
   const fullPathInfo = await getCreationPathInfo(
     connection.bee,
     fullPath,
@@ -136,27 +143,38 @@ export async function uploadData(
   )
   const pathInfo = extractPathInfo(fullPath)
   const now = getUnixTimestamp()
-  const blocksCount = Math.ceil(data.length / options.blockSize)
+  const totalBlocks = Math.ceil(data.length / blockSize)
   const blocks: Blocks = { blocks: [] }
-  for (let i = 0; i < blocksCount; i++) {
-    const currentBlock = data.slice(i * options.blockSize, (i + 1) * options.blockSize)
+  for (let i = 0; i < totalBlocks; i++) {
+    updateUploadProgress(options, UploadProgressType.UploadBlockStart, {
+      totalBlocks,
+      currentBlockId: i,
+      uploadPercentage: calcUploadBlockPercentage(i, totalBlocks),
+    })
+    const currentBlock = data.slice(i * blockSize, (i + 1) * blockSize)
     const result = await uploadBytes(connection, currentBlock)
     blocks.blocks.push({
       size: currentBlock.length,
       compressedSize: currentBlock.length,
       reference: result.reference,
     })
+    updateUploadProgress(options, UploadProgressType.UploadBlockEnd, {
+      totalBlocks,
+      currentBlockId: i,
+      uploadPercentage: calcUploadBlockPercentage(i, totalBlocks),
+    })
   }
 
+  updateUploadProgress(options, UploadProgressType.UploadBlocksMeta)
   const manifestBytes = stringToBytes(blocksToManifest(blocks))
   const blocksReference = (await uploadBytes(connection, manifestBytes)).reference
   const meta: FileMetadata = {
     version: META_VERSION,
     filePath: pathInfo.path,
     fileName: pathInfo.filename,
     fileSize: data.length,
-    blockSize: options.blockSize,
-    contentType: options.contentType,
+    blockSize,
+    contentType,
     compression: '',
     creationTime: now,
     accessTime: now,
@@ -165,7 +183,9 @@ export async function uploadData(
     mode: getFileMode(DEFAULT_FILE_PERMISSIONS),
   }
 
+  updateUploadProgress(options, UploadProgressType.WriteDirectoryInfo)
   await addEntryToDirectory(connection, podWallet, pod.password, pathInfo.path, pathInfo.filename, true)
+  updateUploadProgress(options, UploadProgressType.WriteFileInfo)
   await writeFeedData(
     connection,
     fullPath,
@@ -175,5 +195,7 @@ export async function uploadData(
     getNextEpoch(fullPathInfo?.lookupAnswer.epoch),
   )
 
+  updateUploadProgress(options, UploadProgressType.Done)
+
   return meta
 }

src/file/types.ts

@@ -1,18 +1,93 @@
 import { Reference } from '@ethersphere/bee-js'
 import { RawFileMetadata } from '../pod/types'
 
+/**
+ * Uploading progress types
+ */
+export enum UploadProgressType {
+  /**
+   * Getting pod info
+   */
+  GetPodInfo = 'get-pod-info',
+  /**
+   * Getting path info
+   */
+  GetPathInfo = 'get-path-info',
+  /**
+   * Uploading file block start
+   */
+  UploadBlockStart = 'upload-block-start',
+  /**
+   * Uploading a file block end
+   */
+  UploadBlockEnd = 'upload-block-end',
+  /**
+   * Uploading file blocks meta
+   */
+  UploadBlocksMeta = 'upload-blocks-meta',
+  /**
+   * Writing parent directory info feed
+   */
+  WriteDirectoryInfo = 'update-directory-info',
+  /**
+   * Writing file info feed
+   */
+  WriteFileInfo = 'update-file-info',
+  /**
+   * Done
+   */
+  Done = 'done',
+}
+
+/**
+ * Uploading progress block data
+ */
+export interface UploadProgressBlockData {
+  /**
+   * Total number of blocks that will be uploaded
+   */
+  totalBlocks: number
+  /**
+   * ID of the currently processing block starting from 0
+   */
+  currentBlockId: number
+  /**
+   * Percentage of blocks uploaded
+   */
+  uploadPercentage: number
+}
+
+/**
+ * Upload progress info
+ */
+export interface UploadProgressInfo {
+  /**
+   * Type of the progress
+   */
+  progressType: UploadProgressType
+  /**
+   * Data of the progress
+   */
+  data?: UploadProgressBlockData
+}
+
 /**
  * File upload options
  */
 export interface DataUploadOptions {
   /**
    * Size of blocks in bytes will the file be divided
    */
-  blockSize: number
+  blockSize?: number
   /**
    * Content type of the file
    */
-  contentType: string
+  contentType?: string
+  /**
+   * Progress callback
+   * @param info progress info
+   */
+  progressCallback?: (info: UploadProgressInfo) => void
 }
 
 /**

src/file/utils.ts

@@ -1,7 +1,15 @@
 import { Connection } from '../connection/connection'
 import { Bee, Reference, BeeRequestOptions, UploadResult } from '@ethersphere/bee-js'
 import { PathInfo } from '../pod/utils'
-import { Blocks, FileShareInfo, RawBlock, RawBlocks } from './types'
+import {
+  Blocks,
+  DataUploadOptions,
+  FileShareInfo,
+  RawBlock,
+  RawBlocks,
+  UploadProgressBlockData,
+  UploadProgressType,
+} from './types'
 import { rawBlocksToBlocks } from './adapter'
 import CryptoJS from 'crypto-js'
 import { assertArray, assertString, isNumber, isObject, isString } from '../utils/type'
@@ -239,3 +247,34 @@ export async function readBrowserFileAsBytes(file: File): Promise<Uint8Array> {
 export function getFileMode(mode: number): number {
   return FILE_MODE | mode
 }
+
+/**
+ * Updates upload progress
+ * @param options upload options
+ * @param progressType progress type
+ * @param data progress data
+ */
+export function updateUploadProgress(
+  options: DataUploadOptions,
+  progressType: UploadProgressType,
+  data?: UploadProgressBlockData,
+): void {
+  if (!options.progressCallback) {
+    return
+  }
+
+  options.progressCallback({ progressType, data })
+}
+
+/**
+ * Calculates upload block percentage
+ * @param blockId block id started from 0
+ * @param totalBlocks total blocks
+ */
+export function calcUploadBlockPercentage(blockId: number, totalBlocks: number): number {
+  if (totalBlocks <= 0 || blockId < 0) {
+    return 0
+  }
+
+  return Math.round(((blockId + 1) / totalBlocks) * 100)
+}

src/types.ts

@@ -3,6 +3,7 @@ import { EnsEnvironment } from '@fairdatasociety/fdp-contracts-js'
 import { CacheOptions } from './cache/types'
 
 export { DirectoryItem, FileItem } from './content-items/types'
+export { UploadProgressType, UploadProgressBlockData, UploadProgressInfo, DataUploadOptions } from './file/types'
 
 /**
  * Fair Data Protocol options

test/integration/fdp-class.browser.spec.ts → test/integration/browser/fdp-class.browser.spec.ts

@@ -1,15 +1,15 @@
 import { join } from 'path'
-import { batchId, beeUrl, createFdp, fdpOptions, generateRandomHexString, generateUser, TestUser } from '../utils'
-import '../../src/index'
-import '../index'
+import { batchId, beeUrl, createFdp, fdpOptions, generateRandomHexString, generateUser, TestUser } from '../../utils'
+import '../../../src'
+import '../../index'
 import { JSONArray, JSONObject } from 'puppeteer'
-import { FdpStorage } from '../../src'
-import { MAX_POD_NAME_LENGTH } from '../../src/pod/utils'
-import { createUserV1 } from '../../src/account/account'
-import { Pod, PodShareInfo, RawFileMetadata } from '../../src/pod/types'
-import { FileShareInfo } from '../../src/file/types'
+import { FdpStorage } from '../../../src'
+import { MAX_POD_NAME_LENGTH } from '../../../src/pod/utils'
+import { createUserV1 } from '../../../src/account/account'
+import { Pod, PodShareInfo, RawFileMetadata } from '../../../src/pod/types'
+import { FileShareInfo } from '../../../src/file/types'
 import { BatchId, Reference } from '@ethersphere/bee-js'
-import { ETH_ADDR_HEX_LENGTH } from '../../src/utils/type'
+import { ETH_ADDR_HEX_LENGTH } from '../../../src/utils/type'
 
 jest.setTimeout(400000)
 describe('Fair Data Protocol class - in browser', () => {

test/integration/fdp-class.fairos.spec.ts → test/integration/fairos/fdp-class.fairos.spec.ts

@@ -1,12 +1,12 @@
-import { createFdp, generateRandomHexString, generateUser, topUpAddress, topUpFdp, waitFairOS } from '../utils'
-import { Directories, FairOSApi, PodsList } from '../utils/fairos-api'
+import { createFdp, generateRandomHexString, generateUser, topUpAddress, topUpFdp, waitFairOS } from '../../utils'
+import { Directories, FairOSApi, PodsList } from '../../utils/fairos-api'
 import { Wallet, utils } from 'ethers'
-import { wrapBytesWithHelpers } from '../../src/utils/bytes'
-import { getExtendedPodsListByAccountData } from '../../src/pod/utils'
-import { getRawMetadata } from '../../src/content-items/utils'
-import { RawDirectoryMetadata, RawFileMetadata } from '../../src/pod/types'
-import { DEFAULT_FILE_PERMISSIONS, getFileMode } from '../../src/file/utils'
-import { DEFAULT_DIRECTORY_PERMISSIONS, getDirectoryMode } from '../../src/directory/utils'
+import { wrapBytesWithHelpers } from '../../../src/utils/bytes'
+import { getExtendedPodsListByAccountData } from '../../../src/pod/utils'
+import { getRawMetadata } from '../../../src/content-items/utils'
+import { RawDirectoryMetadata, RawFileMetadata } from '../../../src/pod/types'
+import { DEFAULT_FILE_PERMISSIONS, getFileMode } from '../../../src/file/utils'
+import { DEFAULT_DIRECTORY_PERMISSIONS, getDirectoryMode } from '../../../src/directory/utils'
 
 jest.setTimeout(400000)
 describe('Fair Data Protocol with FairOS-dfs', () => {