feat: ability to upload any amount of items (#293)

* feat: ability to upload any amount of pods

* feat: unlimited pods list

* refactor: upgraded version of fairos and bee

* feat: unlimited directories and files

* feat: fairos tests runs in parallel with nodejs

* refactor: set a bee version to 1.13.0 for tests

* refactor: add --without-bees for the fdp-play

* fix: fixed recursive data reading

* fix: removed --without-bees flag

* fix: using the correct variable name in the runner

* refactor: puppeteer headless

* feat: install chrome for puppeteer

* feat: install chrome for puppeteer

* feat: install chrome for puppeteer

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* feat: puppeteer config

* refactor: disable delete fairos tests

* refactor: removed logs

* refactor: removed comments

* feat: v2 migration

* test: v2 migration tests

* chore: readme update

* feat: data migration improvement

* test: additional v2 migration test

---------

Co-authored-by: Vladan <[email protected]>

.github/workflows/tests.yaml

@@ -9,8 +9,8 @@ on:
       - '**'
 
 env:
-  BEE_VERSION: '1.11.1'
-  FAIROS_IMAGE: 'fairdatasociety/fairos-dfs:v0.9.6'
+  BEE_VERSION: '1.13.0'
+  FAIROS_IMAGE: 'fairdatasociety/fairos-dfs:v0.10.0-rc6'
 
 jobs:
   nodejs:
@@ -39,7 +39,7 @@ jobs:
         run: npm install -g @fairdatasociety/fdp-play
 
       - name: Run fdp-play
-        run: fdp-play start -d --fairos --fairos-image $FAIROS_IMAGE --bee-version $BEE_VERSION
+        run: fdp-play start -d --bee-version $BEE_VERSION
 
       ## Try getting the node modules from cache, if failed npm ci
       - uses: actions/cache@v3
@@ -48,8 +48,8 @@ jobs:
           path: node_modules
           key: ${{ runner.os }}-node-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
-            ${{ runner.OS }}-node-${{ matrix.node }}-${{ env.cache-name }}-
-            ${{ runner.OS }}-node-${{ matrix.node }}-
+            ${{ runner.os }}-node-${{ matrix.node }}-${{ env.cache-name }}-
+            ${{ runner.os }}-node-${{ matrix.node }}-
 
       - name: Install npm deps
         if: steps.cache-npm.outputs.cache-hit != 'true'
@@ -58,6 +58,51 @@ jobs:
       - name: Run unit and integration tests for node
         run: npm run test:node -- --detectOpenHandles
 
+      - name: Run migration tests for node
+        run: npm run test:migration -- --detectOpenHandles
+
+  fairos:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x, 18.x]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Auth to Github Package Docker Registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://docker.pkg.github.com -u ${GITHUB_ACTOR} --password-stdin
+
+      - name: Install fdp-play
+        run: npm install -g @fairdatasociety/fdp-play
+
+      - name: Run fdp-play
+        run: fdp-play start -d --fairos --fairos-image $FAIROS_IMAGE --bee-version $BEE_VERSION
+
+      ## Try getting the node modules from cache, if failed npm ci
+      - uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-node-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-${{ matrix.node }}-${{ env.cache-name }}-
+            ${{ runner.os }}-node-${{ matrix.node }}-
+
+      - name: Install npm deps
+        if: steps.cache-npm.outputs.cache-hit != 'true'
+        run: npm ci
+
       - name: Run unit and integration tests for FairOS
         run: npm run test:fairos -- --detectOpenHandles
 
@@ -96,8 +141,8 @@ jobs:
           path: node_modules
           key: ${{ runner.os }}-node-${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
-            ${{ runner.OS }}-node-${{ matrix.node }}-${{ env.cache-name }}-
-            ${{ runner.OS }}-node-${{ matrix.node }}-
+            ${{ runner.os }}-node-${{ matrix.node }}-${{ env.cache-name }}-
+            ${{ runner.os }}-node-${{ matrix.node }}-
 
       - name: Install npm deps
         if: steps.cache-npm.outputs.cache-hit != 'true'

.gitignore

@@ -30,3 +30,5 @@ node_modules
 docs
 
 .vscode
+
+test/fdp-storage

README.md

@@ -233,7 +233,7 @@ const blocks = []
 for (let i = 0; i < blocksCount; i++) {
   const dataBlock = getDataBlock(data, blockSize, i)
   // fdp instance with or without logged in user
-  blocks.push(await fdp.file.uploadDataBlock(dataBlock, i))
+  blocks.push(await fdp.file.uploadDataBlock(dataBlock, i, dataBlock.length))
 }
 
 // fdp instance with logged in user
@@ -335,6 +335,18 @@ const fdpCache = new FdpStorage('https://localhost:1633', batchId, {
 fdpCache.cache.object = JSON.parse(cache)
 ```
 
+## Data migration
+
+Starting from the version `0.18.0`, pods and directories are stored in different format than the older versions. For all new accounts this doesn't have any impact. But to access pods and folders from existing accounts, migration is required.
+
+Migration is done transparently, but there are some requirements that users should follow.
+
+Pod list is converted to V2 when the `fdp.personalStorage.list()` method is invoked. So before working with existing pods, call the `fdp.personalStorage.list()` method first.
+
+Directories are converted on the fly. Here the same principle applies as for pods. The `fdp.directory.read()` method must be invoked first, before invoking any other operation on the provided directory. The read method will convert not only the provided directory, but also all parent directories. That process happens only once, and all subsequent accesses will work with V2 data instantly.
+
+Existing files are accessible in the new version. But from the `0.18.0` version, files are compressed before upload, which wasn't the case before. To compress files, they must be reuploaded.
+
 ## Documentation
 
 You can generate API docs locally with:

jest-puppeteer.config.js

@@ -1,5 +1,7 @@
 module.exports = {
   launch: {
     dumpio: true, // Forwards browser console into test console for easier debugging
+    headless: 'new', // Opt-in to the new headless mode for Chrome
+    channel: 'chrome',
   },
 }

jest.config.ts

@@ -44,6 +44,8 @@ export default async (): Promise<Config.InitialOptions> => {
     // An array of directory names to be searched recursively up from the requiring module's location
     moduleDirectories: ['node_modules'],
 
+    testTimeout: 90000,
+
     // Run tests from one or more projects
     projects: [
       {
@@ -67,6 +69,11 @@ export default async (): Promise<Config.InitialOptions> => {
         testEnvironment: 'node',
         testRegex: 'test/integration/((?!\\.(browser|fairos)).)*\\.spec\\.ts',
       },
+      {
+        displayName: 'node:migration',
+        testEnvironment: 'node',
+        testRegex: 'test/migration/((?!\\.(browser|fairos)).)*\\.spec\\.ts',
+      },
     ] as unknown[] as string[], // bad types
 
     // The root directory that Jest should scan for tests and modules within

package.json

@@ -43,6 +43,7 @@
     "test:unit:browser": "npm run compile:browser && jest --verbose --selectProjects=dom:unit:browser --config=jest.config.ts",
     "test:integration:node": "jest --verbose --selectProjects=node:integration --config=jest.config.ts",
     "test:integration": "npm run compile:browser && jest --verbose --selectProjects=node:integration dom:integration --config=jest.config.ts",
+    "test:migration": "jest --verbose --selectProjects=node:migration --config=jest.config.ts",
     "test:unit": "jest --verbose --selectProjects=node:unit --config=jest.config.ts ",
     "test:node": "jest --verbose --selectProjects=node:unit node:integration --config=jest.config.ts",
     "test:fairos": "jest --verbose --selectProjects=node:fairos --config=jest.config.ts",
@@ -57,7 +58,8 @@
     "@fairdatasociety/fdp-contracts-js": "^3.10.0",
     "crypto-js": "^4.2.0",
     "ethers": "^5.5.2",
-    "js-sha3": "^0.9.2"
+    "js-sha3": "^0.9.2",
+    "pako": "^2.1.0"
   },
   "devDependencies": {
     "@babel/core": "^7.23.2",
@@ -79,6 +81,7 @@
     "@types/jest": "^29.5.6",
     "@types/jest-environment-puppeteer": "^5.0.5",
     "@types/node": "^20.8.9",
+    "@types/pako": "^2.0.3",
     "@types/webpack-bundle-analyzer": "^4.6.2",
     "@types/ws": "^8.5.8",
     "@typescript-eslint/eslint-plugin": "^6.9.0",

puppeteer.config.js

@@ -0,0 +1,11 @@
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { join } = require('path')
+
+/**
+ * @type {import("puppeteer").Configuration}
+ */
+module.exports = {
+  // Changes the cache location for Puppeteer.
+  cacheDirectory: join(__dirname, '.cache', 'puppeteer-data'),
+  skipDownload: false,
+}

src/account/account.ts

@@ -1,10 +1,11 @@
-import { Bee, PrivateKeyBytes, Reference, Utils } from '@ethersphere/bee-js'
+import { Bee, PrivateKeyBytes, Reference } from '@ethersphere/bee-js'
 import { utils, Wallet } from 'ethers'
 import { IV_LENGTH, decryptBytes, encryptBytes } from '../utils/encryption'
 import { assertChunkSizeLength, CHUNK_SIZE, SEED_SIZE, createCredentialsTopic, HD_PATH } from './utils'
 import { Connection } from '../connection/connection'
 import CryptoJS from 'crypto-js'
 import { wordArrayToBytes } from '../utils/bytes'
+import { EthAddress } from '../utils/eth'
 
 /**
  * User account with seed phrase
@@ -54,7 +55,7 @@ export async function uploadPortableAccount(
  */
 export async function downloadPortableAccount(
   bee: Bee,
-  address: Utils.EthAddress,
+  address: EthAddress,
   username: string,
   password: string,
 ): Promise<UserAccountWithSeed> {

src/account/index.ts

@@ -1,4 +1,3 @@
 export * as Account from './account'
 export * as Encryption from '../utils/encryption'
 export * as Utils from './utils'
-//TODO export every exportable