Colossus: Archive script

setup.sh

@@ -8,7 +8,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # code build tools
     sudo apt-get update -y
     sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \
-      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev
+      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev p7zip-full
 
     # Docker: do not replace existing installation to avoid distrupting running containers
     if ! command -v docker &> /dev/null
@@ -23,7 +23,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     fi
     # install additional packages
     brew update
-    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf || :
+    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf p7zip || :
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
     echo "It is also recommended to install qemu emulators with following command:"
     echo "docker run --privileged --rm tonistiigi/binfmt --install all"

storage-node/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 4.3.0
+
+- Adds `archive` mode / command, which allows downloading, compressing and uploading assigned data objects to an external S3 bucket that can be used as a backup.
+
 ### 4.2.0
 
 - Fix `util:cleanup` script (call `loadDataObjectIdCache` first)

storage-node/package.json

@@ -1,14 +1,16 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"
   },
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@apollo/client": "^3.3.21",
+    "@aws-sdk/client-s3": "^3.675.0",
+    "@aws-sdk/s3-request-presigner": "^3.675.0",
     "@elastic/ecs-winston-format": "^1.3.1",
     "@joystream/metadata-protobuf": "^2.15.0",
     "@joystream/opentelemetry": "1.0.0",
@@ -36,6 +38,7 @@
     "await-lock": "^2.1.0",
     "base64url": "^3.0.1",
     "blake3-wasm": "^2.1.5",
+    "chokidar": "4.0.1",
     "cors": "^2.8.5",
     "cross-fetch": "^3.1.4",
     "express": "4.17.1",