Merge pull request #60 from nevermined-io/fix/stream_crash

Adding support to IPFS and solidifying upload method

README.md

@@ -72,6 +72,9 @@ The Nevermined Node reads the following environment variables allowing the confi
 
 | **GRAPH_URL** | Public URL of the Nevermined subgraphs |
 | **NO_GRAPH** | If `true` the node will read events from the blockchain node instead of from the subgraphs. Depending on the network there could be a limit on the number of blocks to scan. | `false`
+| **IPFS_GATEWAY** | Public IPFS gateway that can be used to fech or upload content. | https://ipfs.infura.io:5001
+| **IPFS_PROJECT_ID** | Ipfs Project Id | `2DSADASD4234234234`
+| **IPFS_PROJECT_SECRET** | Ipfs Project Secret | `ccdafda55666dada`
 | **FILECOIN_GATEWAY** | Public Filecoin gateway that can be used to fech content. The `:cid` part of the url will be replace by the file `cid` | https://dweb.link/ipfs/:cid
 | **ESTUARY_TOKEN** | Estuary is a service that facilitates the interaction with Filecoin. This variable must include the token to use their API. See more here: https://estuary.tech/ | `EST651aa3a7-4756-4bd9-a563-1cdd565894645`
 | **AWS_S3_ACCESS_KEY_ID** | Amazon S3 Access Key Id | `4535hnj43`

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-ts",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Nevermined Node",
   "main": "main.ts",
   "scripts": {
@@ -19,11 +19,11 @@
   },
   "dependencies": {
     "@nestjs/axios": "^1.0.0",
-    "@nestjs/common": "^8.4.0",
-    "@nestjs/core": "^8.4.0",
-    "@nestjs/jwt": "^8.0.0",
+    "@nestjs/common": "^8.4.5",
+    "@nestjs/core": "^8.4.5",
+    "@nestjs/jwt": "^8.0.1",
     "@nestjs/passport": "^8.2.1",
-    "@nestjs/platform-express": "^8.4.1",
+    "@nestjs/platform-express": "^8.4.5",
     "@nestjs/swagger": "^5.2.0",
     "@nestjs/typeorm": "^8.0.3",
     "@nevermined-io/argo-workflows-api": "^0.1.3",
@@ -41,6 +41,7 @@
     "cli-color": "^2.0.1",
     "dotenv": "^16.0.0",
     "eciesjs": "^0.3.15",
+    "ipfs-http-client-lite": "^0.3.0",
     "ethers": "^5.6.3",
     "fetch-blob": "^3.2.0",
     "form-data": "^4.0.0",

src/access/access.controller.ts

@@ -3,6 +3,8 @@ import {
   Body,
   Controller,
   Get,
+  HttpException,
+  HttpStatus,
   NotFoundException,
   Param,
   Post,
@@ -26,10 +28,18 @@ import { TransferDto } from './dto/transfer';
 import { UploadDto } from './dto/upload';
 import { UploadResult } from './dto/upload-result';
 import { AgreementData } from '@nevermined-io/nevermined-sdk-js/dist/node/keeper/contracts/managers';
+import { utils } from '@nevermined-io/nevermined-sdk-js';
+
+export enum UploadBackends {
+  IPFS = 'ipfs',
+  Filecoin = 'filecoin',
+  AmazonS3 = 's3',
+}
 
 @ApiTags('Access')
 @Controller()
 export class AccessController {
+
   constructor(private nvmService: NeverminedService) {}
 
   @Get('access/:agreement_id/:index')
@@ -134,43 +144,52 @@ export class AccessController {
 
   @Post('upload/:backend')
   @ApiOperation({
-    description: 'Access asset',
+    description: 'Uploads a file or some content to a remote storage',
     summary: 'Public',
   })
   @Public()
   @UseInterceptors(FileInterceptor('file'))
   @ApiResponse({
     status: 200,
-    description: 'Return the url of asset',
+    description: 'Return the url of the file uploaded',
   })
   async doUpload(
     @Body() uploadData: UploadDto,
-    @Param('backend') backend: string,
+    @Param('backend') backend: UploadBackends,
     @UploadedFile() file: Express.Multer.File
   ): Promise<UploadResult> {
-    if (!file) {
-      throw new BadRequestException('No file in request');
+    let data: Buffer
+    let fileName: string
+    if (file) {
+      data = file.buffer;
+      fileName = file.originalname
+    } else if (uploadData.message)  {
+      data = Buffer.from(uploadData.message)
+      fileName = `fileUpload_${utils.generateId()}.data${uploadData.encrypt?'.encrypted':''}`
+    } else {
+      throw new BadRequestException('No file or message in request');
     }
-    let data = file.buffer;
-    if (uploadData.encrypt) {
-      // generate password
-      Logger.debug(`Uploading with password, filename ${file.originalname}`);
-      const password = crypto.randomBytes(32).toString('base64url');
-      data = Buffer.from(aes_encryption_256(data, password), 'binary');
-      if (backend === 's3') {
-        const url = await this.nvmService.uploadS3(data, file.originalname);
-        return { url, password };
-      } else if (backend === 'filecoin') {
-        const url = await this.nvmService.uploadFilecoin(data, file.originalname);
+    console.log(`Backend ${backend}`)
+    if (!Object.values(UploadBackends).includes(backend))
+      throw new BadRequestException(`Backend ${backend} not supported`);
+    try {
+      let url: string
+      if (uploadData.encrypt) {
+        // generate password
+        Logger.debug(`Uploading with password, filename ${fileName}`);
+        const password = crypto.randomBytes(32).toString('base64url');
+        data = Buffer.from(aes_encryption_256(data, password), 'binary');
+        url = await this.nvmService.uploadToBackend(backend, data, fileName);
         return { url, password };
       }
-    }
-    if (backend === 's3') {
-      const url = await this.nvmService.uploadS3(data, file.originalname);
-      return { url };
-    } else if (backend === 'filecoin') {
-      const url = await this.nvmService.uploadFilecoin(data, file.originalname);
+
+      url = await this.nvmService.uploadToBackend(backend, data, fileName);
       return { url };
+
+    } catch (error) {
+      Logger.error(`Error processing upload: ${error.message}`);
+      throw new HttpException(error.message, HttpStatus.INTERNAL_SERVER_ERROR);
     }
+
   }
 }

src/access/access.integration.spec.ts

@@ -93,8 +93,27 @@ describe('Info', () => {
   });
   it('upload / no params', async () => {
     const response = await request(app.getHttpServer())
-      .post(`/upload/method`);
+      .post(`/upload/ipfs`);
     expect(response.statusCode).toBe(400);
-    expect((response.error as any).text).toContain('No file');
+    expect((response.error as any).text).toContain('No file or message');
   });
+  it('upload wrong backend', async () => {
+    const response = await request(app.getHttpServer())
+      .post(`/upload/wrong`)
+      .send({
+        message: 'hi there'
+      });
+    expect(response.statusCode).toBe(400);
+    expect((response.error as any).text).toContain('Backend wrong not supported');
+  });
+  it('upload ipfs', async () => {
+    const response = await request(app.getHttpServer())
+      .post(`/upload/ipfs`)
+      .send({
+        message: 'hi there'
+      });
+    expect(response.statusCode).toBe(201);
+    console.log(JSON.stringify(response.body))
+    expect(response.body.url).toContain('cid://');
+  });  
 });

src/access/dto/upload.ts

@@ -1,10 +1,18 @@
 import { ApiProperty } from "@nestjs/swagger";
+import { string } from "joi";
 
 export class UploadDto {
     @ApiProperty({
       description: 'Encrypt uploaded data',
       example: 'false',
       required: false,
     })
-    encrypt: string;
+    encrypt: string;    
+    @ApiProperty({
+      type: string,
+      example: 'Hello!',
+      description: 'Message to upload',
+      required: false,
+    })
+    message: string;
 }

src/shared/nevermined/nvm.service.ts

@@ -14,6 +14,8 @@ import { didZeroX } from '@nevermined-io/nevermined-sdk-js/dist/node/utils';
 import { HttpModuleOptions, HttpService } from '@nestjs/axios';
 import { firstValueFrom } from 'rxjs';
 import { AxiosError } from 'axios';
+import IpfsHttpClientLite from 'ipfs-http-client-lite';
+import { UploadBackends } from 'src/access/access.controller';
 
 @Injectable()
 export class NeverminedService {
@@ -224,7 +226,7 @@ export class NeverminedService {
         throw new InternalServerErrorException(obj.error);
       }
 
-      return 'cid://' + obj.cid;
+      return `cid://${obj.cid}`;
     } catch (e) {
       if (e instanceof AxiosError) {
         Logger.error('Axios Error: ', e.response);
@@ -236,6 +238,47 @@ export class NeverminedService {
     }
   }
 
+  async uploadIPFS(content: Buffer, filename: string): Promise<string> {
+    try {
+      Logger.debug(`Uploading to IPFS ${filename}`);
+      const ipfsAuthToken = this.getIPFSAuthToken();
+
+      const ipfs = IpfsHttpClientLite({
+        apiUrl: this.config.get('IPFS_GATEWAY'),
+        ...(ipfsAuthToken && {
+          headers: { Authorization: `Basic ${ipfsAuthToken}` }
+        })
+      });
+      const addResult = await ipfs.add(content);
+      return `cid://${addResult[0].hash}`;
+    } catch (e) {
+      Logger.error(`Uploading ${filename}: IPFS error ${e}`);
+      throw new InternalServerErrorException(e);
+    }
+  }
+
+  async uploadToBackend(backend: UploadBackends, data: Buffer, fileName: string): Promise<string> {
+    if (backend === 's3') {
+      return await this.uploadS3(data, fileName);      
+    } else if (backend === 'filecoin') {
+      return await this.uploadFilecoin(data, fileName);      
+    } else if (backend === 'ipfs') {
+      return await this.uploadIPFS(data, fileName);      
+    }  
+  }
+
+  private getIPFSAuthToken(): string | undefined {
+
+    if (!this.config.get('IPFS_PROJECT_ID') || !this.config.get('IPFS_PROJECT_SECRET')) {
+      Logger.warn(`Infura IPFS_PROJECT_ID or IPFS_PROJECT_SECRET are not set - disabling ipfs auth`)      
+      return undefined
+    } else {
+        return Buffer.from(
+          `${this.config.get('IPFS_PROJECT_ID')}:${this.config.get('IPFS_PROJECT_SECRET')}`
+        ).toString('base64')
+    }    
+  }
+
   private isDTP(main: MetaDataMain): boolean {
     return main.files && main.files.some((f) => f.encryption === 'dtp');
   }