update demo

.github/workflows/testing-api.yml

@@ -105,6 +105,28 @@ jobs:
         working-directory: api
         run: yarn test:integration
 
+  testing-api-unit:
+    name: Unit Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 18.16
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.16'
+
+      - name: Install API dependencies
+        working-directory: api
+        run: yarn install
+
+      - name: Run API tests + coverage
+        working-directory: api
+        run: yarn test:unit
+
 #      - name: Generate API coverage artifact
 #        uses: actions/upload-artifact@v2
 #        with:

.github/workflows/testing-client.yml

@@ -16,6 +16,7 @@ jobs:
     name: Running client tests
     runs-on: ubuntu-22.04
     timeout-minutes: 30
+    if: ${{ github.ref_name != 'test' }}
     strategy:
       fail-fast: false
     defaults:
@@ -49,7 +50,6 @@ jobs:
           command-prefix: yarn dlx
           command: "yarn test:e2e"
         env:
-          NEXT_PUBLIC_MAPBOX_API_TOKEN: ${{ secrets.NEXT_PUBLIC_MAPBOX_API_TOKEN }}
           NEXT_PUBLIC_API_URL: ${{ secrets.CYPRESS_API_URL }}
           NEXT_TELEMETRY_DISABLED: 1
           NEXTAUTH_URL: http://localhost:3000

.gitignore

@@ -5,9 +5,9 @@ node_modules
 /api/coverage
 .env
 
-## Data
+## Data folders (except for root "data" module which contains all the data engineering bits
 /data/datasets/**
-/data/h3_data_importer/data
+/data/**/data
 
 infrastructure/base/vars/*
 infrastructure/kubernetes/vars/*

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+repos:
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.17.0
+    hooks:
+      - id: gitleaks-docker
+
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.12.1
+    hooks:
+      - id: validate-pyproject
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+        types_or: [python, pyi, jupyter]
+
+      - id: ruff-format
+        types_or: [python, pyi, jupyter]
+
+  #    - repo: https://github.com/pre-commit/mirrors-mypy
+  #      rev: v1.3.0
+  #      hooks:
+  #          - id: mypy
+  #            language_version: python
+  #            exclude: tests/.*
+  #            additional_dependencies:
+  #                - types-simplejson
+  #                - types-attrs
+  #                - pydantic~=2.0
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier

Makefile

@@ -19,6 +19,9 @@ NC :=\033[0m # No Color
 start-api:
 	docker compose --project-name ${COMPOSE_PROJECT_NAME} up --build api
 
+start-tiler:
+	docker compose --project-name ${COMPOSE_PROJECT_NAME} up --build tiler
+
 # Starts the CLIENT application
 start-client:
 	docker compose --project-name ${COMPOSE_PROJECT_NAME} up --build client

api/config/custom-environment-variables.json

@@ -72,5 +72,11 @@
     "email": {
       "sendGridApiKey": "SENDGRID_API_KEY"
     }
+  },
+  "eudr": {
+    "apiKey": "CARTO_API_KEY",
+    "baseUrl": "CARTO_BASE_URL",
+    "credentials": "EUDR_CREDENTIALS",
+    "dataset": "EUDR_DATASET"
   }
 }

api/config/default.json

@@ -81,5 +81,11 @@
     "email": {
       "sendGridApiKey": null
     }
+  },
+  "eudr": {
+    "apiKey": null,
+    "baseUrl": "null",
+    "credentials": null,
+    "dataset": null
   }
 }

api/config/test.json

@@ -46,5 +46,11 @@
     "email": {
       "sendGridApiKey": "SG.forSomeReasonSendGridApiKeysNeedToStartWithSG."
     }
+  },
+  "eudr": {
+    "apiKey": null,
+    "baseUrl": "null",
+    "credentials": null,
+    "dataset": "test_dataset"
   }
 }

api/package.json

@@ -20,26 +20,31 @@
     "test:cov": "node --expose-gc ./node_modules/.bin/jest --config test/jest-config.json --coverage --forceExit",
     "test:debug": "node --inspect-brk --expose-gc -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --config test/jest-config.json --detectOpenHandles --forceExit",
     "test:integration": "node --expose-gc ./node_modules/.bin/jest --config test/jest-config.json --forceExit -i test/integration/",
+    "test:unit": "node --expose-gc ./node_modules/.bin/jest --config test/jest-config.json --forceExit -i test/unit/",
     "test:e2e": "NODE_OPTIONS=\"--max-old-space-size=6144\" node --expose-gc ./node_modules/.bin/jest --config test/jest-config.json --logHeapUsage --forceExit -i test/e2e/"
   },
   "engines": {
     "node": "^18.16"
   },
   "dependencies": {
+    "@google-cloud/bigquery": "^7.5.0",
     "@googlemaps/google-maps-services-js": "~3.3.2",
     "@json2csv/node": "^7.0.1",
     "@nestjs/axios": "^3.0.0",
     "@nestjs/bull": "^10.0.1",
     "@nestjs/cache-manager": "^2.0.1",
     "@nestjs/common": "^10.0.5",
     "@nestjs/core": "^10.0.5",
+    "@nestjs/cqrs": "^10.2.7",
     "@nestjs/jwt": "^10.1.0",
     "@nestjs/microservices": "^10.0.5",
     "@nestjs/passport": "^10.0.0",
     "@nestjs/platform-express": "^10.0.5",
+    "@nestjs/platform-socket.io": "^10.3.7",
     "@nestjs/swagger": "^7.1.1",
     "@nestjs/terminus": "^10.0.1",
     "@nestjs/typeorm": "^10.0.0",
+    "@nestjs/websockets": "^10.3.7",
     "@sendgrid/mail": "^7.7.0",
     "@types/object-hash": "~3.0.1",
     "bcrypt": "~5.1.0",
@@ -69,6 +74,7 @@
     "swagger-ui-express": "~4.6.0",
     "typeorm": "0.3.11",
     "uuid": "~9.0.0",
+    "wellknown": "^0.5.0",
     "xlsx": "~0.18.5",
     "yargs": "^17.3.1"
   },
@@ -85,6 +91,7 @@
     "@types/config": "^3.3.0",
     "@types/express": "^4.17.13",
     "@types/faker": "^6.6.9",
+    "@types/geojson": "^7946.0.14",
     "@types/jest": "^29.5.3",
     "@types/jsonapi-serializer": "^3.6.5",
     "@types/lodash": "^4.14.177",

api/src/app.module.ts

@@ -39,6 +39,9 @@ import { AuthorizationService } from 'modules/authorization/authorization.servic
 import { TasksService } from 'modules/tasks/tasks.service';
 import { NotificationsModule } from 'modules/notifications/notifications.module';
 import { ReportsModule } from 'modules/reports/reports.module';
+import { EudrModule } from 'modules/eudr-alerts/eudr.module';
+import { WebSocketsModule } from 'modules/notifications/websockets/websockets.module';
+import { AppEventsModule } from 'modules/events/app-events.module';
 
 const queueConfig: any = config.get('queue');
 
@@ -84,6 +87,9 @@ const queueConfig: any = config.get('queue');
     AuthorizationModule,
     NotificationsModule,
     ReportsModule,
+    WebSocketsModule,
+    EudrModule,
+    AppEventsModule,
   ],
   providers: [
     {

api/src/create-swagger-specification.ts

@@ -0,0 +1,35 @@
+import * as fs from 'fs';
+import * as crypto from 'crypto';
+import * as process from 'process';
+
+function hashContent(content: string): string {
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+const active: boolean = false;
+
+export async function createOrUpdateSwaggerSpec(document: any): Promise<void> {
+  if (active && process.env.NODE_ENV == 'development') {
+    const documentString: string = JSON.stringify(document);
+    const currentHash: string = hashContent(documentString);
+
+    const specPath: string = './swagger-spec.json';
+    if (fs.existsSync(specPath)) {
+      const existingSpec: string = fs.readFileSync(specPath, 'utf8');
+      const existingHash: string = hashContent(existingSpec);
+
+      if (currentHash !== existingHash) {
+        console.log('Swagger spec has changed. Updating...');
+        fs.writeFileSync(specPath, documentString);
+      } else {
+        console.log('No changes in Swagger spec.');
+      }
+    } else {
+      console.log('Swagger spec does not exist. Creating...');
+      fs.writeFileSync(specPath, documentString);
+    }
+  } else {
+    console.log('Swagger spec update is not active.');
+    return;
+  }
+}

api/src/guards/sensitive-info.guard.ts

@@ -18,6 +18,15 @@ export class SensitiveInfoGuard implements NestInterceptor {
     context: ExecutionContext,
     next: CallHandler<any>,
   ): Observable<any> {
+    const request = context.switchToHttp().getRequest();
+    if (this.dataComesFromAStreamEndpoint(request.url)) {
+      return next.handle();
+    }
+
     return next.handle().pipe(map((data: any) => instanceToPlain(data)));
   }
+
+  dataComesFromAStreamEndpoint(url: string): boolean {
+    return url.includes('eudr');
+  }
 }

api/src/main.ts

@@ -8,6 +8,7 @@ import * as compression from 'compression';
 import { JwtAuthGuard } from 'guards/jwt-auth.guard';
 import { useContainer } from 'class-validator';
 import { SensitiveInfoGuard } from 'guards/sensitive-info.guard';
+import { createOrUpdateSwaggerSpec } from 'create-swagger-specification';
 
 async function bootstrap(): Promise<void> {
   const logger: Logger = new Logger('bootstrap');
@@ -35,6 +36,8 @@ async function bootstrap(): Promise<void> {
   );
   SwaggerModule.setup('/swagger', app, swaggerDocument);
 
+  await createOrUpdateSwaggerSpec(swaggerDocument);
+
   app.useGlobalPipes(
     new ValidationPipe({
       forbidUnknownValues: true,

api/src/modules/admin-regions/admin-regions.controller.ts

@@ -38,7 +38,10 @@ import { CreateAdminRegionDto } from 'modules/admin-regions/dto/create.admin-reg
 import { UpdateAdminRegionDto } from 'modules/admin-regions/dto/update.admin-region.dto';
 import { PaginationMeta } from 'utils/app-base.service';
 import { ApiOkTreeResponse } from 'decorators/api-tree-response.decorator';
-import { GetAdminRegionTreeWithOptionsDto } from 'modules/admin-regions/dto/get-admin-region-tree-with-options.dto';
+import {
+  GetAdminRegionTreeWithOptionsDto,
+  GetEUDRAdminRegions,
+} from 'modules/admin-regions/dto/get-admin-region-tree-with-options.dto';
 import { SetScenarioIdsInterceptor } from 'modules/impact/set-scenario-ids.interceptor';
 
 @Controller(`/api/v1/admin-regions`)

api/src/modules/admin-regions/admin-regions.service.ts

@@ -217,10 +217,6 @@ export class AdminRegionsService extends AppBaseService<
     return this.findTreesWithOptions({ depth: adminRegionTreeOptions.depth });
   }
 
-  async getAdminRegionByIds(ids: string[]): Promise<AdminRegion[]> {
-    return this.adminRegionRepository.findByIds(ids);
-  }
-
   /**
    * @description: Returns an array of all children of given Admin Region's Ids with optional parameters
    * @param {string[]} adminRegionIds - The IDs of the admin regions.

api/src/modules/admin-regions/dto/get-admin-region-tree-with-options.dto.ts

@@ -1,6 +1,9 @@
 import { IsBoolean, IsNumber, IsOptional, IsUUID } from 'class-validator';
-import { ApiPropertyOptional } from '@nestjs/swagger';
-import { CommonFiltersDto } from 'utils/base.query-builder';
+import { ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+import {
+  CommonEUDRFiltersDTO,
+  CommonFiltersDto,
+} from 'utils/base.query-builder';
 import { Type } from 'class-transformer';
 
 export class GetAdminRegionTreeWithOptionsDto extends CommonFiltersDto {
@@ -24,3 +27,7 @@ export class GetAdminRegionTreeWithOptionsDto extends CommonFiltersDto {
   @IsUUID('4')
   scenarioId?: string;
 }
+
+export class GetEUDRAdminRegions extends CommonEUDRFiltersDTO {
+  withSourcingLocations!: boolean;
+}

api/src/modules/business-units/business-units.service.ts

@@ -1,9 +1,4 @@
-import {
-  forwardRef,
-  Inject,
-  Injectable,
-  NotFoundException,
-} from '@nestjs/common';
+import { forwardRef, Inject, Injectable } from '@nestjs/common';
 import {
   AppBaseService,
   JSONAPISerializerConfig,
@@ -52,17 +47,6 @@ export class BusinessUnitsService extends AppBaseService<
     };
   }
 
-  async getBusinessUnitById(id: string): Promise<BusinessUnit> {
-    const found: BusinessUnit | null =
-      await this.businessUnitRepository.findOneBy({ id });
-
-    if (!found) {
-      throw new NotFoundException(`Business Unit with ID "${id}" not found`);
-    }
-
-    return found;
-  }
-
   async getBusinessUnitsById(ids: string[]): Promise<BusinessUnit[]> {
     return this.businessUnitRepository.findBy({ id: In(ids) });
   }
@@ -80,10 +64,6 @@ export class BusinessUnitsService extends AppBaseService<
     return this.businessUnitRepository.saveListToTree(importData, 'mpath');
   }
 
-  async clearTable(): Promise<void> {
-    await this.businessUnitRepository.delete({});
-  }
-
   async getTrees(
     businessUnitTreeOptions: GetBusinessUnitTreeWithOptionsDto,
   ): Promise<BusinessUnit[]> {