Feat/implement new indicator computing

api/src/modules/geo-coding/geo-coding.service.ts

@@ -93,7 +93,7 @@ export class GeoCodingService extends GeoCodingAbstractClass {
         progressTracker.trackProgress();
       } catch (e: any) {
         errors.push({
-          line: i + 5,
+          row: i + 5,
           error: e.message,
           type: 'geo-coding-error',
           sheet: 'sourcingData',

api/src/modules/import-data/import-data.module.ts

@@ -36,6 +36,10 @@ import { SourcingDataDbCleaner } from 'modules/import-data/sourcing-data/sourcin
     }),
     BullModule.registerQueue({
       name: importQueueName,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+      },
     }),
     BullModule.registerQueue({
       name: 'eudr',

api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts

@@ -35,7 +35,7 @@ export class LocationAddressInputValidator
       (!(args.object as SourcingDataExcelValidator).location_latitude_input ||
         !(args.object as SourcingDataExcelValidator).location_longitude_input)
     ) {
-      return typeof addressInput === 'string' && addressInput.length > 2;
+      return typeof addressInput === 'string' && addressInput.length > 1;
     } else {
       return true;
     }

api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts

@@ -1,3 +1,4 @@
+import { Transform } from 'class-transformer';
 import {
   IsEnum,
   IsJSON,
@@ -10,6 +11,7 @@ import {
 //TODO: double check if we only use the material sheet in the excel to activate materials
 
 export class MaterialSheetValidator {
+  @Transform(({ value }) => String(value))
   @IsString({ message: 'Material hs_2017_code must be a string' })
   @MinLength(1, { message: 'Material hs_2017_code is too short' })
   @IsNotEmpty({ message: 'Material hs_2017_code must not be empty' })

api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts

@@ -81,4 +81,20 @@ export class IndicatorCoefficientsDto {
   @IsNotEmpty()
   @IsNumber()
   [INDICATOR_NAME_CODES.GHG_FARM]: number;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @Max(1000000)
+  @Min(0)
+  @IsNotEmpty()
+  @IsNumber()
+  [INDICATOR_NAME_CODES.WW]: number;
+
+  @ApiPropertyOptional()
+  @IsOptional()
+  @Max(1000000)
+  @Min(0)
+  @IsNotEmpty()
+  @IsNumber()
+  [INDICATOR_NAME_CODES.WC]: number;
 }

api/src/modules/indicator-records/indicator-records.module.ts

@@ -9,7 +9,7 @@ import { MaterialsModule } from 'modules/materials/materials.module';
 import { SourcingRecordsModule } from 'modules/sourcing-records/sourcing-records.module';
 import { CachedDataModule } from 'modules/cached-data/cached-data.module';
 import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service';
-import { IndicatorQueryDependencyManager } from 'modules/indicator-records/services/indicator-dependency-manager.service';
+import { ImpactQueryBuilder } from 'modules/indicator-records/services/indicator-dependency-manager.service';
 import { IndicatorRecordRepository } from 'modules/indicator-records/indicator-record.repository';
 
 @Module({
@@ -26,13 +26,13 @@ import { IndicatorRecordRepository } from 'modules/indicator-records/indicator-r
     IndicatorRecordsService,
     IndicatorRecordRepository,
     ImpactCalculator,
-    IndicatorQueryDependencyManager,
+    ImpactQueryBuilder,
   ],
   exports: [
     IndicatorRecordsService,
     IndicatorRecordRepository,
     ImpactCalculator,
-    IndicatorQueryDependencyManager,
+    ImpactQueryBuilder,
   ],
 })
 export class IndicatorRecordsModule {}

api/src/modules/indicator-records/services/impact-calculation.dependencies.ts

@@ -23,6 +23,8 @@ export class QueryPropertyNamesType {
   [INDICATOR_NAME_CODES.NCE]: INDICATOR_NAME_CODES.NCE;
   [INDICATOR_NAME_CODES.FLIL]: INDICATOR_NAME_CODES.FLIL;
   [INDICATOR_NAME_CODES.GHG_FARM]: INDICATOR_NAME_CODES.GHG_FARM;
+  [INDICATOR_NAME_CODES.WW]: INDICATOR_NAME_CODES.WW;
+  [INDICATOR_NAME_CODES.WC]: INDICATOR_NAME_CODES.WC;
 }
 
 export class QueryPropertyTypes {
@@ -38,6 +40,8 @@ export class QueryPropertyTypes {
   [INDICATOR_NAME_CODES.NCE]: number;
   [INDICATOR_NAME_CODES.ENL]: number;
   [INDICATOR_NAME_CODES.GHG_FARM]: number;
+  [INDICATOR_NAME_CODES.WW]: number;
+  [INDICATOR_NAME_CODES.WC]: number;
 }
 
 export const QueryPropertyNames: QueryPropertyNamesType = {
@@ -52,6 +56,8 @@ export const QueryPropertyNames: QueryPropertyNamesType = {
   [INDICATOR_NAME_CODES.NCE]: INDICATOR_NAME_CODES.NCE,
   [INDICATOR_NAME_CODES.FLIL]: INDICATOR_NAME_CODES.FLIL,
   [INDICATOR_NAME_CODES.GHG_FARM]: INDICATOR_NAME_CODES.GHG_FARM,
+  [INDICATOR_NAME_CODES.WW]: INDICATOR_NAME_CODES.WW,
+  [INDICATOR_NAME_CODES.WC]: INDICATOR_NAME_CODES.WC,
 } as const;
 
 export type ImpactQueryPropertyName =
@@ -127,4 +133,12 @@ export const INDICATOR_NAME_CODE_TO_QUERY_MAP: {
     [INDICATOR_NAME_CODES.GHG_FARM]: () =>
       `get_annual_commodity_weighted_material_impact_over_georegion($1, '${INDICATOR_NAME_CODES.GHG_FARM}', $2, 'producer') as "${INDICATOR_NAME_CODES.GHG_FARM}"`,
   },
+  [INDICATOR_NAME_CODES.WW]: {
+    [INDICATOR_NAME_CODES.WW]: () =>
+      `${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WW}', $3, $2) as "${INDICATOR_NAME_CODES.WU}"`,
+  },
+  [INDICATOR_NAME_CODES.WC]: {
+    [INDICATOR_NAME_CODES.WC]: () =>
+      `${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WC}', $3, $2) as "${INDICATOR_NAME_CODES.WC}"`,
+  },
 };

api/src/modules/indicator-records/services/impact-calculator.service.ts

@@ -22,16 +22,15 @@ import { IndicatorRecordCalculatedValuesDto } from 'modules/indicator-records/dt
 import { MaterialsToH3sService } from 'modules/materials/materials-to-h3s.service';
 import { IndicatorsService } from 'modules/indicators/indicators.service';
 import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity';
-import { H3DataService } from 'modules/h3-data/h3-data.service';
-import { IndicatorQueryDependencyManager } from 'modules/indicator-records/services/indicator-dependency-manager.service';
+import { ImpactQueryBuilder } from 'modules/indicator-records/services/indicator-dependency-manager.service';
 import { CachedDataService } from 'modules/cached-data/cached-data.service';
 import {
   CACHED_DATA_TYPE,
   CachedData,
 } from 'modules/cached-data/cached-data.entity';
 import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter';
-import { ImpactCalculationProgressTracker } from '../../impact/progress-tracker/impact-calculation.progress-tracker';
-import { ImportProgressTrackerFactory } from '../../events/import-data/import-progress.tracker.factory';
+import { ImpactCalculationProgressTracker } from 'modules/impact/progress-tracker/impact-calculation.progress-tracker';
+import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory';
 
 /**
  * @description: This is PoC (Proof of Concept) for the updated LG methodology v0.1
@@ -51,7 +50,7 @@ export class ImpactCalculator {
     private readonly indicatorRecordRepository: IndicatorRecordRepository,
     private readonly materialToH3: MaterialsToH3sService,
     private readonly indicatorService: IndicatorsService,
-    private readonly dependencyManager: IndicatorQueryDependencyManager,
+    private readonly dependencyManager: ImpactQueryBuilder,
     private readonly cachedDataService: CachedDataService,
     private readonly dataSource: DataSource,
     private readonly importProgress: ImportProgressEmitter,
@@ -383,6 +382,12 @@ export class ImpactCalculator {
             (100 * rawData.production) || 0
         );
       },
+      [INDICATOR_NAME_CODES.WW]: () => {
+        return rawData[INDICATOR_NAME_CODES.WW] * tonnage || 0;
+      },
+      [INDICATOR_NAME_CODES.WC]: () => {
+        return rawData[INDICATOR_NAME_CODES.WC] * tonnage || 0;
+      },
     };
 
     for (const [key, value] of Object.entries(calculations)) {
@@ -423,39 +428,35 @@ export class ImpactCalculator {
       //      indicator value calculation has not been refactored. It remains to be reworked
       const response: any = await this.dataSource.query(
         `
-        SELECT
-          -- TODO: Hack to retrieve 1 materialH3Id for each sourcingRecord. This should include a year fallback strategy in the stored procedures
-          --       used below
-          distinct on (sr.id)
-          sr.id as "sourcingRecordId",
-          sr.tonnage,
-          sr.year,
-          slwithmaterialh3data.id as "sourcingLocationId",
-          slwithmaterialh3data."materialH3DataId",
-          ${params}
-
-      FROM
-          sourcing_records sr
-          INNER JOIN
-              (
-                  SELECT
-                      sourcing_location.id,
-                      "scenarioInterventionId",
-                      "interventionType",
-                      mth."h3DataId" as "materialH3DataId",
-                      ${query}
-
-                  FROM
-                      sourcing_location
-                  inner join
-                    material_to_h3 mth
-                  on
-                    mth."materialId" = sourcing_location."materialId"
-                  WHERE "scenarioInterventionId" IS NULL
-                  AND "interventionType" IS NULL
-                  and mth."type" = 'producer'
-              ) as slwithmaterialh3data
-              on sr."sourcingLocationId" = slwithmaterialh3data.id`,
+          SELECT
+            -- TODO: Hack to retrieve 1 materialH3Id for each sourcingRecord. This should include a year fallback strategy in the stored procedures
+            --       used below
+            distinct
+          on (sr.id)
+            sr.id as "sourcingRecordId",
+            sr.tonnage,
+            sr.year,
+            slwithmaterialh3data.id as "sourcingLocationId",
+            slwithmaterialh3data."materialH3DataId",
+            ${params}
+
+          FROM
+            sourcing_records sr
+            INNER JOIN
+            (
+            SELECT
+            sourcing_location.id, "scenarioInterventionId", "interventionType", mth."h3DataId" as "materialH3DataId", ${query}
+            FROM
+            sourcing_location
+            inner join
+            material_to_h3 mth
+            on
+            mth."materialId" = sourcing_location."materialId"
+            WHERE "scenarioInterventionId" IS NULL
+            AND "interventionType" IS NULL
+            and mth."type" = 'producer'
+            ) as slwithmaterialh3data
+          on sr."sourcingLocationId" = slwithmaterialh3data.id`,
       );
       if (!response.length)
         this.logger.warn(

api/src/modules/indicator-records/services/indicator-dependency-manager.service.ts

@@ -12,11 +12,20 @@ type IndicatorNameCodeToQuery = {
   [key in ImpactQueryPropertyName]?: ImpactPropertyToQueryFunction;
 };
 
+/**
+ * @description: Dynamically builds the query for calculating the raw impact values. Based on which indicators are activated in the platform,
+ * it gets all dependencies for each indicator, defined in the queryMap, and builds the stringified query that then will injected withint the main query.
+ */
+
 @Injectable()
-export class IndicatorQueryDependencyManager {
+export class ImpactQueryBuilder {
   queryMap: typeof INDICATOR_NAME_CODE_TO_QUERY_MAP =
     INDICATOR_NAME_CODE_TO_QUERY_MAP;
 
+  /**
+   * @description: Builds the query for the intervention impact calculation
+   */
+
   buildQueryForIntervention(nameCodes: INDICATOR_NAME_CODES[]): string {
     const queries: ImpactQueryExpression[] = [];
     for (const nameCode of nameCodes) {
@@ -29,6 +38,14 @@ export class IndicatorQueryDependencyManager {
     return [...new Set(queries)].join(', ');
   }
 
+  /**
+   * @description: Builds the query for the import impact calculation. Since for the main query whole columns will be used instead of specific value:
+   * i.e: All geoRegion and material Ids from sourcing locations, instead of specific Ids, we need to quote wrap the table plus column names so that can be
+   * injected into the main query.
+   *
+   * @returns: Object with the params and query string that will be injected into the main query.
+   */
+
   buildQueryForImport(nameCodes: INDICATOR_NAME_CODES[]): {
     params: string;
     query: string;

api/src/modules/indicators/indicator.entity.ts

@@ -31,6 +31,8 @@ export enum INDICATOR_NAME_CODES {
   ENL = 'ENL',
   NCE = 'NCE',
   FLIL = 'FLIL',
+  WW = 'WW',
+  WC = 'WC',
 }
 
 export const indicatorResource: BaseServiceResource = {

api/test/integration/indicator-record/calculations/indicator-records.service.spec.ts

@@ -247,6 +247,8 @@ describe('Impact Calculator Tests', () => {
         [INDICATOR_NAME_CODES.FLIL]: 0.3,
         [INDICATOR_NAME_CODES.ENL]: 0.1,
         [INDICATOR_NAME_CODES.GHG_FARM]: 0.2,
+        [INDICATOR_NAME_CODES.WW]: 0.3,
+        [INDICATOR_NAME_CODES.WC]: 0.4,
       };
 
       //ACT