From 43bd635758924501d385663ee96c3fabf9496a0a Mon Sep 17 00:00:00 2001 From: alexeh Date: Fri, 13 Dec 2024 05:11:12 +0100 Subject: [PATCH 1/9] add WW and WC indicator name codes --- api/src/modules/indicators/indicator.entity.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/modules/indicators/indicator.entity.ts b/api/src/modules/indicators/indicator.entity.ts index daad64915..ca3d194b3 100644 --- a/api/src/modules/indicators/indicator.entity.ts +++ b/api/src/modules/indicators/indicator.entity.ts @@ -31,6 +31,8 @@ export enum INDICATOR_NAME_CODES { ENL = 'ENL', NCE = 'NCE', FLIL = 'FLIL', + WW = 'WW', + WC = 'WC', } export const indicatorResource: BaseServiceResource = { From ff7d38f9310f7c3366a6bb82bf5e98d4b7697969 Mon Sep 17 00:00:00 2001 From: alexeh Date: Fri, 13 Dec 2024 05:11:36 +0100 Subject: [PATCH 2/9] add WW and WC computing dependencies (same as WU) --- .../services/impact-calculation.dependencies.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/src/modules/indicator-records/services/impact-calculation.dependencies.ts b/api/src/modules/indicator-records/services/impact-calculation.dependencies.ts index 1adcfb2dd..deac4a7a1 100644 --- a/api/src/modules/indicator-records/services/impact-calculation.dependencies.ts +++ b/api/src/modules/indicator-records/services/impact-calculation.dependencies.ts @@ -23,6 +23,8 @@ export class QueryPropertyNamesType { [INDICATOR_NAME_CODES.NCE]: INDICATOR_NAME_CODES.NCE; [INDICATOR_NAME_CODES.FLIL]: INDICATOR_NAME_CODES.FLIL; [INDICATOR_NAME_CODES.GHG_FARM]: INDICATOR_NAME_CODES.GHG_FARM; + [INDICATOR_NAME_CODES.WW]: INDICATOR_NAME_CODES.WW; + [INDICATOR_NAME_CODES.WC]: INDICATOR_NAME_CODES.WC; } export class QueryPropertyTypes { @@ -38,6 +40,8 @@ export class QueryPropertyTypes { [INDICATOR_NAME_CODES.NCE]: number; [INDICATOR_NAME_CODES.ENL]: number; [INDICATOR_NAME_CODES.GHG_FARM]: number; + [INDICATOR_NAME_CODES.WW]: number; + [INDICATOR_NAME_CODES.WC]: number; } export const QueryPropertyNames: QueryPropertyNamesType = { @@ -52,6 +56,8 @@ export const QueryPropertyNames: QueryPropertyNamesType = { [INDICATOR_NAME_CODES.NCE]: INDICATOR_NAME_CODES.NCE, [INDICATOR_NAME_CODES.FLIL]: INDICATOR_NAME_CODES.FLIL, [INDICATOR_NAME_CODES.GHG_FARM]: INDICATOR_NAME_CODES.GHG_FARM, + [INDICATOR_NAME_CODES.WW]: INDICATOR_NAME_CODES.WW, + [INDICATOR_NAME_CODES.WC]: INDICATOR_NAME_CODES.WC, } as const; export type ImpactQueryPropertyName = @@ -127,4 +133,12 @@ export const INDICATOR_NAME_CODE_TO_QUERY_MAP: { [INDICATOR_NAME_CODES.GHG_FARM]: () => `get_annual_commodity_weighted_material_impact_over_georegion($1, '${INDICATOR_NAME_CODES.GHG_FARM}', $2, 'producer') as "${INDICATOR_NAME_CODES.GHG_FARM}"`, }, + [INDICATOR_NAME_CODES.WW]: { + [INDICATOR_NAME_CODES.WW]: () => + `${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WW}', $3, $2) as "${INDICATOR_NAME_CODES.WU}"`, + }, + [INDICATOR_NAME_CODES.WC]: { + [INDICATOR_NAME_CODES.WC]: () => + `${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WC}', $3, $2) as "${INDICATOR_NAME_CODES.WC}"`, + }, }; From b037cef97366439519e1629815ac4d95966ee78c Mon Sep 17 00:00:00 2001 From: alexeh Date: Fri, 13 Dec 2024 05:14:44 +0100 Subject: [PATCH 3/9] add new namecodes to indicator-coefficients.dto.ts --- .../dto/indicator-coefficients.dto.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts b/api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts index 1b2860416..9503027e0 100644 --- a/api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts +++ b/api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts @@ -81,4 +81,20 @@ export class IndicatorCoefficientsDto { @IsNotEmpty() @IsNumber() [INDICATOR_NAME_CODES.GHG_FARM]: number; + + @ApiPropertyOptional() + @IsOptional() + @Max(1000000) + @Min(0) + @IsNotEmpty() + @IsNumber() + [INDICATOR_NAME_CODES.WW]: number; + + @ApiPropertyOptional() + @IsOptional() + @Max(1000000) + @Min(0) + @IsNotEmpty() + @IsNumber() + [INDICATOR_NAME_CODES.WC]: number; } From 5f7461b71437eeba11f10e69eb333d77f828d2de Mon Sep 17 00:00:00 2001 From: alexeh Date: Fri, 13 Dec 2024 05:16:46 +0100 Subject: [PATCH 4/9] add WW and WC value processing to impact-calculator.service.ts --- .../services/impact-calculator.service.ts | 72 ++++++++++--------- .../indicator-records.service.spec.ts | 2 + 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/api/src/modules/indicator-records/services/impact-calculator.service.ts b/api/src/modules/indicator-records/services/impact-calculator.service.ts index b624a1e5a..8f97dcb63 100644 --- a/api/src/modules/indicator-records/services/impact-calculator.service.ts +++ b/api/src/modules/indicator-records/services/impact-calculator.service.ts @@ -23,7 +23,7 @@ import { MaterialsToH3sService } from 'modules/materials/materials-to-h3s.servic import { IndicatorsService } from 'modules/indicators/indicators.service'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; import { H3DataService } from 'modules/h3-data/h3-data.service'; -import { IndicatorQueryDependencyManager } from 'modules/indicator-records/services/indicator-dependency-manager.service'; +import { ImpactQueryBuilder } from 'modules/indicator-records/services/indicator-dependency-manager.service'; import { CachedDataService } from 'modules/cached-data/cached-data.service'; import { CACHED_DATA_TYPE, @@ -51,7 +51,7 @@ export class ImpactCalculator { private readonly indicatorRecordRepository: IndicatorRecordRepository, private readonly materialToH3: MaterialsToH3sService, private readonly indicatorService: IndicatorsService, - private readonly dependencyManager: IndicatorQueryDependencyManager, + private readonly dependencyManager: ImpactQueryBuilder, private readonly cachedDataService: CachedDataService, private readonly dataSource: DataSource, private readonly importProgress: ImportProgressEmitter, @@ -383,6 +383,12 @@ export class ImpactCalculator { (100 * rawData.production) || 0 ); }, + [INDICATOR_NAME_CODES.WW]: () => { + return rawData[INDICATOR_NAME_CODES.WW] * tonnage || 0; + }, + [INDICATOR_NAME_CODES.WC]: () => { + return rawData[INDICATOR_NAME_CODES.WC] * tonnage || 0; + }, }; for (const [key, value] of Object.entries(calculations)) { @@ -423,39 +429,35 @@ export class ImpactCalculator { // indicator value calculation has not been refactored. It remains to be reworked const response: any = await this.dataSource.query( ` - SELECT - -- TODO: Hack to retrieve 1 materialH3Id for each sourcingRecord. This should include a year fallback strategy in the stored procedures - -- used below - distinct on (sr.id) - sr.id as "sourcingRecordId", - sr.tonnage, - sr.year, - slwithmaterialh3data.id as "sourcingLocationId", - slwithmaterialh3data."materialH3DataId", - ${params} - - FROM - sourcing_records sr - INNER JOIN - ( - SELECT - sourcing_location.id, - "scenarioInterventionId", - "interventionType", - mth."h3DataId" as "materialH3DataId", - ${query} - - FROM - sourcing_location - inner join - material_to_h3 mth - on - mth."materialId" = sourcing_location."materialId" - WHERE "scenarioInterventionId" IS NULL - AND "interventionType" IS NULL - and mth."type" = 'producer' - ) as slwithmaterialh3data - on sr."sourcingLocationId" = slwithmaterialh3data.id`, + SELECT + -- TODO: Hack to retrieve 1 materialH3Id for each sourcingRecord. This should include a year fallback strategy in the stored procedures + -- used below + distinct + on (sr.id) + sr.id as "sourcingRecordId", + sr.tonnage, + sr.year, + slwithmaterialh3data.id as "sourcingLocationId", + slwithmaterialh3data."materialH3DataId", + ${params} + + FROM + sourcing_records sr + INNER JOIN + ( + SELECT + sourcing_location.id, "scenarioInterventionId", "interventionType", mth."h3DataId" as "materialH3DataId", ${query} + FROM + sourcing_location + inner join + material_to_h3 mth + on + mth."materialId" = sourcing_location."materialId" + WHERE "scenarioInterventionId" IS NULL + AND "interventionType" IS NULL + and mth."type" = 'producer' + ) as slwithmaterialh3data + on sr."sourcingLocationId" = slwithmaterialh3data.id`, ); if (!response.length) this.logger.warn( diff --git a/api/test/integration/indicator-record/calculations/indicator-records.service.spec.ts b/api/test/integration/indicator-record/calculations/indicator-records.service.spec.ts index 762890afc..6c2063cd7 100644 --- a/api/test/integration/indicator-record/calculations/indicator-records.service.spec.ts +++ b/api/test/integration/indicator-record/calculations/indicator-records.service.spec.ts @@ -247,6 +247,8 @@ describe('Impact Calculator Tests', () => { [INDICATOR_NAME_CODES.FLIL]: 0.3, [INDICATOR_NAME_CODES.ENL]: 0.1, [INDICATOR_NAME_CODES.GHG_FARM]: 0.2, + [INDICATOR_NAME_CODES.WW]: 0.3, + [INDICATOR_NAME_CODES.WC]: 0.4, }; //ACT From 344b2ac6a5c4066aa56710d4b24872bbc4c5ab50 Mon Sep 17 00:00:00 2001 From: alexeh Date: Fri, 13 Dec 2024 05:26:50 +0100 Subject: [PATCH 5/9] renaming for better readability --- .../indicator-records.module.ts | 6 +++--- .../services/impact-calculator.service.ts | 5 ++--- .../indicator-dependency-manager.service.ts | 19 ++++++++++++++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/api/src/modules/indicator-records/indicator-records.module.ts b/api/src/modules/indicator-records/indicator-records.module.ts index 29a0dc4e0..0b9673101 100644 --- a/api/src/modules/indicator-records/indicator-records.module.ts +++ b/api/src/modules/indicator-records/indicator-records.module.ts @@ -9,7 +9,7 @@ import { MaterialsModule } from 'modules/materials/materials.module'; import { SourcingRecordsModule } from 'modules/sourcing-records/sourcing-records.module'; import { CachedDataModule } from 'modules/cached-data/cached-data.module'; import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service'; -import { IndicatorQueryDependencyManager } from 'modules/indicator-records/services/indicator-dependency-manager.service'; +import { ImpactQueryBuilder } from 'modules/indicator-records/services/indicator-dependency-manager.service'; import { IndicatorRecordRepository } from 'modules/indicator-records/indicator-record.repository'; @Module({ @@ -26,13 +26,13 @@ import { IndicatorRecordRepository } from 'modules/indicator-records/indicator-r IndicatorRecordsService, IndicatorRecordRepository, ImpactCalculator, - IndicatorQueryDependencyManager, + ImpactQueryBuilder, ], exports: [ IndicatorRecordsService, IndicatorRecordRepository, ImpactCalculator, - IndicatorQueryDependencyManager, + ImpactQueryBuilder, ], }) export class IndicatorRecordsModule {} diff --git a/api/src/modules/indicator-records/services/impact-calculator.service.ts b/api/src/modules/indicator-records/services/impact-calculator.service.ts index 8f97dcb63..2075a5a05 100644 --- a/api/src/modules/indicator-records/services/impact-calculator.service.ts +++ b/api/src/modules/indicator-records/services/impact-calculator.service.ts @@ -22,7 +22,6 @@ import { IndicatorRecordCalculatedValuesDto } from 'modules/indicator-records/dt import { MaterialsToH3sService } from 'modules/materials/materials-to-h3s.service'; import { IndicatorsService } from 'modules/indicators/indicators.service'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; -import { H3DataService } from 'modules/h3-data/h3-data.service'; import { ImpactQueryBuilder } from 'modules/indicator-records/services/indicator-dependency-manager.service'; import { CachedDataService } from 'modules/cached-data/cached-data.service'; import { @@ -30,8 +29,8 @@ import { CachedData, } from 'modules/cached-data/cached-data.entity'; import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter'; -import { ImpactCalculationProgressTracker } from '../../impact/progress-tracker/impact-calculation.progress-tracker'; -import { ImportProgressTrackerFactory } from '../../events/import-data/import-progress.tracker.factory'; +import { ImpactCalculationProgressTracker } from 'modules/impact/progress-tracker/impact-calculation.progress-tracker'; +import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory'; /** * @description: This is PoC (Proof of Concept) for the updated LG methodology v0.1 diff --git a/api/src/modules/indicator-records/services/indicator-dependency-manager.service.ts b/api/src/modules/indicator-records/services/indicator-dependency-manager.service.ts index f3889d926..e4bc5856c 100644 --- a/api/src/modules/indicator-records/services/indicator-dependency-manager.service.ts +++ b/api/src/modules/indicator-records/services/indicator-dependency-manager.service.ts @@ -12,11 +12,20 @@ type IndicatorNameCodeToQuery = { [key in ImpactQueryPropertyName]?: ImpactPropertyToQueryFunction; }; +/** + * @description: Dynamically builds the query for calculating the raw impact values. Based on which indicators are activated in the platform, + * it gets all dependencies for each indicator, defined in the queryMap, and builds the stringified query that then will injected withint the main query. + */ + @Injectable() -export class IndicatorQueryDependencyManager { +export class ImpactQueryBuilder { queryMap: typeof INDICATOR_NAME_CODE_TO_QUERY_MAP = INDICATOR_NAME_CODE_TO_QUERY_MAP; + /** + * @description: Builds the query for the intervention impact calculation + */ + buildQueryForIntervention(nameCodes: INDICATOR_NAME_CODES[]): string { const queries: ImpactQueryExpression[] = []; for (const nameCode of nameCodes) { @@ -29,6 +38,14 @@ export class IndicatorQueryDependencyManager { return [...new Set(queries)].join(', '); } + /** + * @description: Builds the query for the import impact calculation. Since for the main query whole columns will be used instead of specific value: + * i.e: All geoRegion and material Ids from sourcing locations, instead of specific Ids, we need to quote wrap the table plus column names so that can be + * injected into the main query. + * + * @returns: Object with the params and query string that will be injected into the main query. + */ + buildQueryForImport(nameCodes: INDICATOR_NAME_CODES[]): { params: string; query: string; From 175273fb090ceb64939f486602ca48b32e2e0786 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sat, 14 Dec 2024 05:15:20 +0100 Subject: [PATCH 6/9] force parse material sheet hs_2017_code to string --- .../validation/validators/material.sheet-validator.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts index 7d65757da..ac23a45db 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts @@ -1,3 +1,4 @@ +import { Transform } from 'class-transformer'; import { IsEnum, IsJSON, @@ -10,6 +11,7 @@ import { //TODO: double check if we only use the material sheet in the excel to activate materials export class MaterialSheetValidator { + @Transform(({ value }) => String(value)) @IsString({ message: 'Material hs_2017_code must be a string' }) @MinLength(1, { message: 'Material hs_2017_code is too short' }) @IsNotEmpty({ message: 'Material hs_2017_code must not be empty' }) From 895c3c4622308388dfdc3d925aa13f52ee1a11d0 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sat, 14 Dec 2024 05:15:45 +0100 Subject: [PATCH 7/9] sheet location address input length >1 instead of >2 --- .../validation/validators/address-input.custom.validator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts index 4ef76533d..c9c079969 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts @@ -35,7 +35,7 @@ export class LocationAddressInputValidator (!(args.object as SourcingDataExcelValidator).location_latitude_input || !(args.object as SourcingDataExcelValidator).location_longitude_input) ) { - return typeof addressInput === 'string' && addressInput.length > 2; + return typeof addressInput === 'string' && addressInput.length > 1; } else { return true; } From d9dc357ccfc477f2b6f7b48e8000de6146f0df74 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sat, 14 Dec 2024 05:38:01 +0100 Subject: [PATCH 8/9] force remove queue jobs on fail and complete --- api/src/modules/import-data/import-data.module.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/modules/import-data/import-data.module.ts b/api/src/modules/import-data/import-data.module.ts index fe01a8744..6c62a9537 100644 --- a/api/src/modules/import-data/import-data.module.ts +++ b/api/src/modules/import-data/import-data.module.ts @@ -36,6 +36,10 @@ import { SourcingDataDbCleaner } from 'modules/import-data/sourcing-data/sourcin }), BullModule.registerQueue({ name: importQueueName, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + }, }), BullModule.registerQueue({ name: 'eudr', From f193c75d6d6fe9729d461291c871727b5a460d55 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sat, 14 Dec 2024 05:43:04 +0100 Subject: [PATCH 9/9] fix missing row in geocoding --- api/src/modules/geo-coding/geo-coding.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/modules/geo-coding/geo-coding.service.ts b/api/src/modules/geo-coding/geo-coding.service.ts index da513aab4..e97ebb3b1 100644 --- a/api/src/modules/geo-coding/geo-coding.service.ts +++ b/api/src/modules/geo-coding/geo-coding.service.ts @@ -93,7 +93,7 @@ export class GeoCodingService extends GeoCodingAbstractClass { progressTracker.trackProgress(); } catch (e: any) { errors.push({ - line: i + 5, + row: i + 5, error: e.message, type: 'geo-coding-error', sheet: 'sourcingData',