Skip to content

Commit

Permalink
remove old methodology dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexeh
alexeh committed Sep 4, 2023
1 parent 8ffdc7a commit 6d0a6f0
Show file tree
Hide file tree
Showing 37 changed files with 373 additions and 1,307 deletions.
8 changes: 5 additions & 3 deletions api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { ScenariosService } from 'modules/scenarios/scenarios.service';
import { IndicatorsService } from 'modules/indicators/indicators.service';
import { Indicator } from 'modules/indicators/indicator.entity';
import { ImpactService } from 'modules/impact/impact.service';
import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service';

export interface LocationData {
locationAddressInput?: string;
Expand Down Expand Up @@ -74,11 +75,12 @@ export class SourcingDataImportService {
protected readonly fileService: FileService<SourcingRecordsSheets>,
protected readonly dtoProcessor: SourcingRecordsDtoProcessorService,
protected readonly geoCodingService: GeoCodingAbstractClass,
protected readonly indicatorRecordsService: IndicatorRecordsService,
protected readonly tasksService: TasksService,
protected readonly scenarioService: ScenariosService,
protected readonly indicatorService: IndicatorsService,
protected readonly indicatorRecordService: IndicatorRecordsService,
protected readonly impactService: ImpactService,
protected readonly impactCalculator: ImpactCalculator,
) {}

async importSourcingData(filePath: string, taskId: string): Promise<any> {
Expand Down Expand Up @@ -184,7 +186,7 @@ export class SourcingDataImportService {
// Getting H3 data for calculations is done within DB so we need to improve the error handling
// TBD: What to do when there is no H3 for a Material
try {
await this.indicatorRecordsService.calculateImpactWithNewMethodology(
await this.impactCalculator.calculateImpactForAllSourcingRecords(
activeIndicators,
);

Expand Down Expand Up @@ -245,7 +247,7 @@ export class SourcingDataImportService {
await this.indicatorService.deactivateAllIndicators();
await this.materialService.deactivateAllMaterials();
await this.scenarioService.clearTable();
await this.indicatorRecordsService.clearTable();
await this.indicatorRecordService.clearTable();
await this.businessUnitService.clearTable();
await this.supplierService.clearTable();
await this.sourcingLocationService.clearTable();
Expand Down
58 changes: 9 additions & 49 deletions api/src/modules/indicator-coefficients/dto/indicator-coefficients.dto.ts
View file
Original file line number Diff line number Diff line change
@@ -1,100 +1,60 @@
import {
INDICATOR_TYPES,
INDICATOR_TYPES_NEW,
} from 'modules/indicators/indicator.entity';
import { INDICATOR_TYPES } from 'modules/indicators/indicator.entity';
import { IsNotEmpty, IsNumber, IsOptional, Max, Min } from 'class-validator';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { ApiPropertyOptional } from '@nestjs/swagger';

export class IndicatorCoefficientsDto {
@ApiProperty()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES.DEFORESTATION]: number;

@ApiProperty()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES.UNSUSTAINABLE_WATER_USE]: number;

@ApiProperty()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES.BIODIVERSITY_LOSS]: number;

@ApiProperty()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES.CARBON_EMISSIONS]: number;

@ApiProperty()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES.CARBON_EMISSIONS]: number;
}

export class IndicatorCoefficientsDtoV2 {
@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNumber()
[INDICATOR_TYPES_NEW.WATER_USE]: number;
[INDICATOR_TYPES.WATER_USE]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.UNSUSTAINABLE_WATER_USE]: number;
[INDICATOR_TYPES.UNSUSTAINABLE_WATER_USE]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.CLIMATE_RISK]: number;
[INDICATOR_TYPES.CLIMATE_RISK]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.DEFORESTATION_RISK]: number;
[INDICATOR_TYPES.DEFORESTATION_RISK]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.LAND_USE]: number;
[INDICATOR_TYPES.LAND_USE]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.SATELLIGENCE_DEFORESTATION]: number;
[INDICATOR_TYPES.SATELLIGENCE_DEFORESTATION]: number;

@ApiPropertyOptional()
@IsOptional()
@Max(1000000)
@Min(0)
@IsNotEmpty()
@IsNumber()
[INDICATOR_TYPES_NEW.SATELLIGENCE_DEFORESTATION_RISK]: number;
[INDICATOR_TYPES.SATELLIGENCE_DEFORESTATION_RISK]: number;
}
15 changes: 1 addition & 14 deletions api/src/modules/indicator-records/dto/indicator-record-calculated-values.dto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@
* on the fly
* @note: This could be probably done within the DB in the future
*/
import {
INDICATOR_TYPES,
INDICATOR_TYPES_NEW,
} from 'modules/indicators/indicator.entity';
import { INDICATOR_TYPES } from 'modules/indicators/indicator.entity';

export class IndicatorRecordCalculatedValuesDto {
sourcingRecordId: string;
Expand All @@ -17,13 +14,3 @@ export class IndicatorRecordCalculatedValuesDto {
landUse: number;
values: Map<INDICATOR_TYPES, number>;
}

export class IndicatorRecordCalculatedValuesDtoV2 {
sourcingRecordId: string;
production: number;
materialH3DataId: string;

landPerTon: number;
landUse: number;
values: Map<INDICATOR_TYPES_NEW, number>;
}
145 changes: 2 additions & 143 deletions api/src/modules/indicator-records/indicator-record.repository.ts
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,7 @@
import { DataSource, SelectQueryBuilder } from 'typeorm';
import { DataSource } from 'typeorm';
import { IndicatorRecord } from 'modules/indicator-records/indicator-record.entity';
import {
Injectable,
Logger,
ServiceUnavailableException,
} from '@nestjs/common';
import { SourcingRecordsWithIndicatorRawDataDto } from 'modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto';
import { MissingH3DataError } from 'modules/indicator-records/errors/missing-h3-data.error';
import { H3Data } from 'modules/h3-data/h3-data.entity';
import { INDICATOR_TYPES } from 'modules/indicators/indicator.entity';
import { MATERIAL_TO_H3_TYPE } from 'modules/materials/material-to-h3.entity';
import { IndicatorRecordValueSQLStrategies } from 'modules/indicator-records/strategies/indicator-record-value.strategies';
import { Injectable, Logger } from '@nestjs/common';
import { AppBaseRepository } from 'utils/app-base.repository';
import { IMPACT_VIEW_NAME } from 'modules/impact/views/impact.materialized-view.entity';

@Injectable()
export class IndicatorRecordRepository extends AppBaseRepository<IndicatorRecord> {
Expand All @@ -21,134 +10,4 @@ export class IndicatorRecordRepository extends AppBaseRepository<IndicatorRecord
}

logger: Logger = new Logger(IndicatorRecordRepository.name);

async getIndicatorRawDataForAllSourcingRecords(): Promise<
SourcingRecordsWithIndicatorRawDataDto[]
> {
try {
// TODO due to possible performance issues this query that makes use of the stored procedures for
// indicator value calculation has not been refactored. It remains to be reworked
const response: any = await this.query(`
SELECT
-- TODO: Hack to retrieve 1 materialH3Id for each sourcingRecord. This should include a year fallback strategy in the stored procedures
-- used below
distinct on (sr.id)
sr.id as "sourcingRecordId",
sr.tonnage,
sr.year,
slwithmaterialh3data.id as "sourcingLocationId",
slwithmaterialh3data.production,
slwithmaterialh3data."harvestedArea",
slwithmaterialh3data."rawDeforestation",
slwithmaterialh3data."rawBiodiversity",
slwithmaterialh3data."rawCarbon",
slwithmaterialh3data."rawWater",
slwithmaterialh3data."materialH3DataId"
FROM
sourcing_records sr
INNER JOIN
(
SELECT
sourcing_location.id,
sum_material_over_georegion(sourcing_location."geoRegionId", sourcing_location."materialId", 'producer') as production,
sum_material_over_georegion(sourcing_location."geoRegionId", sourcing_location."materialId", 'harvest') as "harvestedArea",
sum_weighted_deforestation_over_georegion(sourcing_location."geoRegionId", sourcing_location."materialId", 'harvest') as "rawDeforestation",
sum_weighted_biodiversity_over_georegion(sourcing_location."geoRegionId", sourcing_location."materialId", 'harvest') as "rawBiodiversity",
sum_weighted_carbon_over_georegion(sourcing_location."geoRegionId", sourcing_location."materialId", 'harvest') as "rawCarbon",
sum_weighted_water_over_georegion(sourcing_location."geoRegionId") as "rawWater",
"scenarioInterventionId",
"interventionType",
mth."h3DataId" as "materialH3DataId"
FROM
sourcing_location
inner join
material_to_h3 mth
on
mth."materialId" = sourcing_location."materialId"
WHERE "scenarioInterventionId" IS NULL
AND "interventionType" IS NULL
and mth."type" = 'harvest'
) as slwithmaterialh3data
on sr."sourcingLocationId" = slwithmaterialh3data.id`);
if (!response.length)
this.logger.warn(
`Could not retrieve Sourcing Records with weighted indicator values`,
);

return response;
} catch (err: any) {
this.logger.error(
`Error querying data from DB to calculate Indicator Records: ${err.message}`,
);
throw new MissingH3DataError(
`Could net retrieve Indicator Raw data from Sourcing Locations: ${err}`,
);
}
}

/**
* Calculates the impact raw value for the given indicator type and georegion, and basing the calculations
* on the provided H3 Data of materials and indicators (since the calculation might have dependencies on other
* indicators)
* @param geoRegionId
* @param indicatorType
* @param indicatorH3s
* @param materialH3s
*/
async getIndicatorRecordRawValue(
geoRegionId: string,
indicatorType: INDICATOR_TYPES,
indicatorH3s: Map<INDICATOR_TYPES, H3Data>,
materialH3s: Map<MATERIAL_TO_H3_TYPE, H3Data>,
): Promise<number> {
const query: SelectQueryBuilder<any> = IndicatorRecordValueSQLStrategies[
indicatorType
](this.dataSource, geoRegionId, indicatorH3s, materialH3s);

try {
const res: { sum: number }[] = await query.getRawMany();
return res[0].sum;
} catch (error: any) {
this.logger.error(
`Could not calculate raw Indicator values for GeoRegion Id: ${geoRegionId} and Indicator ${indicatorType} - ${error}`,
);
throw new ServiceUnavailableException(
`Could not calculate Raw Indicator values for new Scenario: ${error.message}`,
);
}
}

/**
* Returns the sum of h3 value of a given H3 Data over a georegion
* @param geoRegionId
* @param h3Data
*/
async getH3SumOverGeoRegionSQL(
geoRegionId: string,
h3Data: H3Data,
): Promise<number> {
const query: string = `SELECT sum(h3Table."${h3Data.h3columnName}")
FROM
get_h3_uncompact_geo_region('${geoRegionId}', ${h3Data.h3resolution}) geo_region
INNER JOIN
"${h3Data.h3tableName}" h3Table ON geo_region.h3index = h3Table.h3index`;

try {
const res: any[] = await this.query(query);
return res[0].sum;
} catch (error: any) {
this.logger.error(
`Could not calculate raw Indicator values for GeoRegion Id: ${geoRegionId} and H3 Data Id: ${h3Data.id} `,
);
throw new ServiceUnavailableException(
`Could not calculate H3 sum over georegion: ${error.message}`,
);
}
}

async updateImpactView(): Promise<void> {
return this.dataSource.query(
`REFRESH MATERIALIZED VIEW CONCURRENTLY ${IMPACT_VIEW_NAME} WITH DATA`,
);
}
}
Loading

0 comments on commit 6d0a6f0

Please sign in to comment.