Skip to content
This repository has been archived by the owner on Oct 27, 2024. It is now read-only.

Commit

Permalink
Merge pull request #27 from rodekruis/uga-multi-thresholds
Browse files Browse the repository at this point in the history 
Uganda multi thresholds
  • Loading branch information
@jmargutt
jmargutt authored May 16, 2024
2 parents dd4bda2 + baa5390 commit 66bcb17
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 210 deletions.
102 changes: 48 additions & 54 deletions pipeline/lib/flood_model/dynamicDataDb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def getDisasterType(self):
disasterType = 'floods'
return disasterType


def uploadCalculatedAffected2(self):
for indicator, values in self.EXPOSURE_DATA_SOURCES.items():
adminlevels=self.admin_level
Expand All @@ -79,6 +80,7 @@ def uploadCalculatedAffected2(self):
self.apiPostRequest('admin-area-dynamic-data/exposure', body=body)
logger.info('Uploaded calculated_affected for indicator: ' + 'population_affected_percentage for admin level: ' + str(adminlevels))


def uploadCalculatedAffected(self):
for adminlevels in SETTINGS[self.countryCodeISO3]['levels']:#range(1,self.admin_level+1):
for indicator, values in self.EXPOSURE_DATA_SOURCES.items():
Expand Down Expand Up @@ -117,9 +119,7 @@ def uploadCalculatedAffected(self):
self.apiPostRequest('admin-area-dynamic-data/exposure', body=body)
logger.info(f'Uploaded calculated_affected for indicator: {indicator}' +' for admin level: ' + str(adminlevels))



if self.countryCodeISO3 == 'MWII':
if self.countryCodeISO3 == 'MWI':
for indicator, values in self.EXPOSURE_DATA_UBR_SOURCES.items():
affectedIndicatorPath = self.affectedFolder + \
'affected_' + self.leadTimeLabel + '_' + self.countryCodeISO3 + '_admin_' + str(adminlevels) + '_' + indicator + '.json'
Expand All @@ -130,17 +130,18 @@ def uploadCalculatedAffected(self):
self.apiPostRequest('admin-area-dynamic-data/exposure', body=body)
logger.info('Uploaded calculated_affected for indicator: ' + f'{indicator} for admin level: ' + str(adminlevels))


def uploadRasterFile(self):
disasterType = self.getDisasterType()
rasterFile = RASTER_OUTPUT + '0/flood_extents/flood_extent_' + self.leadTimeLabel + '_' + self.countryCodeISO3 + '.tif'
files = {'file': open(rasterFile,'rb')}
self.apiPostRequest('admin-area-dynamic-data/raster/' + disasterType, files=files)
logger.info(f'Uploaded raster-file: {rasterFile}')


def uploadImage(self,eventName='no-name'):
disasterType = self.getDisasterType()


imageFile = PIPELINE_OUTPUT + self.countryCodeISO3 + '_' +self.leadTimeLabel +'_floods-map-image.png'

files = {
Expand All @@ -167,50 +168,43 @@ def uploadTriggerPerStation(self):
with open(filename) as json_file:
triggers = json.load(json_file)

stationForecasts = []
for key in triggers:

if key['eapAlertClass']:
alertclass=key['eapAlertClass']
else:
alertclass='no'

stationForecasts.append({
"stationCode": key['stationCode'],
"forecastLevel": int(key['fc']),
"eapAlertClass": alertclass, #key['eapAlertClass'],
"forecastReturnPeriod": key['fc_rp'],
"triggerLevel": int(key['triggerLevel'])
stationForecasts = {
"forecastLevel": [],
"eapAlertClass": [],
"forecastReturnPeriod": [],
"triggerLevel": []
}

for key in triggers:
stationForecasts["forecastLevel"].append({
"fid": key['stationCode'],
"value": int(key['fc'])
})
stationForecasts["eapAlertClass"].append({
"fid": key['stationCode'],
"value": 'no' if not key['eapAlertClass'] else key['eapAlertClass']
})
stationForecasts["forecastReturnPeriod"].append({
"fid": key['stationCode'],
"value": key['fc_rp']
})
stationForecasts["triggerLevel"].append({
"fid": key['stationCode'],
"value": 0 if not key['triggerLevel'] else int(key['triggerLevel'])
})

body = {
"countryCodeISO3": self.countryCodeISO3,
"leadTime": self.leadTimeLabel,
"date": self.uploadTime,
"stationForecasts": stationForecasts
}

dfStation = pd.DataFrame(index=df.index)
dfStation['stationCode'] = df['stationCode']
dfStation['forecastLevel'] = df['fc'].astype(np.float64,errors='ignore')
#dfStation['forecastProbability'] = df['fc_prob'].astype(np.float64,errors='ignore')
#dfStation['forecastTrigger'] = df['fc_trigger'].astype(np.int32,errors='ignore')
dfStation['eapAlertClass'] = df['eapAlertClass']
dfStation['forecastReturnPeriod'] = df['fc_rp'].astype(np.int32,errors='ignore')
dfStation['triggerLevel'] = df['triggerLevel'].astype(np.int32,errors='ignore')
stationForecasts = json.loads(dfStation.to_json(orient='records'))

body2 = {
'countryCodeISO3': self.countryCodeISO3,
'leadTime': self.leadTimeLabel,
'date': self.uploadTime,
'stationForecasts': stationForecasts
}
#body['disasterType'] = self.getDisasterType()
print(body)
self.apiPostRequest('glofas-stations/triggers', body=body)
logger.info('Uploaded triggers per station')
for key in ["forecastLevel", "eapAlertClass", "forecastReturnPeriod", "triggerLevel"]:
body = {
"leadTime": self.leadTimeLabel,
"date": self.uploadTime,
"disasterType": self.getDisasterType(),
"pointDataCategory": "glofas_stations",
"key": key,
"dynamicPointData": stationForecasts[key]
}
self.apiPostRequest('point-data/dynamic', body=body)
logger.info(f'Uploaded triggers per station, key: {key}')


def uploadTriggersPerLeadTime(self):
with open(self.triggerFolder +
Expand All @@ -220,16 +214,16 @@ def uploadTriggersPerLeadTime(self):
for key in triggers:
triggersPerLeadTime.append({
'leadTime': str(key),
'triggered': triggers[key]
'triggered': triggers[key]['triggered'],
"thresholdReached": triggers[key]['thresholdReached']
})
body = {
'countryCodeISO3': self.countryCodeISO3,
'disasterType': self.getDisasterType(),
'triggersPerLeadTime': triggersPerLeadTime
}
#body['disasterType'] = self.getDisasterType()
body['date']=self.uploadTime
self.apiPostRequest('event/triggers-per-leadtime', body=body)
body = {
'countryCodeISO3': self.countryCodeISO3,
'disasterType': self.getDisasterType(),
'triggersPerLeadTime': triggersPerLeadTime
}
body['date']=self.uploadTime
self.apiPostRequest('event/triggers-per-leadtime', body=body)
logger.info('Uploaded triggers per leadTime')

def apiGetRequest(self, path, countryCodeISO3):
Expand Down
36 changes: 26 additions & 10 deletions pipeline/lib/flood_model/exposure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
import time
import logging
logger = logging.getLogger(__name__)
os.environ['GDAL_DATA'] = '/usr/local/lib/python3.8/dist-packages/fiona/gdal_data/gcs.csv'


class Exposure:

"""Class used to calculate the exposure per exposure type"""
Expand All @@ -40,29 +43,41 @@ def __init__(self, leadTimeLabel, countryCodeISO3, admin_area_gdf, population_to
return_period_='25'
self.floodExt_raster = RASTER_INPUT +'flood_extent/'+ self.countryCodeISO3 + '_flood_' +return_period_+'year.tif'


#self.image_name= PIPELINE_OUTPUT + 'Map_of_affeced_areas_'+ leadTimeLabel + '_' + countryCodeISO3 + '.png'

self.image_name = PIPELINE_OUTPUT + self.countryCodeISO3 + '_' +self.leadTimeLabel +'_floods-map-image.png'

self.popAffeced = RASTER_OUTPUT + "0/" +\
SETTINGS[countryCodeISO3]['EXPOSURE_DATA_SOURCES']['population']['source'] + \
'_' + self.leadTimeLabel + ".tif"


#self.ADMIN_AREA_GDF_TMP_PATH = os.path.join(PIPELINE_OUTPUT,"admin-areas_TMP.geojson") #s.geojson', driver='GeoJSON')
self.ADMIN_AREA_GDF_TMP_PATH = PIPELINE_OUTPUT+"admin-areas_TMP.shp"
self.EXPOSURE_DATA_SOURCES = SETTINGS[countryCodeISO3]['EXPOSURE_DATA_SOURCES']
if self.countryCodeISO3 == 'MWI':
self.EXPOSURE_DATA_UBR_SOURCES = SETTINGS[countryCodeISO3]['EXPOSURE_DATA_UBR_SOURCES']

self.triggersPerStationPath = PIPELINE_OUTPUT + \
'triggers_rp_per_station/triggers_rp_' + \
self.leadTimeLabel + '_' + countryCodeISO3 + '.json'
self.levels = SETTINGS[countryCodeISO3]['levels']
self.pcode_df=pcodes
self.db = DatabaseManager(leadTimeLabel, countryCodeISO3,admin_level)
if "population" in self.EXPOSURE_DATA_SOURCES:
self.population_total = population_total


def classifyAlertThreshold(self, triggersPerStation):
'''
Function to determine alert threshold value
'''
if any(d['eapAlertClass'] == 'max' for d in triggersPerStation):
alert_threshold = 1
elif any(d['eapAlertClass'] == 'med' for d in triggersPerStation):
alert_threshold = 0.7
elif any(d['eapAlertClass'] == 'min' for d in triggersPerStation):
alert_threshold = 0.3
elif any(d['eapAlertClass'] == 'no' for d in triggersPerStation):
alert_threshold = 0
return alert_threshold


def callAllExposure(self):
for indicator, values in self.EXPOSURE_DATA_SOURCES.items():
logger.info(f'indicator: {indicator}')
Expand All @@ -73,7 +88,6 @@ def callAllExposure(self):
stats = self.calcAffected(self.disasterExtentRaster, indicator, values['rasterValue'])
df_stats=pd.DataFrame(stats)
df_stats = pd.merge(self.pcode_df,df_stats, how='left',left_on=f"placeCode_{self.admin_level}" , right_on ='placeCode')


#stats_dff = pd.merge(df,self.pcode_df, how='left',left_on='placeCode', right_on = f'placeCode_{self.admin_level}')
for adm_level in SETTINGS[self.countryCodeISO3]['levels']:
Expand All @@ -85,7 +99,6 @@ def callAllExposure(self):
df_stats_levl.reset_index(inplace=True)
df_stats_levl['placeCode']=df_stats_levl[f'placeCode_{adm_level}']
df_stats_levl=df_stats_levl[['amount','placeCode']].to_dict(orient='records')


self.statsPath = PIPELINE_OUTPUT + 'calculated_affected/affected_' + \
self.leadTimeLabel + '_' + self.countryCodeISO3 +'_admin_' +str(adm_level) + '_' + indicator + '.json'
Expand Down Expand Up @@ -132,7 +145,7 @@ def callAllExposure(self):
with open(population_affected_percentage_file_path, 'w') as fp:
json.dump(population_affected_percentage_records, fp)

# define alert_threshold layer
# define alert_threshold layer
alert_threshold = list(map(self.get_alert_threshold, df_stats_levl))

alert_threshold_file_path = PIPELINE_OUTPUT + 'calculated_affected/affected_' + \
Expand Down Expand Up @@ -200,7 +213,10 @@ def get_alert_threshold(self, population_affected):
# population_total = next((x for x in self.population_total if x['placeCode'] == population_affected['placeCode']), None)
alert_threshold = 0
if (population_affected['amount'] > 0):
alert_threshold = 1
# alert_threshold = 1
with open(self.triggersPerStationPath) as fp:
triggersPerStation = json.load(fp)
alert_threshold = self.classifyAlertThreshold(triggersPerStation)
else:
alert_threshold = 0
return {
Expand Down
Loading

0 comments on commit 66bcb17

Please sign in to comment.