Skip to content

Commit

Permalink
Merge pull request #572 from gridsingularity/feature/GSYE-846
Browse files Browse the repository at this point in the history 
GSYE-846: move query from Entsoye logic to gsy-web
  • Loading branch information
@BigTava
BigTava authored Feb 13, 2025
2 parents 7aade29 + 511b5da commit c7dead8
Show file tree
Hide file tree
Showing 10 changed files with 426 additions and 577 deletions.
175 changes: 108 additions & 67 deletions gsy_framework/read_user_profile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

import ast
import csv
import os
Expand All @@ -24,11 +25,20 @@
from pendulum import DateTime, duration, from_format, from_timestamp, today

from gsy_framework.constants_limits import (
DATE_TIME_FORMAT, DATE_TIME_FORMAT_SECONDS, TIME_FORMAT, TIME_ZONE, GlobalConfig)
DATE_TIME_FORMAT,
DATE_TIME_FORMAT_SECONDS,
TIME_FORMAT,
TIME_ZONE,
GlobalConfig,
)
from gsy_framework.exceptions import GSyReadProfileException
from gsy_framework.utils import (
convert_kW_to_kWh, get_from_profile_same_weekday_and_time, generate_market_slot_list,
return_ordered_dict, convert_kWh_to_W)
convert_kW_to_kWh,
get_from_profile_same_weekday_and_time,
generate_market_slot_list,
return_ordered_dict,
convert_kWh_to_W,
)

DATE_TIME_FORMAT_SPACED = "YYYY-MM-DD HH:mm:ss"

Expand All @@ -45,10 +55,12 @@ class InputProfileTypes(Enum):
REBASE_W = 3 # Profile power values in W from REBASE API
# (deprecated; only kept for old profiles in DB)
ENERGY_KWH = 4
CARBON_RATIO_G_KWH = 5 # Carbon ratio in gCO2eq/kWh


class LiveProfileTypes(Enum):
"""Enum for type of live data profiles"""

NO_LIVE_DATA = 0
FORECAST = 1
MEASUREMENT = 2
Expand All @@ -65,7 +77,8 @@ def _str_to_datetime(time_string, time_format) -> DateTime:
return time
if time_format == TIME_FORMAT:
return GlobalConfig.start_date.add(
hours=time.hour, minutes=time.minute, seconds=time.second)
hours=time.hour, minutes=time.minute, seconds=time.second
)
raise GSyReadProfileException("Provided time_format invalid.")


Expand All @@ -76,9 +89,10 @@ def default_profile_dict(val=None, current_timestamp=None) -> Dict[DateTime, int
"""
if val is None:
val = 0
return {time_slot: val
for time_slot in
generate_market_slot_list(start_timestamp=current_timestamp)}
return {
time_slot: val
for time_slot in generate_market_slot_list(start_timestamp=current_timestamp)
}


def is_number(number):
Expand Down Expand Up @@ -118,14 +132,19 @@ def _eval_time_format(time_dict: Dict) -> str:
:return: TIME_FORMAT or DATE_TIME_FORMAT or DATE_TIME_FORMAT_SECONDS
"""

for time_format in [TIME_FORMAT, DATE_TIME_FORMAT, DATE_TIME_FORMAT_SECONDS,
DATE_TIME_FORMAT_SPACED]:
for time_format in [
TIME_FORMAT,
DATE_TIME_FORMAT,
DATE_TIME_FORMAT_SECONDS,
DATE_TIME_FORMAT_SPACED,
]:
if _eval_single_format(time_dict, time_format):
return time_format

raise GSyReadProfileException(
f"Format of time-stamp is not one of ('{TIME_FORMAT}', "
f"'{DATE_TIME_FORMAT}', '{DATE_TIME_FORMAT_SECONDS}')")
f"'{DATE_TIME_FORMAT}', '{DATE_TIME_FORMAT_SECONDS}')"
)


def _read_csv(path: str) -> Dict:
Expand All @@ -141,16 +160,19 @@ def _read_csv(path: str) -> Dict:
for row in csv_rows:
if len(row) == 0:
raise GSyReadProfileException(
f"There must not be an empty line in the profile file {path}")
f"There must not be an empty line in the profile file {path}"
)
if len(row) != 2:
row = row[0].split(";")
try:
profile_data[row[0]] = float(row[1])
except ValueError:
pass
time_format = _eval_time_format(profile_data)
return dict((_str_to_datetime(time_str, time_format), value)
for time_str, value in profile_data.items())
return dict(
(_str_to_datetime(time_str, time_format), value)
for time_str, value in profile_data.items()
)


def _interpolate_profile_values_to_slot(profile_data_W, slot_length):
Expand All @@ -165,12 +187,11 @@ def _interpolate_profile_values_to_slot(profile_data_W, slot_length):
input_power_list_W = [float(dp) for dp in profile_data_W.values()]

time0 = from_timestamp(0)
input_time_seconds_list = [(ti - time0).in_seconds()
for ti in input_time_list]
input_time_seconds_list = [(ti - time0).in_seconds() for ti in input_time_list]

slot_time_list = list(range(input_time_seconds_list[0], input_time_seconds_list[-1],
slot_length.in_seconds())
)
slot_time_list = list(
range(input_time_seconds_list[0], input_time_seconds_list[-1], slot_length.in_seconds())
)

second_power_list_W = [
input_power_list_W[index - 1]
Expand All @@ -182,14 +203,14 @@ def _interpolate_profile_values_to_slot(profile_data_W, slot_length):
for index, _ in enumerate(slot_time_list):
first_index = index * slot_length.in_seconds()
if first_index <= len(second_power_list_W):
avg_power_kW.append(second_power_list_W[first_index] / 1000.)
avg_power_kW.append(second_power_list_W[first_index] / 1000.0)

return avg_power_kW, slot_time_list


def _calculate_energy_from_power_profile(
profile_data_W: Dict[DateTime, float],
slot_length: duration) -> Dict[DateTime, float]:
profile_data_W: Dict[DateTime, float], slot_length: duration
) -> Dict[DateTime, float]:
# pylint: disable=invalid-name
"""
Calculates energy from power profile. Does not use numpy, calculates avg power for each
Expand All @@ -202,9 +223,9 @@ def _calculate_energy_from_power_profile(
avg_power_kW, slot_time_list = _interpolate_profile_values_to_slot(profile_data_W, slot_length)
slot_energy_kWh = list(map(lambda x: convert_kW_to_kWh(x, slot_length), avg_power_kW))

return {from_timestamp(slot_time_list[ii]): energy
for ii, energy in enumerate(slot_energy_kWh)
}
return {
from_timestamp(slot_time_list[ii]): energy for ii, energy in enumerate(slot_energy_kWh)
}


def _fill_gaps_in_profile(input_profile: Dict = None, out_profile: Dict = None) -> Dict:
Expand All @@ -225,8 +246,9 @@ def _fill_gaps_in_profile(input_profile: Dict = None, out_profile: Dict = None)

for time in out_profile.keys():
if time not in input_profile:
temp_val = get_from_profile_same_weekday_and_time(input_profile, time,
ignore_not_found=True)
temp_val = get_from_profile_same_weekday_and_time(
input_profile, time, ignore_not_found=True
)
if temp_val is not None:
current_val = temp_val
else:
Expand All @@ -237,7 +259,8 @@ def _fill_gaps_in_profile(input_profile: Dict = None, out_profile: Dict = None)


def _read_from_different_sources_todict(
input_profile: Any, current_timestamp: DateTime = None) -> Dict[DateTime, float]:
input_profile: Any, current_timestamp: DateTime = None
) -> Dict[DateTime, float]:
"""
Reads arbitrary profile.
Handles csv, dict and string input.
Expand All @@ -262,8 +285,7 @@ def _read_from_different_sources_todict(
profile.pop("filename", None)
profile = _remove_header(profile)
time_format = _eval_time_format(profile)
profile = {_str_to_datetime(key, time_format): val
for key, val in profile.items()}
profile = {_str_to_datetime(key, time_format): val for key, val in profile.items()}
elif isinstance(list(input_profile.keys())[0], DateTime):
return input_profile

Expand All @@ -273,28 +295,28 @@ def _read_from_different_sources_todict(
# Remove filename from profile
input_profile.pop("filename", None)
time_format = _eval_time_format(input_profile)
profile = {_str_to_datetime(key, time_format): val
for key, val in input_profile.items()}
profile = {
_str_to_datetime(key, time_format): val for key, val in input_profile.items()
}

elif isinstance(list(input_profile.keys())[0], (float, int)):
# input is hourly profile

profile = dict(
(today(tz=TIME_ZONE).add(hours=hour), val)
for hour, val in input_profile.items()
(today(tz=TIME_ZONE).add(hours=hour), val) for hour, val in input_profile.items()
)

else:
raise GSyReadProfileException(
"Unsupported input type : " + str(list(input_profile.keys())[0]))
"Unsupported input type : " + str(list(input_profile.keys())[0])
)

elif isinstance(input_profile, (float, int, tuple)):
# input is single value
profile = default_profile_dict(val=input_profile, current_timestamp=current_timestamp)

else:
raise GSyReadProfileException(
f"Unsupported input type: {str(input_profile)}")
raise GSyReadProfileException(f"Unsupported input type: {str(input_profile)}")

return profile

Expand All @@ -304,9 +326,11 @@ def _hour_time_str(hour: float, minute: float) -> str:


def _copy_profile_to_multiple_days(
in_profile: Dict, current_timestamp: Optional[datetime] = None) -> Dict:
in_profile: Dict, current_timestamp: Optional[datetime] = None
) -> Dict:
daytime_dict = dict(
(_hour_time_str(time.hour, time.minute), time) for time in in_profile.keys())
(_hour_time_str(time.hour, time.minute), time) for time in in_profile.keys()
)
out_profile = {}
for slot_time in generate_market_slot_list(start_timestamp=current_timestamp):
if slot_time not in out_profile.keys():
Expand All @@ -317,9 +341,9 @@ def _copy_profile_to_multiple_days(


@return_ordered_dict
def read_arbitrary_profile(profile_type: InputProfileTypes,
input_profile,
current_timestamp: DateTime = None) -> Optional[Dict[DateTime, float]]:
def read_arbitrary_profile(
profile_type: InputProfileTypes, input_profile, current_timestamp: DateTime = None
) -> Optional[Dict[DateTime, float]]:
"""
Reads arbitrary profile.
Handles csv, dict and string input.
Expand All @@ -334,24 +358,32 @@ def read_arbitrary_profile(profile_type: InputProfileTypes,
"""
if input_profile in [{}, None]:
return {}
profile = _read_from_different_sources_todict(input_profile,
current_timestamp=current_timestamp)
profile = _read_from_different_sources_todict(
input_profile, current_timestamp=current_timestamp
)
profile_time_list = list(profile.keys())
profile_duration = profile_time_list[-1] - profile_time_list[0]
if ((GlobalConfig.sim_duration > duration(days=1) >= profile_duration) or
GlobalConfig.is_canary_network()):
if (
GlobalConfig.sim_duration > duration(days=1) >= profile_duration
) or GlobalConfig.is_canary_network():
profile = _copy_profile_to_multiple_days(profile, current_timestamp=current_timestamp)

if profile is not None:
if profile_type is InputProfileTypes.ENERGY_KWH:
profile = {ts: convert_kWh_to_W(energy, GlobalConfig.slot_length)
for ts, energy in profile.items()}
profile = {
ts: convert_kWh_to_W(energy, GlobalConfig.slot_length)
for ts, energy in profile.items()
}
if profile_type is InputProfileTypes.CARBON_RATIO_G_KWH:
return profile
zero_value_slot_profile = default_profile_dict(current_timestamp=current_timestamp)
filled_profile = _fill_gaps_in_profile(profile, zero_value_slot_profile)
if profile_type in [InputProfileTypes.POWER_W, InputProfileTypes.REBASE_W,
InputProfileTypes.ENERGY_KWH]:
return _calculate_energy_from_power_profile(
filled_profile, GlobalConfig.slot_length)
if profile_type in [
InputProfileTypes.POWER_W,
InputProfileTypes.REBASE_W,
InputProfileTypes.ENERGY_KWH,
]:
return _calculate_energy_from_power_profile(filled_profile, GlobalConfig.slot_length)
return filled_profile
return None

Expand Down Expand Up @@ -384,21 +416,22 @@ def read_profile_without_config(input_profile: Dict, slot_length_mins=15) -> Dic
profile_values, slots = _interpolate_profile_values_to_slot(
filled_profile, duration(minutes=slot_length_mins)
)
return {
from_timestamp(slots[ii]): energy
for ii, energy in enumerate(profile_values)
}
return {from_timestamp(slots[ii]): energy for ii, energy in enumerate(profile_values)}

raise GSyReadProfileException(
"Profile file cannot be read successfully. Please reconfigure the file path.")
"Profile file cannot be read successfully. Please reconfigure the file path."
)


def _generate_time_slots(
slot_length: timedelta, sim_duration: timedelta, start_date: datetime
slot_length: timedelta, sim_duration: timedelta, start_date: datetime
) -> Generator:
return (start_date + timedelta(seconds=slot_length.total_seconds() * time_diff_count)
for time_diff_count in range(
int(sim_duration.total_seconds() / slot_length.total_seconds())))
return (
start_date + timedelta(seconds=slot_length.total_seconds() * time_diff_count)
for time_diff_count in range(
int(sim_duration.total_seconds() / slot_length.total_seconds())
)
)


def _get_from_profile(profile: Dict[datetime, float], key: datetime) -> float:
Expand All @@ -409,8 +442,11 @@ def _get_from_profile(profile: Dict[datetime, float], key: datetime) -> float:


def resample_hourly_energy_profile(
input_profile: Dict[DateTime, float], slot_length: timedelta, sim_duration: timedelta,
start_date: datetime) -> Dict[DateTime, float]:
input_profile: Dict[DateTime, float],
slot_length: timedelta,
sim_duration: timedelta,
start_date: datetime,
) -> Dict[DateTime, float]:
"""Resample hourly energy profile in order to fit to the set slot_length."""
slot_length_minutes = slot_length.total_seconds() / 60
if slot_length_minutes < 60:
Expand All @@ -419,8 +455,13 @@ def resample_hourly_energy_profile(
scaling_factor = 60 / slot_length_minutes
out_dict = {}
for time_slot in _generate_time_slots(slot_length, sim_duration, start_date):
hour_time_slot = datetime(time_slot.year, time_slot.month, time_slot.day,
time_slot.hour, tzinfo=time_slot.tzinfo)
hour_time_slot = datetime(
time_slot.year,
time_slot.month,
time_slot.day,
time_slot.hour,
tzinfo=time_slot.tzinfo,
)
out_dict[time_slot] = _get_from_profile(input_profile, hour_time_slot) / scaling_factor
return out_dict
if slot_length_minutes > 60:
Expand All @@ -429,9 +470,9 @@ def resample_hourly_energy_profile(
number_of_aggregated_slots = int(slot_length_minutes / 60)
return {
time_slot: sum(
_get_from_profile(
input_profile, time_slot.add(minutes=slot_length_minutes * nn))
for nn in range(number_of_aggregated_slots))
_get_from_profile(input_profile, time_slot.add(minutes=slot_length_minutes * nn))
for nn in range(number_of_aggregated_slots)
)
for time_slot in _generate_time_slots(slot_length, sim_duration, start_date)
}
return input_profile
Loading

0 comments on commit c7dead8

Please sign in to comment.