diff --git a/conda/meta.yaml b/conda/meta.yaml index d7d37749..1c071010 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -18,23 +18,24 @@ requirements: - python - setuptools run: + - bottleneck + - bw2data + - bw2io >=0.8.10 + - constructive_geometries>=0.8.2 + - cryptography + - datapackage - numpy - pandas - - bw2io >=0.8.10 - - bw2data - - wurst - - xarray + - platformdirs - prettytable + - pyarrow - pycountry - - cryptography - pyYaml - - sparse>=0.14.0 - - schema - - datapackage - requests - - bottleneck - - constructive_geometries>=0.8.2 - - pyarrow + - schema + - sparse>=0.14.0 + - wurst + - xarray test: imports: diff --git a/premise/__init__.py b/premise/__init__.py index 2f0ae4ed..af001f36 100644 --- a/premise/__init__.py +++ b/premise/__init__.py @@ -1,11 +1,6 @@ __all__ = ("NewDatabase", "clear_cache", "get_regions_definition") __version__ = (1, 7, 6) -from pathlib import Path - -DATA_DIR = Path(__file__).resolve().parent / "data" -INVENTORY_DIR = Path(__file__).resolve().parent / "data" / "additional_inventories" -VARIABLES_DIR = Path(__file__).resolve().parent / "iam_variables_mapping" from .ecoinvent_modification import NewDatabase from .utils import clear_cache, get_regions_definition diff --git a/premise/activity_maps.py b/premise/activity_maps.py index 7ae8bef9..5f6b8f86 100644 --- a/premise/activity_maps.py +++ b/premise/activity_maps.py @@ -3,7 +3,6 @@ mapping between ``premise`` and ``ecoinvent`` terminology. """ -import sys from collections import defaultdict from pathlib import Path from typing import List, Union @@ -11,7 +10,7 @@ import yaml from wurst import searching as ws -from . import DATA_DIR, VARIABLES_DIR +from .filesystem_constants import DATA_DIR, VARIABLES_DIR POWERPLANT_TECHS = VARIABLES_DIR / "electricity_variables.yaml" FUELS_TECHS = VARIABLES_DIR / "fuels_variables.yaml" diff --git a/premise/cement.py b/premise/cement.py index 47e38a7d..e044d183 100644 --- a/premise/cement.py +++ b/premise/cement.py @@ -8,15 +8,10 @@ """ -import logging.config from collections import defaultdict -from pathlib import Path - -import yaml from .logger import create_logger from .transformation import BaseTransformation, Dict, IAMDataCollection, List, np, ws -from .utils import DATA_DIR logger = create_logger("cement") diff --git a/premise/clean_datasets.py b/premise/clean_datasets.py index 5f49fdfc..af965c90 100644 --- a/premise/clean_datasets.py +++ b/premise/clean_datasets.py @@ -14,8 +14,8 @@ from bw2data.database import DatabaseChooser from wurst import searching as ws -from . import DATA_DIR from .data_collection import get_delimiter +from .filesystem_constants import DATA_DIR def remove_uncertainty(database): diff --git a/premise/data_collection.py b/premise/data_collection.py index 43a362b5..e31ea460 100644 --- a/premise/data_collection.py +++ b/premise/data_collection.py @@ -19,7 +19,7 @@ import yaml from cryptography.fernet import Fernet -from . import DATA_DIR, VARIABLES_DIR +from .filesystem_constants import DATA_DIR, IAM_OUTPUT_DIR, VARIABLES_DIR from .marginal_mixes import consequential_method IAM_ELEC_VARS = VARIABLES_DIR / "electricity_variables.yaml" @@ -30,11 +30,9 @@ IAM_STEEL_VARS = VARIABLES_DIR / "steel_variables.yaml" IAM_DAC_VARS = VARIABLES_DIR / "direct_air_capture_variables.yaml" IAM_OTHER_VARS = VARIABLES_DIR / "other_variables.yaml" -FILEPATH_FLEET_COMP = ( - DATA_DIR / "iam_output_files" / "fleet_files" / "fleet_all_vehicles.csv" -) +FILEPATH_FLEET_COMP = IAM_OUTPUT_DIR / "fleet_files" / "fleet_all_vehicles.csv" FILEPATH_IMAGE_TRUCKS_FLEET_COMP = ( - DATA_DIR / "iam_output_files" / "fleet_files" / "image_fleet_trucks.csv" + IAM_OUTPUT_DIR / "fleet_files" / "image_fleet_trucks.csv" ) VEHICLES_MAP = DATA_DIR / "transport" / "vehicles_map.yaml" IAM_CARBON_CAPTURE_VARS = VARIABLES_DIR / "carbon_capture_variables.yaml" diff --git a/premise/direct_air_capture.py b/premise/direct_air_capture.py index c715f60f..ad881e32 100644 --- a/premise/direct_air_capture.py +++ b/premise/direct_air_capture.py @@ -3,20 +3,13 @@ """ import copy -import logging.config -from pathlib import Path +import numpy as np import wurst import yaml +from .filesystem_constants import DATA_DIR from .logger import create_logger -from .utils import DATA_DIR - -logger = create_logger("dac") - - -import numpy as np - from .transformation import ( BaseTransformation, IAMDataCollection, @@ -26,6 +19,8 @@ ws, ) +logger = create_logger("dac") + HEAT_SOURCES = DATA_DIR / "fuels" / "heat_sources_map.yml" diff --git a/premise/ecoinvent_modification.py b/premise/ecoinvent_modification.py index 71be8813..90033eb2 100644 --- a/premise/ecoinvent_modification.py +++ b/premise/ecoinvent_modification.py @@ -18,7 +18,7 @@ import datapackage import yaml -from . import DATA_DIR, INVENTORY_DIR, __version__ +from . import __version__ from .cement import _update_cement from .clean_datasets import DatabaseCleaner from .data_collection import IAMDataCollection @@ -34,6 +34,7 @@ ) from .external import ExternalScenario from .external_data_validation import check_external_scenarios, check_inventories +from .filesystem_constants import DATA_DIR, DIR_CACHED_DB, IAM_OUTPUT_DIR, INVENTORY_DIR from .fuels import _update_fuels from .inventory_imports import AdditionalInventory, DefaultInventory from .report import generate_change_report, generate_summary_report @@ -51,8 +52,6 @@ write_brightway2_database, ) -DIR_CACHED_DB = DATA_DIR / "cache" - FILEPATH_OIL_GAS_INVENTORIES = INVENTORY_DIR / "lci-ESU-oil-and-gas.xlsx" FILEPATH_CARMA_INVENTORIES = INVENTORY_DIR / "lci-Carma-CCS.xlsx" FILEPATH_CO_FIRING_INVENTORIES = INVENTORY_DIR / "lci-co-firing-power-plants.xlsx" @@ -344,7 +343,8 @@ def check_scenarios(scenario: dict, key: bytes) -> dict: filepath = scenario["filepath"] scenario["filepath"] = check_filepath(filepath) else: - scenario["filepath"] = DATA_DIR / "iam_output_files" + # Note: A directory path, not a file path + scenario["filepath"] = IAM_OUTPUT_DIR if key is None: raise ValueError( "You need to provide the encryption key to decrypt the IAM output files provided by `premise`." @@ -633,13 +633,11 @@ def __find_cached_db(self, db_name: str, keep_uncertainty_data: bool) -> List[di :param db_name: database name :return: database """ - # check that directory exists, otherwise create it - Path(DIR_CACHED_DB).mkdir(parents=True, exist_ok=True) # build file path if db_name is None and self.source_type == "ecospold": db_name = f"ecospold_{self.system_model}_{self.version}" - file_name = Path( + file_name = ( DIR_CACHED_DB / f"cached_{''.join(tuple(map( str , __version__ )))}_{db_name.strip().lower()}.pickle" ) @@ -663,13 +661,11 @@ def __find_cached_inventories(self, db_name: str) -> Union[None, List[dict]]: :param db_name: database name :return: database """ - # check that directory exists, otherwise create it - Path(DIR_CACHED_DB).mkdir(parents=True, exist_ok=True) # build file path if db_name is None and self.source_type == "ecospold": db_name = f"ecospold_{self.system_model}_{self.version}" - file_name = Path( + file_name = ( DIR_CACHED_DB / f"cached_{''.join(tuple(map( str , __version__ )))}_{db_name.strip().lower()}_inventories.pickle" ) @@ -1602,7 +1598,7 @@ def write_datapackage(self, name: str = f"datapackage_{date.today()}"): cached_inventories = self.__find_cached_inventories(self.source) if not cached_inventories: - cache_fp = DATA_DIR / "cache" / f"cached_{self.source}_inventories.pickle" + cache_fp = DIR_CACHED_DB / f"cached_{self.source}_inventories.pickle" raise ValueError(f"No cached inventories found at {cache_fp}.") cache = {} diff --git a/premise/electricity.py b/premise/electricity.py index b5193aff..31998848 100644 --- a/premise/electricity.py +++ b/premise/electricity.py @@ -13,14 +13,13 @@ import re from collections import defaultdict from functools import lru_cache -from pprint import pprint import wurst import yaml -from . import VARIABLES_DIR from .data_collection import get_delimiter from .export import biosphere_flows_dictionary +from .filesystem_constants import DATA_DIR, VARIABLES_DIR from .logger import create_logger from .transformation import ( BaseTransformation, @@ -35,7 +34,7 @@ uuid, ws, ) -from .utils import DATA_DIR, eidb_label, get_efficiency_solar_photovoltaics +from .utils import eidb_label, get_efficiency_solar_photovoltaics LOSS_PER_COUNTRY = DATA_DIR / "electricity" / "losses_per_country.csv" IAM_BIOMASS_VARS = VARIABLES_DIR / "biomass_variables.yaml" diff --git a/premise/emissions.py b/premise/emissions.py index 9d022760..272fed7e 100644 --- a/premise/emissions.py +++ b/premise/emissions.py @@ -3,9 +3,7 @@ from GAINS. """ -import logging.config from functools import lru_cache -from pathlib import Path from typing import Union import numpy as np @@ -14,6 +12,7 @@ import yaml from numpy import ndarray +from .filesystem_constants import DATA_DIR from .logger import create_logger from .transformation import ( BaseTransformation, @@ -24,7 +23,6 @@ Set, ws, ) -from .utils import DATA_DIR logger = create_logger("emissions") diff --git a/premise/export.py b/premise/export.py index 305bba43..18419e9d 100644 --- a/premise/export.py +++ b/premise/export.py @@ -11,10 +11,9 @@ import uuid from collections import defaultdict from functools import lru_cache -from multiprocessing import Pool as ProcessPool from multiprocessing.pool import ThreadPool as Pool from pathlib import Path -from typing import Any, Dict, List, Set, Tuple, Union +from typing import Any, Dict, List import numpy as np import pandas as pd @@ -24,8 +23,9 @@ from pandas import DataFrame from scipy import sparse as nsp -from . import DATA_DIR, __version__ +from . import __version__ from .data_collection import get_delimiter +from .filesystem_constants import DATA_DIR from .inventory_imports import get_correspondence_bio_flows from .transformation import BaseTransformation from .utils import check_database_name diff --git a/premise/external.py b/premise/external.py index 7db91b58..e26c5f28 100644 --- a/premise/external.py +++ b/premise/external.py @@ -1,14 +1,23 @@ """ Implements external scenario data. """ +import logging +import uuid +from collections import defaultdict +from pathlib import Path +from typing import List, Union - +import numpy as np import wurst -from numpy import ndarray +import xarray as xr +import yaml +from wurst import searching as ws from .clean_datasets import get_biosphere_flow_uuid +from .data_collection import IAMDataCollection +from .filesystem_constants import DATA_DIR from .inventory_imports import generate_migration_maps, get_correspondence_bio_flows -from .transformation import * +from .transformation import BaseTransformation, get_shares_from_production_volume from .utils import eidb_label LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml" @@ -679,14 +688,14 @@ def check_existence_of_market_suppliers(self): def fetch_supply_share( self, i: int, region: str, var: str, variables: list - ) -> ndarray: + ) -> np.ndarray: """ Return the supply share of a given variable in a given region. :param i: index of the scenario :param region: region :param var: variable :param variables: list of all variables - :return: ndarray + :return: np.ndarray """ return np.clip( @@ -858,7 +867,7 @@ def adjust_efficiency_of_new_markets( ineff["variable"], region, eff_data, self.year ) - if not "includes" in ineff: + if "includes" not in ineff: wurst.change_exchanges_by_constant_factor(datatset, scaling_factor) else: diff --git a/premise/external_data_validation.py b/premise/external_data_validation.py index e4a117ea..0490877a 100644 --- a/premise/external_data_validation.py +++ b/premise/external_data_validation.py @@ -1,10 +1,6 @@ """ Validates datapackages that contain external scenario data. """ - -import sys -from pprint import pprint - import numpy as np import pandas as pd import yaml diff --git a/premise/filesystem_constants.py b/premise/filesystem_constants.py new file mode 100644 index 00000000..16e7fc4b --- /dev/null +++ b/premise/filesystem_constants.py @@ -0,0 +1,20 @@ +from pathlib import Path + +import platformdirs + +# Directories for data which comes with Premise +DATA_DIR = Path(__file__).resolve().parent / "data" +INVENTORY_DIR = DATA_DIR / "additional_inventories" +# Todo: Should this be in the data directory? +VARIABLES_DIR = Path(__file__).resolve().parent / "iam_variables_mapping" +IAM_OUTPUT_DIR = DATA_DIR / "iam_output_files" + +# Directories for user-created data +USER_DATA_BASE_DIR = platformdirs.user_data_path(appname="premise", appauthor="pylca") +USER_DATA_BASE_DIR.mkdir(parents=True, exist_ok=True) + +DIR_CACHED_DB = USER_DATA_BASE_DIR / "cache" +DIR_CACHED_DB.mkdir(parents=True, exist_ok=True) + +USER_LOGS_DIR = platformdirs.user_log_path(appname="premise", appauthor="pylca") +USER_LOGS_DIR.mkdir(parents=True, exist_ok=True) diff --git a/premise/fuels.py b/premise/fuels.py index 225681aa..6fb852f8 100644 --- a/premise/fuels.py +++ b/premise/fuels.py @@ -3,9 +3,7 @@ """ import copy -import logging.config from functools import lru_cache -from pathlib import Path from typing import Union import wurst @@ -13,7 +11,7 @@ import yaml from numpy import ndarray -from . import VARIABLES_DIR +from .filesystem_constants import DATA_DIR, VARIABLES_DIR from .inventory_imports import get_biosphere_code from .logger import create_logger from .transformation import ( @@ -30,7 +28,7 @@ uuid, ws, ) -from .utils import DATA_DIR, get_crops_properties +from .utils import get_crops_properties logger = create_logger("fuel") diff --git a/premise/geomap.py b/premise/geomap.py index bcaf89ab..dcb1ea67 100644 --- a/premise/geomap.py +++ b/premise/geomap.py @@ -9,7 +9,7 @@ import yaml from wurst import geomatcher -from . import DATA_DIR, VARIABLES_DIR +from .filesystem_constants import VARIABLES_DIR ECO_IAM_MAPPING = VARIABLES_DIR / "missing_geography_equivalences.yaml" diff --git a/premise/inventory_imports.py b/premise/inventory_imports.py index 0049cd19..af62de45 100644 --- a/premise/inventory_imports.py +++ b/premise/inventory_imports.py @@ -5,7 +5,6 @@ import csv import itertools -import sys import uuid from functools import lru_cache from pathlib import Path @@ -19,9 +18,9 @@ from prettytable import PrettyTable from wurst import searching as ws -from . import DATA_DIR, INVENTORY_DIR from .clean_datasets import remove_categories, remove_uncertainty from .data_collection import get_delimiter +from .filesystem_constants import DATA_DIR, DIR_CACHED_DB, INVENTORY_DIR from .geomap import Geomap FILEPATH_MIGRATION_MAP = INVENTORY_DIR / "migration_map.csv" @@ -822,8 +821,7 @@ def load_inventory(self, path): # online file # we need to save it locally first response = requests.get(path) - Path(DATA_DIR / "cache").mkdir(parents=True, exist_ok=True) - path = str(Path(DATA_DIR / "cache" / "temp.csv")) + path = DIR_CACHED_DB / "temp.csv" with open(path, "w", encoding="utf-8") as f: writer = csv.writer( f, @@ -835,9 +833,9 @@ def load_inventory(self, path): for line in response.iter_lines(): writer.writerow(line.decode("utf-8").split(",")) - if Path(path).suffix == ".xlsx": + if path.suffix == ".xlsx": return ExcelImporter(path) - elif Path(path).suffix == ".csv": + elif path.suffix == ".csv": return CSVImporter(path) else: raise ValueError( diff --git a/premise/logger.py b/premise/logger.py index f2aa3beb..903df805 100644 --- a/premise/logger.py +++ b/premise/logger.py @@ -1,10 +1,9 @@ import logging.config -import multiprocessing from pathlib import Path import yaml -from . import DATA_DIR +from .filesystem_constants import DATA_DIR LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml" DIR_LOG_REPORT = Path.cwd() / "export" / "logs" diff --git a/premise/marginal_mixes.py b/premise/marginal_mixes.py index af952769..b03c49c8 100644 --- a/premise/marginal_mixes.py +++ b/premise/marginal_mixes.py @@ -13,7 +13,7 @@ import yaml from numpy import ndarray -from premise import DATA_DIR +from .filesystem_constants import DATA_DIR IAM_LEADTIMES = DATA_DIR / "consequential" / "leadtimes.yaml" IAM_LIFETIMES = DATA_DIR / "consequential" / "lifetimes.yaml" diff --git a/premise/report.py b/premise/report.py index dced5b6b..46cb496b 100644 --- a/premise/report.py +++ b/premise/report.py @@ -15,10 +15,10 @@ from openpyxl.utils import get_column_letter from openpyxl.utils.dataframe import dataframe_to_rows from openpyxl.worksheet.dimensions import ColumnDimension, DimensionHolder -from pandas._libs.parsers import ParserError from pandas.errors import EmptyDataError -from . import DATA_DIR, VARIABLES_DIR, __version__ +from . import __version__ +from .filesystem_constants import DATA_DIR, VARIABLES_DIR IAM_ELEC_VARS = VARIABLES_DIR / "electricity_variables.yaml" IAM_FUELS_VARS = VARIABLES_DIR / "fuels_variables.yaml" diff --git a/premise/steel.py b/premise/steel.py index f9e31554..28858e01 100644 --- a/premise/steel.py +++ b/premise/steel.py @@ -1,17 +1,13 @@ """ Integrates projections regarding steel production. """ -import logging.config -from pathlib import Path from typing import Dict, List import wurst -import yaml from .data_collection import IAMDataCollection from .logger import create_logger from .transformation import BaseTransformation, ws -from .utils import DATA_DIR logger = create_logger("steel") diff --git a/premise/transformation.py b/premise/transformation.py index 71dfa501..36b9ef88 100644 --- a/premise/transformation.py +++ b/premise/transformation.py @@ -25,8 +25,9 @@ from .activity_maps import InventorySet from .data_collection import IAMDataCollection +from .filesystem_constants import DATA_DIR from .geomap import Geomap -from .utils import DATA_DIR, get_fuel_properties +from .utils import get_fuel_properties LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml" # directory for log files diff --git a/premise/transport.py b/premise/transport.py index 60f093da..3430679b 100644 --- a/premise/transport.py +++ b/premise/transport.py @@ -14,16 +14,14 @@ from wurst import searching as ws from wurst import transformations as wt -from . import DATA_DIR, INVENTORY_DIR +from .filesystem_constants import DATA_DIR, IAM_OUTPUT_DIR, INVENTORY_DIR from .inventory_imports import VariousVehicles from .transformation import BaseTransformation, IAMDataCollection from .utils import eidb_label -FILEPATH_FLEET_COMP = ( - DATA_DIR / "iam_output_files" / "fleet_files" / "fleet_all_vehicles.csv" -) +FILEPATH_FLEET_COMP = IAM_OUTPUT_DIR / "fleet_files" / "fleet_all_vehicles.csv" FILEPATH_IMAGE_TRUCKS_FLEET_COMP = ( - DATA_DIR / "iam_output_files" / "fleet_files" / "image_fleet_trucks.csv" + IAM_OUTPUT_DIR / "fleet_files" / "image_fleet_trucks.csv" ) FILEPATH_TWO_WHEELERS = INVENTORY_DIR / "lci-two_wheelers.xlsx" FILEPATH_TRUCKS = INVENTORY_DIR / "lci-trucks.xlsx" diff --git a/premise/utils.py b/premise/utils.py index 48efd81e..d89105c4 100644 --- a/premise/utils.py +++ b/premise/utils.py @@ -7,7 +7,7 @@ import uuid from functools import lru_cache from pathlib import Path -from typing import List +from typing import List, Optional import pandas as pd import xarray as xr @@ -16,16 +16,12 @@ from bw2io.importers.base_lci import LCIImporter from country_converter import CountryConverter from prettytable import ALL, PrettyTable -from wurst.linking import ( - change_db_name, - check_duplicate_codes, - check_internal_linking, - link_internal, -) +from wurst.linking import change_db_name, check_internal_linking, link_internal from wurst.searching import equals, get_many -from . import DATA_DIR, VARIABLES_DIR, __version__ +from . import __version__ from .data_collection import get_delimiter +from .filesystem_constants import DATA_DIR, DIR_CACHED_DB, VARIABLES_DIR from .geomap import Geomap FUELS_PROPERTIES = VARIABLES_DIR / "fuels_variables.yaml" @@ -160,7 +156,7 @@ def get_regions_definition(model: str) -> None: print(table) -def clear_existing_cache(): +def clear_existing_cache(all_versions: Optional[bool] = False) -> None: """Clears the cache folder, except for files which contain __version__ in name. Useful when updating `premise` or encountering issues with @@ -168,14 +164,15 @@ def clear_existing_cache(): """ [ f.unlink() - for f in Path(DATA_DIR / "cache").glob("*") - if f.is_file() and "".join(tuple(map(str, __version__))) not in f.name + for f in DIR_CACHED_DB.glob("*") + if f.is_file() + and (all_versions or "".join(tuple(map(str, __version__))) not in f.name) ] # clear the cache folder -def clear_cache(): - [f.unlink() for f in Path(DATA_DIR / "cache").glob("*") if f.is_file()] +def clear_cache() -> None: + clear_existing_cache(all_versions=True) print("Cache folder cleared!") diff --git a/requirements.txt b/requirements.txt index f57a1164..346c35a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,19 @@ +bottleneck +bw2data +bw2io>=0.8.10 +constructive_geometries>=0.8.2 +cryptography +datapackage numpy pandas -bw2io>=0.8.10 -bw2data -wurst -xarray +platformdirs +premise_gwp prettytable +pyarrow pycountry -cryptography -premise_gwp pyYaml -sparse>=0.14.0 -schema -datapackage requests -bottleneck -constructive_geometries>=0.8.2 -pyarrow +schema +sparse>=0.14.0 +wurst +xarray diff --git a/setup.py b/setup.py index d777d93e..7cad73bf 100644 --- a/setup.py +++ b/setup.py @@ -41,25 +41,26 @@ def package_files(directory): # Only if you have non-python data (CSV, etc.). Might need to change the directory name as well. include_package_data=True, install_requires=[ - "numpy", - "wurst", + "bottleneck", + "bw2data", # bw2io 0.8.7 or 0.8.10 "bw2io >=0.8.10", + "constructive_geometries>=0.8.2", + "cryptography", + "datapackage", + "numpy", "pandas", - "bw2data", - "xarray", + "platformdirs", + "premise_gwp", "prettytable", + "pyarrow", "pycountry", - "cryptography", - "premise_gwp", "pyYaml", - "sparse>=0.14.0", - "schema", - "datapackage", "requests", - "bottleneck", - "constructive_geometries>=0.8.2", - "pyarrow", + "schema", + "sparse>=0.14.0", + "wurst", + "xarray", ], url="https://github.com/polca/premise", description="Coupling IAM output to ecoinvent LCA database ecoinvent for prospective LCA", diff --git a/tests/test_electricity.py b/tests/test_electricity.py index 08d0b6c3..5f941cb0 100644 --- a/tests/test_electricity.py +++ b/tests/test_electricity.py @@ -1,9 +1,12 @@ # content of test_electricity.py import os +from pathlib import Path + +import pytest -from premise import DATA_DIR from premise.data_collection import IAMDataCollection from premise.electricity import Electricity +from premise.filesystem_constants import DATA_DIR LOSS_PER_COUNTRY = DATA_DIR / "electricity" / "losses_per_country.csv" LHV_FUELS = DATA_DIR / "fuels_lower_heating_value.txt" @@ -40,38 +43,48 @@ def get_db(): return dummy_db, version +# This won't work with PRs because PRs from outside contributors don't have +# access to secrets (for good reason). if "IAM_FILES_KEY" in os.environ: key = os.environ["IAM_FILES_KEY"] else: - with open("/Users/romain/Dropbox/Notebooks/key.txt") as f: - lines = f.readlines() - key = lines[0] + # This won't work on most computers :) + if Path("/Users/romain/Dropbox/Notebooks/key.txt").is_file(): + with open("/Users/romain/Dropbox/Notebooks/key.txt") as f: + lines = f.readlines() + key = lines[0] + else: + key = None + -rdc = IAMDataCollection( - model="remind", - pathway="SSP2-Base", - year=2012, - filepath_iam_files=DATA_DIR / "iam_output_files", - key=str.encode(key), -) -db, _ = get_db() -el = Electricity( - database=db, - iam_data=rdc, - model="remind", - pathway="SSP2-Base", - year=2012, - version="3.5", - system_model="cutoff", - modified_datasets={}, -) +if key: + rdc = IAMDataCollection( + model="remind", + pathway="SSP2-Base", + year=2012, + filepath_iam_files=DATA_DIR / "iam_output_files", + key=str.encode(key), + ) + db, _ = get_db() + el = Electricity( + database=db, + iam_data=rdc, + model="remind", + pathway="SSP2-Base", + year=2012, + version="3.5", + system_model="cutoff", + modified_datasets={}, + ) +@pytest.mark.skipif(not key, reason="No access to decryption key") def test_losses(): assert len(el.network_loss) == 13 assert el.network_loss["CAZ"]["high"]["transf_loss"] == 0.035483703331573094 +@pytest.mark.skipif(not key, reason="No access to decryption key") def test_powerplant_map(): s = el.powerplant_map["Biomass IGCC CCS"] assert isinstance(s, set) diff --git a/tests/test_filesystem_constants.py b/tests/test_filesystem_constants.py new file mode 100644 index 00000000..73de56a6 --- /dev/null +++ b/tests/test_filesystem_constants.py @@ -0,0 +1,61 @@ +import os +from pathlib import Path + +from premise.filesystem_constants import ( + DATA_DIR, + DIR_CACHED_DB, + IAM_OUTPUT_DIR, + INVENTORY_DIR, + VARIABLES_DIR, +) + +dd_root = str(DATA_DIR.parent.parent) +cd_root = str(DIR_CACHED_DB.parent.parent) + + +def test_data_dir(): + assert isinstance(DATA_DIR, Path) + assert DATA_DIR.is_dir() + assert len(list(DATA_DIR.iterdir())) > 2 + for fp in DATA_DIR.iterdir(): + assert os.access(fp, os.R_OK) + assert dd_root not in cd_root and cd_root not in dd_root + + +def test_inventory_dir(): + assert isinstance(INVENTORY_DIR, Path) + assert INVENTORY_DIR.is_dir() + for fp in INVENTORY_DIR.iterdir(): + assert os.access(fp, os.R_OK) + assert len(list(INVENTORY_DIR.iterdir())) > 2 + assert str(DATA_DIR) in str(INVENTORY_DIR) + + +def test_variables_dir(): + assert isinstance(VARIABLES_DIR, Path) + assert VARIABLES_DIR.is_dir() + assert len(list(VARIABLES_DIR.iterdir())) > 2 + for fp in VARIABLES_DIR.iterdir(): + assert os.access(fp, os.R_OK) + assert str(DATA_DIR) not in str(VARIABLES_DIR) + assert dd_root in str(VARIABLES_DIR) + assert cd_root not in str(VARIABLES_DIR) + + +def test_iam_output_dir(): + assert isinstance(IAM_OUTPUT_DIR, Path) + assert IAM_OUTPUT_DIR.is_dir() + for fp in IAM_OUTPUT_DIR.iterdir(): + assert os.access(fp, os.R_OK) + assert len(list(IAM_OUTPUT_DIR.iterdir())) > 2 + assert str(DATA_DIR) in str(IAM_OUTPUT_DIR) + + +def test_user_data_dir(): + assert isinstance(DIR_CACHED_DB, Path) + assert DIR_CACHED_DB.is_dir() + assert os.access(DIR_CACHED_DB, os.W_OK) + assert os.access(DIR_CACHED_DB, os.R_OK) + assert cd_root in str(DIR_CACHED_DB) + assert dd_root not in str(DIR_CACHED_DB) + assert "cache" in str(DIR_CACHED_DB) diff --git a/tests/test_import_inventories.py b/tests/test_import_inventories.py index ff2749dc..f9378f9d 100644 --- a/tests/test_import_inventories.py +++ b/tests/test_import_inventories.py @@ -3,7 +3,7 @@ import pytest -from premise import INVENTORY_DIR +from premise.filesystem_constants import INVENTORY_DIR from premise.inventory_imports import BaseInventoryImport, DefaultInventory FILEPATH_CARMA_INVENTORIES = INVENTORY_DIR / "lci-Carma-CCS.xlsx"