Refactor core package

.gitignore

@@ -133,6 +133,6 @@ dmypy.json
 build/
 cplus_scenario_output.*
 [Skip output]
-reports
+# reports
 
 ext-libs/

admin.py

@@ -213,6 +213,7 @@ def build(
     if clean and output_directory.exists():
         shutil.rmtree(str(output_directory), ignore_errors=True)
     output_directory.mkdir(parents=True, exist_ok=True)
+    plugin_setup(clean)
     copy_source_files(output_directory, tests=tests)
     icon_path = copy_icon(output_directory)
     if icon_path is None:
@@ -552,5 +553,87 @@ def _get_latest_releases(
     return latest_stable, latest_experimental
 
 
+###############################################################################
+# Setup dependencies and install package
+###############################################################################
+
+
+def not_comments(lines, s, e):
+    """Return non comment line (does not start with #).
+
+    :param lines: list of line
+    :type lines: list of str
+
+    :param s: start index
+    :type s: int
+
+    :param e: end index
+    :type e: int
+
+    :return: list of line without commented lines
+    :rtype: list of str
+    """
+    return [line for line in lines[s:e] if line[0] != "#"]
+
+
+def read_requirements():
+    """Return a list of runtime and list of test requirements."""
+    with open("requirements.txt") as f:
+        lines = f.readlines()
+    lines = [line for line in [line.strip() for line in lines] if line]
+
+    return not_comments(lines, 0, len(lines)), []
+
+
+def _safe_remove_folder(rootdir):
+    """
+    Supports removing a folder that may have symlinks in it.
+
+    Needed on windows to avoid removing the original files linked to within
+    each folder.
+
+    :param rootdir: Root directory to clean.
+    :type rootdir: str
+    """
+    rootdir = Path(rootdir)
+    if rootdir.is_symlink():
+        rootdir.rmdir()
+    else:
+        folders = [path for path in Path(rootdir).iterdir() if path.is_dir()]
+        for folder in folders:
+            if folder.is_symlink():
+                folder.rmdir()
+            else:
+                shutil.rmtree(folder)
+        files = [path for path in Path(rootdir).iterdir()]
+        for file in files:
+            file.unlink()
+        shutil.rmtree(rootdir)
+
+
+@app.command()
+def plugin_setup(clean=True, pip="pip"):
+    """install plugin dependencies.
+
+    :param clean: Clean out dependencies first.
+    :type clean: bool
+
+    :param pip: Path to pip, usually 'pip' or 'pip3'.
+    :type pip: str
+    """
+    ext_libs = os.path.join(LOCAL_ROOT_DIR, "src", "cplus_plugin", "ext-libs")
+
+    if clean and os.path.exists(ext_libs):
+        _safe_remove_folder(ext_libs)
+
+    os.makedirs(ext_libs, exist_ok=True)
+    runtime, test = read_requirements()
+
+    os.environ["PYTHONPATH"] = ext_libs
+
+    for req in runtime + test:
+        subprocess.check_call([pip, "install", "--upgrade", "-t", ext_libs, req])
+
+
 if __name__ == "__main__":
     app()

requirements-dev.txt

@@ -26,15 +26,17 @@ jinja2 == 3.0.3
 markdown == 3.3.6
 markupsafe == 2.0.1
 mergedeep == 1.3.4
-mkdocs-git-revision-date-localized-plugin
+mkdocs-autorefs == 1.2.0
+mkdocs-git-revision-date-localized-plugin == 0.11.1
 mkdocs-material-extensions == 1.0.3
-mkdocs-material
-mkdocstrings-python
-mkdocs-video
-mkdocs
-packaging == 21.3
+mkdocs-material == 8.1.7
+mkdocstrings-python == 1.6.0
+mkdocs-video == 1.1.0
+mkdocs == 1.2.3
+# packaging == 21.3
+packaging == 24.0
 pygments == 2.11.2
-pymdown-extensions
+pymdown-extensions == 9.1
 pyparsing == 3.0.6
 pyqt5-qt5 == 5.15.2
 pyqt5-sip == 12.9.0

requirements.txt

@@ -0,0 +1,2 @@
+# cplus-core
+git+https://github.com/kartoza/[email protected]

src/cplus_plugin/__init__.py

@@ -30,6 +30,19 @@
     sys.path.append(LIB_DIR)
 
 
+def _add_at_front_of_path(d):
+    """add a folder at front of path"""
+    sys.path, remainder = sys.path[:1], sys.path[1:]
+    site.addsitedir(d)
+    sys.path.extend(remainder)
+
+
+# init ext-libs directory
+plugin_dir = os.path.dirname(os.path.realpath(__file__))
+# Put ext-libs folder near the front of the path (important on Linux)
+_add_at_front_of_path(str(Path(plugin_dir) / "ext-libs"))
+
+
 # noinspection PyPep8Naming
 def classFactory(iface):  # pylint: disable=invalid-name
     """Load QgisCplus class

src/cplus_plugin/api/base.py

@@ -13,7 +13,7 @@
 from qgis.core import QgsTask
 
 from .request import CplusApiRequest
-from ..models.base import Scenario, ScenarioResult, NcsPathway, Activity
+from cplus_core.models.base import Scenario, ScenarioResult, NcsPathway, Activity
 
 
 class BaseScenarioTask(QgsTask):

src/cplus_plugin/api/request.py

@@ -16,7 +16,7 @@
     QgsProcessingFeedback,
 )
 
-from ..models.base import Scenario, SpatialExtent, Activity
+from cplus_core.models.base import Scenario, SpatialExtent, Activity
 from ..conf import settings_manager, Settings
 from ..definitions.defaults import BASE_API_URL
 from ..trends_earth import auth

src/cplus_plugin/api/scenario_history_tasks.py

@@ -14,8 +14,8 @@
 from .request import CplusApiRequest, CplusApiRequestError
 from .scenario_task_api_client import ScenarioAnalysisTaskApiClient
 from ..conf import settings_manager
-from ..models.base import Scenario
-from ..models.base import SpatialExtent
+from cplus_core.models.base import Scenario
+from cplus_core.analysis import TaskConfig
 from ..utils import log
 
 
@@ -133,38 +133,36 @@ class FetchScenarioOutputTask(ScenarioAnalysisTaskApiClient):
 
     def __init__(
         self,
-        analysis_scenario_name,
-        analysis_scenario_description,
-        analysis_activities,
-        analysis_priority_layers_groups,
-        analysis_extent,
-        scenario,
-        scenario_directory,
+        task_config: TaskConfig,
+        extent_box,
     ):
-        super(FetchScenarioOutputTask, self).__init__(
-            analysis_scenario_name,
-            analysis_scenario_description,
-            analysis_activities,
-            analysis_priority_layers_groups,
-            analysis_extent,
-            scenario,
-            SpatialExtent(scenario.extent.bbox),
-        )
+        super(FetchScenarioOutputTask, self).__init__(task_config, extent_box)
         self.request = CplusApiRequest()
         self.status_pooling = None
         self.logs = []
         self.total_file_output = 0
         self.downloaded_output = 0
         self.scenario_status = None
-        self.scenario_directory = scenario_directory
-        self.scenario_api_uuid = scenario.uuid
-        self.scenario = scenario
+        self.scenario_api_uuid = task_config.scenario.uuid
+        self.scenario = task_config.scenario
 
         self.scenario_directory = None
         self.processing_cancelled = False
         self.scenario_result = None
         self.output_list = None
-        self.created_datetime
+        self.created_datetime = None
+
+    def _get_scenario_directory(self, created_datetime: datetime.datetime) -> str:
+        """Generate scenario directory for current task.
+
+        :return: Path to scenario directory
+        :rtype: str
+        """
+        base_dir = self.task_config.base_dir
+        return os.path.join(
+            f"{base_dir}",
+            "scenario_" f'{created_datetime.strftime("%Y_%m_%d_%H_%M_%S")}',
+        )
 
     def run(self):
         """Execute the task logic.
@@ -183,7 +181,9 @@ def run(self):
             self.created_datetime = datetime.datetime.strptime(
                 self.new_scenario_detail["submitted_on"], "%Y-%m-%dT%H:%M:%SZ"
             )
-            self.scenario_directory = self.get_scenario_directory()
+            self.scenario_directory = self._get_scenario_directory(
+                self.created_datetime
+            )
             if os.path.exists(self.scenario_directory):
                 for file in os.listdir(self.scenario_directory):
                     if file != "processing.log":

src/cplus_plugin/api/scenario_task_api_client.py

@@ -15,8 +15,8 @@
 )
 from ..api.base import BaseFetchScenarioOutput
 from ..conf import settings_manager, Settings
-from ..models.base import Activity, NcsPathway, Scenario
-from ..tasks import ScenarioAnalysisTask
+from cplus_core.models.base import Activity, NcsPathway
+from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig
 from ..utils import FileUtils, CustomJsonEncoder, todict
 
 
@@ -42,43 +42,15 @@ def clean_filename(filename):
 class ScenarioAnalysisTaskApiClient(ScenarioAnalysisTask, BaseFetchScenarioOutput):
     """Prepares and runs the scenario analysis in Cplus API
 
-    :param analysis_scenario_name: Scenario name
-    :type analysis_scenario_name: str
-
-    :param analysis_scenario_description: Scenario description
-    :type analysis_scenario_description: str
-
-    :param analysis_activities: List of activity to be processed
-    :type analysis_activities: typing.List[Activity]
-
-    :param analysis_priority_layers_groups: List of priority layer groups
-    :type analysis_priority_layers_groups: typing.List[dict]
-
-    :param analysis_extent: Extents of the Scenario
-    :type analysis_extent: typing.List[float]
-
     :param scenario: Scenario object
     :type scenario: Scenario
+
+    :param extent_box: Project extent
+    :type extent_box: list of float
     """
 
-    def __init__(
-        self,
-        analysis_scenario_name: str,
-        analysis_scenario_description: str,
-        analysis_activities: typing.List[Activity],
-        analysis_priority_layers_groups: typing.List[dict],
-        analysis_extent: typing.List[float],
-        scenario: Scenario,
-        extent_box,
-    ):
-        super(ScenarioAnalysisTaskApiClient, self).__init__(
-            analysis_scenario_name,
-            analysis_scenario_description,
-            analysis_activities,
-            analysis_priority_layers_groups,
-            analysis_extent,
-            scenario,
-        )
+    def __init__(self, task_config: TaskConfig, extent_box):
+        super().__init__(task_config)
         self.total_file_upload_size = 0
         self.total_file_upload_chunks = 0
         self.uploaded_chunks = 0
@@ -142,7 +114,7 @@ def run(self) -> bool:
         :rtype: bool
         """
         self.request = CplusApiRequest()
-        self.scenario_directory = self.get_scenario_directory()
+        self.scenario_directory = self.task_config.base_dir
         FileUtils.create_new_dir(self.scenario_directory)
 
         try:
@@ -516,8 +488,10 @@ def build_scenario_detail_json(self) -> None:
         snap_rescale = self.get_settings_value(
             Settings.RESCALE_VALUES, default=False, setting_type=bool
         )
-        resampling_method = self.get_settings_value(
-            Settings.RESAMPLING_METHOD, default=0
+        resampling_method = int(
+            self.get_settings_value(
+                Settings.RESAMPLING_METHOD, default=0, setting_type=int
+            )
         )
         ncs_with_carbon = self.get_settings_value(
             Settings.NCS_WITH_CARBON, default=False, setting_type=bool

src/cplus_plugin/conf.py

@@ -29,7 +29,7 @@
     UUID_ATTRIBUTE,
 )
 from .definitions.defaults import PRIORITY_LAYERS
-from .models.base import (
+from cplus_core.models.base import (
     Activity,
     NcsPathway,
     Scenario,

src/cplus_plugin/gui/activity_editor_dialog.py

@@ -29,7 +29,7 @@
     ACTIVITY_SCENARIO_STYLE_ATTRIBUTE,
 )
 from ..definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE
-from ..models.base import Activity
+from cplus_core.models.base import Activity
 from ..utils import FileUtils, generate_random_color, open_documentation, tr
 
 WidgetUi, _ = loadUiType(

src/cplus_plugin/gui/activity_widget.py

@@ -19,7 +19,7 @@
     ActivityComponentWidget,
     NcsComponentWidget,
 )
-from ..models.base import Activity, NcsPathway
+from cplus_core.models.base import Activity, NcsPathway
 
 from ..utils import FileUtils
 

src/cplus_plugin/gui/component_item_model.py

@@ -12,7 +12,7 @@
 
 from qgis.PyQt import QtCore, QtGui
 
-from ..models.base import (
+from cplus_core.models.base import (
     BaseModelComponent,
     BaseModelComponentType,
     Activity,

src/cplus_plugin/gui/financials/npv_manager_dialog.py

@@ -21,7 +21,7 @@
 from ..component_item_model import ActivityItemModel
 from ...conf import settings_manager
 from ...definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE
-from ...models.base import Activity
+from cplus_core.models.base import Activity
 from ...models.financial import ActivityNpv, ActivityNpvCollection, NpvParameters
 from .npv_financial_model import NpvFinancialModel
 from ...lib.financials import compute_discount_value

src/cplus_plugin/gui/items_selection_dialog.py

@@ -11,7 +11,7 @@
 
 from qgis.PyQt.uic import loadUiType
 
-from ..models.base import Activity, PriorityLayer
+from cplus_core.models.base import Activity, PriorityLayer
 
 from ..conf import settings_manager
 

src/cplus_plugin/gui/map_repeat_item_widget.py

@@ -18,7 +18,7 @@
 
 from ..conf import settings_manager
 from ..lib.reports.layout_items import CplusMapRepeatItem, CPLUS_MAP_REPEAT_ITEM_TYPE
-from ..models.base import ModelComponentType
+from cplus_core.models.base import ModelComponentType
 from ..utils import FileUtils, tr