diff --git a/.flake8 b/.flake8 index bdc269ef..e0f2fb22 100644 --- a/.flake8 +++ b/.flake8 @@ -59,7 +59,7 @@ per-file-ignores = # QGS104 Use 'import qgis.PyQt.pyrcc_main' instead of 'import PyQt5.pyrcc_main' # just not available in OSGeo4W installations # script should run immediately which site.addsitedir(REPO) before other imports - scripts/*.py : E402 + # scripts/*.py : E402 enmapbox/eo4qapps/geetimeseriesexplorerapp/*.py : F821 enmapbox/__main__.py: E402 enmapbox/gui/__init__.py: F401 diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index ed970c16..16f7ec31 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.12" - name: Run flake8 uses: julianwachholz/flake8-action@v2 with: diff --git a/enmapbox/__init__.py b/enmapbox/__init__.py index af53b688..02477984 100644 --- a/enmapbox/__init__.py +++ b/enmapbox/__init__.py @@ -34,10 +34,9 @@ import warnings from osgeo import gdal - from qgis.PyQt.QtCore import QSettings from qgis.PyQt.QtGui import QIcon -from qgis.core import Qgis, QgsApplication, QgsProcessingRegistry, QgsProcessingProvider, QgsProcessingAlgorithm +from qgis.core import Qgis, QgsApplication, QgsProcessingAlgorithm, QgsProcessingProvider, QgsProcessingRegistry from qgis.gui import QgisInterface, QgsMapLayerConfigWidgetFactory # provide shortcuts @@ -96,7 +95,6 @@ ENMAP_BOX_KEY = 'EnMAP-Box' -_ENMAPBOX_PROCESSING_PROVIDER: QgsProcessingProvider = None _ENMAPBOX_MAPLAYER_CONFIG_WIDGET_FACTORIES: typing.List[QgsMapLayerConfigWidgetFactory] = [] gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', 'YES') @@ -262,13 +260,11 @@ def registerEnMAPBoxProcessingProvider(): assert isinstance(registry, QgsProcessingRegistry) provider = registry.providerById(ID) if not isinstance(provider, QgsProcessingProvider): - provider = EnMAPBoxProcessingProvider() + provider = EnMAPBoxProcessingProvider.instance() registry.addProvider(provider) assert isinstance(provider, EnMAPBoxProcessingProvider) assert id(registry.providerById(ID)) == id(provider) - global _ENMAPBOX_PROCESSING_PROVIDER - _ENMAPBOX_PROCESSING_PROVIDER = provider try: existingAlgNames = [a.name() for a in registry.algorithms() if a.groupId() == provider.id()] @@ -293,8 +289,7 @@ def unregisterEnMAPBoxProcessingProvider(): provider = registry.providerById(ID) if isinstance(provider, EnMAPBoxProcessingProvider): - global _ENMAPBOX_PROCESSING_PROVIDER - _ENMAPBOX_PROCESSING_PROVIDER = None + EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER = None # this deletes the C++ object registry.removeProvider(ID) diff --git a/enmapbox/algorithmprovider.py b/enmapbox/algorithmprovider.py index bc891459..e37f4cca 100644 --- a/enmapbox/algorithmprovider.py +++ b/enmapbox/algorithmprovider.py @@ -50,6 +50,15 @@ class EnMAPBoxProcessingProvider(QgsProcessingProvider): It enhances the "standard" processing.core.AlgorithmProvider by functionality to add and remove GeoAlgorithms during runtime. """ + _ENMAPBOX_PROCESSING_PROVIDER = None + + @staticmethod + def instance() -> 'EnMAPBoxProcessingProvider': + + if EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER is None: + EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER = EnMAPBoxProcessingProvider() + return EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER + def __init__(self): super(EnMAPBoxProcessingProvider, self).__init__() # internal list of GeoAlgorithms. Is used on re-loads and can be manipulated diff --git a/enmapboxprocessing/enmapalgorithm.py b/enmapboxprocessing/enmapalgorithm.py index b1653248..f3515816 100644 --- a/enmapboxprocessing/enmapalgorithm.py +++ b/enmapboxprocessing/enmapalgorithm.py @@ -2,36 +2,35 @@ from enum import Enum from math import nan from os import makedirs -from os.path import isabs, join, dirname, exists, splitext, abspath +from os.path import abspath, dirname, exists, isabs, join, splitext from time import time -from typing import Any, Dict, Iterable, Optional, List, Tuple, TextIO +from typing import Any, Dict, Iterable, List, Optional, TextIO, Tuple import numpy as np from osgeo import gdal - import processing +from qgis.PyQt.QtCore import QVariant +from qgis.PyQt.QtGui import QIcon +from qgis.core import (Qgis, QgsCategorizedSymbolRenderer, QgsCoordinateReferenceSystem, QgsMapLayer, + QgsPalettedRasterRenderer, QgsProcessing, QgsProcessingAlgorithm, QgsProcessingContext, + QgsProcessingException, QgsProcessingFeedback, QgsProcessingOutputLayerDefinition, + QgsProcessingParameterBand, QgsProcessingParameterBoolean, QgsProcessingParameterCrs, + QgsProcessingParameterDefinition, QgsProcessingParameterEnum, QgsProcessingParameterExtent, + QgsProcessingParameterField, QgsProcessingParameterFile, QgsProcessingParameterFileDestination, + QgsProcessingParameterFolderDestination, QgsProcessingParameterMapLayer, + QgsProcessingParameterMatrix, QgsProcessingParameterMultipleLayers, QgsProcessingParameterNumber, + QgsProcessingParameterRange, QgsProcessingParameterRasterLayer, QgsProcessingParameterString, + QgsProcessingParameterVectorDestination, QgsProcessingParameterVectorLayer, QgsProcessingUtils, + QgsProject, QgsProperty, QgsRasterLayer, QgsRectangle, QgsVectorLayer) + from enmapbox.typeguard import typechecked from enmapboxprocessing.driver import Driver from enmapboxprocessing.glossary import injectGlossaryLinks from enmapboxprocessing.parameter.processingparameterrasterdestination import ProcessingParameterRasterDestination from enmapboxprocessing.processingfeedback import ProcessingFeedback -from enmapboxprocessing.typing import CreationOptions, GdalResamplingAlgorithm, ClassifierDump, \ - TransformerDump, RegressorDump, ClustererDump +from enmapboxprocessing.typing import ClassifierDump, ClustererDump, CreationOptions, GdalResamplingAlgorithm, \ + RegressorDump, TransformerDump from enmapboxprocessing.utils import Utils -from qgis.PyQt.QtCore import QVariant -from qgis.PyQt.QtGui import QIcon -from qgis.core import (QgsProcessingAlgorithm, QgsProcessingParameterRasterLayer, QgsProcessingParameterVectorLayer, - QgsProcessingContext, QgsProcessingFeedback, - QgsRasterLayer, QgsVectorLayer, QgsProcessingParameterNumber, QgsProcessingParameterDefinition, - QgsProcessingParameterField, QgsProcessingParameterBoolean, QgsProcessingParameterEnum, Qgis, - QgsProcessingParameterString, QgsProcessingParameterBand, QgsCategorizedSymbolRenderer, - QgsPalettedRasterRenderer, QgsProcessingParameterMapLayer, QgsMapLayer, - QgsProcessingParameterExtent, QgsCoordinateReferenceSystem, QgsRectangle, - QgsProcessingParameterFileDestination, QgsProcessingParameterFile, QgsProcessingParameterRange, - QgsProcessingParameterCrs, QgsProcessingParameterVectorDestination, QgsProcessing, - QgsProcessingUtils, QgsProcessingParameterMultipleLayers, QgsProcessingException, - QgsProcessingParameterFolderDestination, QgsProject, QgsProcessingOutputLayerDefinition, - QgsProperty, QgsProcessingParameterMatrix) class AlgorithmCanceledException(Exception): @@ -42,8 +41,8 @@ class AlgorithmCanceledException(Exception): class EnMAPProcessingAlgorithm(QgsProcessingAlgorithm): O_RESAMPLE_ALG = 'NearestNeighbour Bilinear Cubic CubicSpline Lanczos Average Mode Min Q1 Med Q3 Max'.split() NearestNeighbourResampleAlg, BilinearResampleAlg, CubicResampleAlg, CubicSplineResampleAlg, LanczosResampleAlg, \ - AverageResampleAlg, ModeResampleAlg, MinResampleAlg, Q1ResampleAlg, MedResampleAlg, Q3ResampleAlg, \ - MaxResampleAlg = range(12) + AverageResampleAlg, ModeResampleAlg, MinResampleAlg, Q1ResampleAlg, MedResampleAlg, Q3ResampleAlg, \ + MaxResampleAlg = range(12) O_DATA_TYPE = 'Byte Int16 UInt16 UInt32 Int32 Float32 Float64'.split() Byte, Int16, UInt16, Int32, UInt32, Float32, Float64 = range(len(O_DATA_TYPE)) PickleFileFilter = 'Pickle files (*.pkl)' @@ -1051,7 +1050,7 @@ class Group(Enum): RasterProjections = 'Raster projections' Regression = 'Regression' Sampling = 'Sampling' - SpectralLibrary = 'Spectral library' + SpectralLibrary = 'Spectral Library' SpectralResampling = 'Spectral resampling' Testdata = 'Testdata' Transformation = 'Transformation' diff --git a/scripts/EnFireMAP/sample_data.py b/scripts/EnFireMAP/sample_data.py index 9676b1ce..ed59e965 100644 --- a/scripts/EnFireMAP/sample_data.py +++ b/scripts/EnFireMAP/sample_data.py @@ -1,9 +1,12 @@ from os import listdir, makedirs -from os.path import join, exists, dirname +from os.path import dirname, exists, join +from qgis.core import QgsVectorFileWriter, QgsVectorLayer + +from enmapbox import initAll +from enmapbox.testing import start_app from enmapboxprocessing.algorithm.samplerastervaluesalgorithm import SampleRasterValuesAlgorithm from enmapboxprocessing.enmapalgorithm import EnMAPProcessingAlgorithm -from qgis.core import QgsVectorFileWriter, QgsVectorLayer rootCube = r'D:\data\EnFireMap\cube' tilingScheme = QgsVectorLayer(r'D:\data\EnFireMap\cube\shp\grid2.geojson') @@ -21,9 +24,6 @@ locations = QgsVectorLayer(r'D:\data\EnFireMap\lib_points\lib_points.shp') rootOutputSample = r'D:\data\EnFireMap\sample3' -from enmapbox import initAll -from enmapbox.testing import start_app - qgsApp = start_app() initAll() diff --git a/scripts/create_plugin.py b/scripts/create_plugin.py index 0c7ae409..017552d2 100644 --- a/scripts/create_plugin.py +++ b/scripts/create_plugin.py @@ -26,29 +26,24 @@ import pathlib import re import shutil -import site import sys import textwrap import typing import warnings -import markdown from typing import Union - -import docutils.core from os.path import exists -from qgis.core import QgsUserProfileManager, QgsUserProfile, QgsFileUtils - -site.addsitedir(pathlib.Path(__file__).parents[1]) # noqa - +import markdown +import docutils.core +from qgis.core import QgsFileUtils, QgsUserProfile, QgsUserProfileManager from qgis.testing import start_app -app = start_app() import enmapbox from enmapbox import DIR_REPO from enmapbox.qgispluginsupport.qps.make.deploy import QGISMetadataFileWriter, userProfileManager from enmapbox.qgispluginsupport.qps.utils import zipdir +app = start_app() # consider default Git location on Windows systems to avoid creating a Start-Up Script try: import git diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 23098f79..5514d25e 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -1,195 +1,411 @@ +import argparse +import datetime +import os import re import subprocess +import warnings +from concurrent.futures.thread import ThreadPoolExecutor from os import makedirs -from os.path import abspath, join, dirname, exists, basename -from shutil import rmtree -from typing import List +from pathlib import Path +from typing import Dict, List, Union + +from qgis.core import QgsApplication, QgsProcessingAlgorithm, QgsProcessingDestinationParameter, \ + QgsProcessingParameterDefinition import enmapbox -from enmapboxprocessing.algorithm.algorithms import algorithms +from enmapbox.algorithmprovider import EnMAPBoxProcessingProvider +from enmapbox.testing import start_app from enmapboxprocessing.enmapalgorithm import EnMAPProcessingAlgorithm, Group from enmapboxprocessing.glossary import injectGlossaryLinks -from qgis.core import QgsProcessingParameterDefinition, QgsProcessingDestinationParameter -try: - enmapboxdocumentation = __import__('enmapboxdocumentation') -except ModuleNotFoundError as ex: - raise ex +rootCodeRepo = Path(__file__).parent.parent dryRun = False # a fast way to check if all parameters are documented +PREFIX_AUTOGEN = '..\n ## AUTOGENERATED ' + +# environment to run qgis_process from python shell command +QGIS_PROCESS_ENV = os.environ.copy() +for k in ['QGIS_CUSTOM_CONFIG_PATH', 'QT3D_RENDERER']: + if k in QGIS_PROCESS_ENV: + QGIS_PROCESS_ENV.pop(k) + + +def parseRSTSections(text: str) -> Dict[Union[str, int], str]: + """ + Splits the intput text into manually defined and autogenerated sections + key = int -> manually defined section + key = str -> name of autogenerated section + """ + rx = re.compile(r'(\.\.\n ## AUTOGENERATED (.*) START)') + matches = rx.findall(text) + if len(matches) == 0: + return {0: text} + + results = dict() + current_start = 0 + current_man_section = 0 + for match in matches: + section = match[1] + key_start = match[0] + key_end = key_start[0:-5] + 'END\n' + + i0 = text.find(key_start) + ie = text.find(key_end) + len(key_end) + + if i0 > current_start: + results[current_man_section] = text[current_start: i0] + + results[section] = text[i0:ie] + + current_start = ie + current_man_section += 1 + + if current_start < len(text): + results[current_man_section + 1] = text[current_start:] + return results + + +def update_rst(text_old: str, text_new: str) -> str: + """ + Updates the autogenerated sections in text_old by autogenerated sections in text_new + """ + old_sections = parseRSTSections(text_old) + new_sections = parseRSTSections(text_new) + + merges_section = old_sections.copy() + for k, text in new_sections.items(): + if isinstance(k, str): + merges_section[k] = new_sections[k] + + final = '' + for key, text in merges_section.items(): + if isinstance(key, str): # autogenerated section + if len(final) > 0 and not final[-1] in ['\n']: + final += '\n' + final += text + return final + + +def create_or_update_rst(file, text: str): + """ + Create or update a rst file that contains the text. + In case the rst file already exists, it will be updated by + autogenerated content in text. + """ + file = Path(file) + if isinstance(text, list): + text = '\n'.join(text) + + if file.is_file(): + # update text with existing, none-autogenerated filetext + with open(file, 'r') as f: + old_text = f.read() + + if PREFIX_AUTOGEN in old_text: + text = update_rst(old_text, text) + else: + warnings.warn(f'Overwrite none-autogenerated file:\n {file}') + + with open(file, 'w', encoding='utf-8') as f: + f.write(text) + + return file + + +def generateAlgorithmRSTs(rootRst: Union[Path, str], + algorithms: List[QgsProcessingAlgorithm], + load_process_help: bool = True) -> List[str]: + """ + Create an rst file for each provided QgsProcessingAlgorithm. + """ + rootRst = Path(rootRst) + if isinstance(algorithms, QgsProcessingAlgorithm): + algorithms = [algorithms] + + n = len(algorithms) + if load_process_help: + print(f'Collect qgis_process help string for {n} algorithms (takes a while)... ', end='') + t0 = datetime.datetime.now() + qgis_process_help = collectQgsProcessAlgorithmHelp(algorithms, run_async=True) + print(f'Done: {datetime.datetime.now() - t0}') + else: + qgis_process_help = dict() + + print(f'Create rst files for {n} algorithms...') + for i, alg in enumerate(algorithms): + + afilename = alg.displayName().lower() + for c in r'/() ': + afilename = afilename.replace(c, '_') + filename = rootRst / groupFolderName(alg.group()) / '{}.rst'.format(afilename) + + section_adds = dict() + + # check for old-style processing_algorithms_includes files and include the rst code + filename_includes = rootRst.parent / f'{rootRst.name}_includes' / groupFolderName( + alg.group()) / '{}.rst'.format(afilename) + if filename_includes.is_file(): + with open(filename_includes) as f: + text = f.read() + section_adds['DESCRIPTION'] = text -def generateRST(): - # create folder - rootCodeRepo = abspath(join(dirname(enmapbox.__file__), '..')) - rootDocRepo = abspath(join(dirname(enmapboxdocumentation.__file__), '..')) + text = v3(alg, section_adds=section_adds, qgis_process_help=qgis_process_help) + + path = create_or_update_rst(filename, text) + print(f'Created {i + 1}/{n}: {path}') + + +def groupFolderName(group: str) -> str: + name = group.lower() + for c in ' ,*': + name = name.replace(c, '_') + return name + + +def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List[str]: + rootRst = Path(rootRst) + + groups = set([a.group() for a in algorithms]) + + index_files: List[str] = [] + n = len(groups) + print(f'Create index.rst files for {n} groups...') + for i, group in enumerate(groups): + # create group folder + groupFolder = rootRst / groupFolderName(group) + makedirs(groupFolder, exist_ok=True) + + group_points = '=' * len(group) + title = [ + f'.. _{group}:\n', + f'{group}', + f'{group_points}', + ] + text = wrapAutoGenerated(title, 'TITLE') + + toc = [ + '.. toctree::', + ' :maxdepth: 0', + ' :glob:\n', + ' *\n' + ] + text += wrapAutoGenerated(toc, 'TOC') + + filename = groupFolder / 'index.rst' + path = create_or_update_rst(filename, text) + index_files.append(f'{groupFolder.name}/{filename.name}') + print(f'Created {i + 1}/{n}: {path}') + + # write processing_algorithm.rst + title = [ + '.. _Processing Algorithms:\n', + 'Processing Algorithms', + '*********************\n'] + toc = ['.. toctree::', + ' :maxdepth: 1\n'] + for f in index_files: + toc.append(f' {f}') + + text = wrapAutoGenerated(title, 'TITLE') + wrapAutoGenerated(toc, 'TOC') + filename = rootRst / 'processing_algorithms.rst' + path = create_or_update_rst(filename, text) + print(f'Created {path}') + return index_files + + +def doc_repo_root() -> Path: + if 'PATH_DOCU_REPO' in os.environ: + rootDocRepo = Path(os.environ['PATH_DOCU_REPO']) + else: + rootDocRepo = rootCodeRepo.parent / 'enmap-box-documentation' + return rootDocRepo + + +def generateRST(rootRst: Union[Path, str], + algorithmIds: List[str] = None, + load_process_help: bool = True): + rootRst = Path(rootRst) + assert rootRst.is_dir() print(rootCodeRepo) - print(rootDocRepo) - - rootRst = join(rootDocRepo, 'source', 'usr_section', 'usr_manual', 'processing_algorithms') print(rootRst) - if exists(rootRst): - print('Delete root folder') - rmtree(rootRst) - makedirs(rootRst) - - groups = dict() - - nalg = 0 - algs = algorithms() - for alg in algs: - # print(alg.displayName()) - if Group.Experimental.name in alg.group(): - raise RuntimeError('Remove experimental algorithms from final release!') - if alg.group() not in groups: - groups[alg.group()] = dict() - groups[alg.group()][alg.displayName()] = alg - nalg += 1 - - print(f'Found {nalg} algorithms.') - - textProcessingAlgorithmsRst = '''Processing Algorithms -********************* - -.. toctree:: - :maxdepth: 1 - -''' + assert isinstance(EnMAPBoxProcessingProvider.instance(), EnMAPBoxProcessingProvider) + makedirs(rootRst, exist_ok=True) + + # filter algorithms + if algorithmIds is None: + algs = EnMAPBoxProcessingProvider.instance().algorithms() + for alg in EnMAPBoxProcessingProvider.instance().algorithms(): + if Group.Experimental.name in alg.group(): + raise RuntimeError('Remove experimental algorithms from final release!') + + else: + algs = QgsApplication.instance().processingRegistry().algorithms() + + algs = [a for a in algs if + a.id() in algorithmIds or a.name() in algorithmIds] + + # create group folders, /index.rst and processing_algorithms.rst + print(f'Create *.rst files for {len(algs)} algorithms') + generateGroupRSTs(rootRst, algs) + generateAlgorithmRSTs(rootRst, algs, load_process_help=load_process_help) + + +def wrapWithNewLines(text: str) -> str: + """ + Ensures that the input text begins and ends with a newline character + In case it does not end with a newline, 2 empty lines will be appended + :param text: str + :return: str + """ + + if isinstance(text, str) and len(text) > 0: + if text[0] != '\n': + text = '\n' + text + if text[-1] != '\n': + text += '\n\n' + return text + else: + return '' - for gkey in sorted(groups.keys()): - # create group folder - groupId = gkey.lower() - for c in ' ,*': - groupId = groupId.replace(c, '_') - groupFolder = join(rootRst, groupId) - makedirs(groupFolder) +def wrapAutoGenerated(rst_text: Union[str, list], section: str) -> str: + if isinstance(rst_text, list): + rst_text = '\n'.join(rst_text) - textProcessingAlgorithmsRst += '\n {}/index.rst'.format(basename(groupFolder)) + """Wraps autogenerated content with start and end markers""" + return f"\n{PREFIX_AUTOGEN}{section} START\n{wrapWithNewLines(rst_text)}\n{PREFIX_AUTOGEN}{section} END\n\n" - # create group index.rst - text = '''.. _{}:\n\n{} -{} -.. toctree:: - :maxdepth: 0 - :glob: +rx_needs_escape = re.compile(r'(?()\[\]])') - * -'''.format(gkey, gkey, '=' * len(gkey)) - filename = join(groupFolder, 'index.rst') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(text) - for akey in groups[gkey]: +def escape_rst(text: str) -> str: + if text is None: + return '' + parts = re.findall('(`[^`]*`_|[^`]*)', text) + for i, part in enumerate(parts): + if not (part.startswith('`') and part.endswith('`_')): + parts[i] = rx_needs_escape.sub(r'\\\g<1>', part) - algoId = akey.lower() - for c in [' ']: - algoId = algoId.replace(c, '_') + return ''.join(parts) - text = '''.. _{}: -{} -{} -{} +def collectQgsProcessAlgorithmHelp(algorithms: List[QgsProcessingAlgorithm], run_async: bool = False) -> Dict[str, str]: + results = dict() + n = len(algorithms) -'''.format(akey, '*' * len(akey), akey, '*' * len(akey)) + if not run_async: + for alg in algorithms: + results[alg.id()] = qgisProcessHelp(alg) + else: + def process_algorithm(alg): + return alg.id(), qgisProcessHelp(alg) - alg = groups[gkey][akey] - print(alg) + with ThreadPoolExecutor(max_workers=min(10, os.cpu_count())) as executor: + futures = executor.map(process_algorithm, algorithms) - if isinstance(alg, EnMAPProcessingAlgorithm): - alg.initAlgorithm() - text = v3(alg, text, groupFolder, algoId) - else: - print(f'skip {alg}') - continue - # assert 0 + for alg_id, help_text in futures: + results[alg_id] = help_text - filename = join(groupFolder, '{}.rst'.format(algoId)) - for c in r'/()': - filename = filename.replace(c, '_') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(text) + return results - filename = join(rootRst, 'processing_algorithms.rst') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(textProcessingAlgorithmsRst) - print('created RST file: ', filename) +def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None, qgis_process_help: dict = None): + """ -def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): - try: + :param alg: QgsProcessingAlgorithm + :param section_adds: dictionary with rst code snippeds to append to an autogenerated section. + """ + if isinstance(alg, EnMAPProcessingAlgorithm): helpParameters = {k: v for k, v in alg.helpParameters()} - except Exception: - assert 0 - - text += injectGlossaryLinks(alg.shortDescription()) + '\n\n' - - # include manual algo description - optionalRstFilename = join(groupFolder, algoId + '.rst').replace( - 'processing_algorithms', 'processing_algorithms_includes' - ) - if exists(optionalRstFilename): - text += f'.. include:: ../../processing_algorithms_includes/{basename(groupFolder)}/{algoId}.rst\n\n' - - text += '**Parameters**\n\n' + else: + helpParameters = dict() + + if section_adds is None: + section_adds = dict() + if qgis_process_help is None: + qgis_process_help = dict() + + # Title Section + title = alg.displayName() + dotline = '*' * len(title) + title_text = f".. _{title}:\n\n{dotline}\n{title}\n{dotline}\n" + title_text = utilsConvertHtmlLinksToRstLinks(title_text) # Convert any HTML links + + text = wrapAutoGenerated(title_text, "TITLE") + text += wrapWithNewLines(section_adds.get('TITLE')) + + # Description Section + description_text = injectGlossaryLinks(alg.shortDescription()) + description_text = utilsConvertHtmlLinksToRstLinks(description_text) + description_text = escape_rst(description_text) + text += wrapAutoGenerated(description_text, "DESCRIPTION") + text += wrapWithNewLines(section_adds.get('DESCRIPTION')) + + # Parameters Section + param_text = '**Parameters**\n\n' outputsHeadingCreated = False for pd in alg.parameterDefinitions(): assert isinstance(pd, QgsProcessingParameterDefinition) - pdhelp = helpParameters.get(pd.description(), 'undocumented') - if pdhelp == '': # an empty strings has to be set by the algo to actively hide an parameter - continue - if pdhelp == 'undocumented': # 'undocumented' is the default and must be overwritten by the algo! - assert 0, pd.description() + pdhelp = helpParameters.get(pd.description(), pd.description()) if not outputsHeadingCreated and isinstance(pd, QgsProcessingDestinationParameter): - text += '**Outputs**\n\n' + param_text += '\n\n**Outputs**\n\n' outputsHeadingCreated = True - text += '\n:guilabel:`{}` [{}]\n'.format(pd.description(), pd.type()) - + param_text += f'\n:guilabel:`{pd.description()}` [{pd.type()}]\n' pdhelp = injectGlossaryLinks(pdhelp) - - if False: # todo pd.flags() auswerten - text += ' Optional\n' - + pdhelp = utilsConvertHtmlLinksToRstLinks(pdhelp) # Convert HTML links in help text for line in pdhelp.split('\n'): - text += ' {}\n'.format(line) - - text += '\n' + param_text += f' {escape_rst(line)}\n' if pd.defaultValue() is not None: if isinstance(pd.defaultValue(), str) and '\n' in pd.defaultValue(): - text += ' Default::\n\n' + param_text += ' Default::\n\n' for line in pd.defaultValue().split('\n'): - text += ' {}\n'.format(line) + param_text += f' {escape_rst(line)}\n' else: - text += ' Default: *{}*\n\n'.format(pd.defaultValue()) + param_text += f' Default: *{escape_rst(str(pd.defaultValue()))}*\n\n' - if dryRun: - return '' + text += wrapAutoGenerated(param_text, "PARAMETERS") + text += wrapWithNewLines(section_adds.get('PARAMETERS')) + + # Command-line usage + if alg.id() in qgis_process_help: + helptext = qgis_process_help[alg.id()] + helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] + helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text - # convert HTML weblinks into RST weblinks - htmlLinks = utilsFindHtmlWeblinks(text) - for htmlLink in htmlLinks: - rstLink = utilsHtmlWeblinkToRstWeblink(htmlLink) - text = text.replace(htmlLink, rstLink) + usage_text = f"**Command-line usage**\n\n``>qgis_process help {alg.id()}``::\n\n" + usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) - # add qgis_process help - algoId = 'enmapbox:' + alg.name() - print(algoId) - result = subprocess.run(['qgis_process', 'help', algoId], stdout=subprocess.PIPE) - helptext = result.stdout.decode('cp1252') # use Windows codepage 1252 to avoid problems with special characters - helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] - helptext = '\n'.join([' ' + line for line in helptext.splitlines()]) + text += wrapAutoGenerated(usage_text, "COMMAND USAGE") + text += wrapWithNewLines(section_adds.get('COMMAND USAGE')) + + return text - text += '**Command-line usage**\n\n' \ - f'``>qgis_process help {algoId}``::\n\n' - text += helptext +def qgisProcessHelp(algorithm: QgsProcessingAlgorithm) -> str: + result = subprocess.run(['qgis_process', 'help', algorithm.id()], + env=QGIS_PROCESS_ENV, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.returncode != 0: + s = "" + assert result.returncode == 0, result.stderr.decode() + helptext = result.stdout.decode('cp1252') + return helptext + + +def utilsConvertHtmlLinksToRstLinks(text: str) -> str: + """Convert all HTML-style links in the text to RST-style links.""" + links = utilsFindHtmlWeblinks(text) # Find all HTML links in the text + for html_link in links: + rst_link = utilsHtmlWeblinkToRstWeblink(html_link) + text = text.replace(html_link, rst_link) return text @@ -211,4 +427,30 @@ def utilsHtmlWeblinkToRstWeblink(htmlText: str) -> str: if __name__ == '__main__': - generateRST() + parser = argparse.ArgumentParser(description='Generates the documentation for EnMAPBox processing algorithms ', + formatter_class=argparse.RawTextHelpFormatter) + + doc_root_default = doc_repo_root() / 'source' / 'usr_section' / 'usr_manual' / 'processing_algorithms' + + parser.add_argument('-r', '--rst_root', + required=False, + default=str(doc_root_default), + help=f'Root folder to write the RST files. Defaults to {doc_root_default}') + + parser.add_argument('-a', '--algs', + required=False, + nargs='*', + default=None, + help='List of algorithms ids to generate the documentation for') + + args = parser.parse_args() + + rootRst = Path(args.rst_root) + if not rootRst.is_absolute(): + rootRst = rootCodeRepo / rootRst + + assert rootRst.is_dir(), f'Directory does not exists: {rootRst}. Use --rst_root to specify location.' + + start_app() + enmapbox.initAll() + generateRST(rootRst, args.algs) diff --git a/tests/enmap-box/scripts/test_create_processing_rst.py b/tests/enmap-box/scripts/test_create_processing_rst.py new file mode 100644 index 00000000..4817e178 --- /dev/null +++ b/tests/enmap-box/scripts/test_create_processing_rst.py @@ -0,0 +1,144 @@ +import datetime +import os +import shutil +import subprocess +import unittest + +from enmapbox.algorithmprovider import EnMAPBoxProcessingProvider +from enmapbox.testing import start_app, TestCase +from enmapbox import registerEnMAPBoxProcessingProvider +from scripts.create_processing_rst import __file__ as path_processing_script, collectQgsProcessAlgorithmHelp, \ + escape_rst, \ + generateRST, QGIS_PROCESS_ENV, update_rst + +start_app() +registerEnMAPBoxProcessingProvider() + +RST_ORIGINAL = """ +before +.. + ## AUTOGENERATED FOO START + +blablala +.. + ## AUTOGENERATED FOO END +ending 1""" + +RST_UPDATE = """ + +.. + ## AUTOGENERATED FOO START +changed blablabla +.. + ## AUTOGENERATED FOO END + +DO_NOT_COPY_CODE_FROM_OUTSIDE_AUTOGENERATED_SECTIONS + +.. + ## AUTOGENERATED NEW START +new blablabla +.. + ## AUTOGENERATED NEW END +""" + +RST_RESULT = """ +before +.. + ## AUTOGENERATED FOO START +changed blablabla +.. + ## AUTOGENERATED FOO END +ending 1 +.. + ## AUTOGENERATED NEW START +new blablabla +.. + ## AUTOGENERATED NEW END +""" + + +class CreateProcessingRSTTestCases(TestCase): + + def test_update_rst(self): + self.assertEqual(update_rst(RST_ORIGINAL, ''), RST_ORIGINAL) + self.assertEqual(update_rst(RST_ORIGINAL, 'not in AUTOGENERATE section'), RST_ORIGINAL) + self.assertEqual(update_rst(RST_ORIGINAL, RST_UPDATE), RST_RESULT) + + def test_generateRST(self): + dir_tmp = self.createTestOutputDirectory() + dir_rst_root = dir_tmp / 'rst_root1' + generateRST(dir_rst_root, + algorithmIds=['gdal:translate', 'enmapbox:Build3DCube'], + load_process_help=False) + + expected_paths = [ + dir_rst_root / 'auxilliary' / 'build_3d_cube.rst', + dir_rst_root / 'raster_conversion' / 'translate__convert_format_.rst' + ] + for p in expected_paths: + self.assertTrue(p.is_file(), msg=f'File not created exist: {p}') + + @unittest.skipIf(TestCase.runsInCI(), 'Manual calls only') + def test_buildall(self): + + result = subprocess.run(['python', str(path_processing_script)], + env=QGIS_PROCESS_ENV, + stderr=subprocess.PIPE + ) + self.assertEqual(result.returncode, 0, msg=result.stderr.decode()) + + def test_escape_rst(self): + + examples = [ + ('* wor*d `line `_ ', '\\* wor\\*d `line `_ '), + ('a*c', 'a\\*c'), + ('abc', 'abc'), + ('*c', '\\*c'), + ('c*', 'c\\*'), + ('[a]', '\\[a\\]'), + (']a[', '\\]a\\['), + ] + for (text, expected) in examples: + result = escape_rst(text) + print(f'{text} -> {result} ({expected})') + self.assertEqual(result, expected) + + @unittest.skipIf(TestCase.runsInCI(), 'Manual testing only') + def test_collect_algorithm_help(self): + + algs = EnMAPBoxProcessingProvider.instance().algorithms() + algs = algs[0:100] + t0 = datetime.datetime.now() + results1 = collectQgsProcessAlgorithmHelp(algs, run_async=True) + t1 = datetime.datetime.now() + results2 = collectQgsProcessAlgorithmHelp(algs, run_async=False) + t2 = datetime.datetime.now() + self.assertEqual(results1, results2) + print(f'Async: {t1 - t0}\nNormal: {t2 - t1}') + + def test_script(self): + dir_tmp = self.createTestOutputDirectory() + dir_rst_root = dir_tmp / 'rst_root2' + if dir_rst_root.is_dir(): + shutil.rmtree(dir_rst_root) + os.makedirs(dir_rst_root) + + result = subprocess.run(['python', str(path_processing_script), + '-r', str(dir_rst_root), + '-a', 'Build3DCube'], + env=QGIS_PROCESS_ENV, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + self.assertEqual(result.returncode, 0, msg=result.stderr.decode()) + + expected_files = [ + dir_rst_root / 'processing_algorithms.rst', + dir_rst_root / 'auxilliary' / 'build_3d_cube.rst', + dir_rst_root / 'auxilliary' / 'index.rst', + ] + for p in expected_files: + self.assertTrue(p.is_file(), msg=f'Missing file: {p}') + + +if __name__ == '__main__': + unittest.main()