From 44b69bc195a43297f71921ceb7c28a9e2ad6bd18 Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 16 Sep 2024 12:19:48 +0200 Subject: [PATCH 01/16] resolved issue_80 : the processing algorithms now have autogenerated sections and manually added sections. --- scripts/create_processing_rst.py | 93 +++++++++++++++++++------------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 23098f79..9a12c8bb 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -1,7 +1,9 @@ +import os import re import subprocess from os import makedirs from os.path import abspath, join, dirname, exists, basename +from pathlib import Path from shutil import rmtree from typing import List @@ -11,18 +13,21 @@ from enmapboxprocessing.glossary import injectGlossaryLinks from qgis.core import QgsProcessingParameterDefinition, QgsProcessingDestinationParameter -try: - enmapboxdocumentation = __import__('enmapboxdocumentation') -except ModuleNotFoundError as ex: - raise ex +rootCodeRepo = Path(__file__).parent.parent + +if 'PATH_DOCU_REPO' in os.environ: + rootDocRepo = Path(os.environ['PATH_DOCU_REPO']) +else: + rootDocRepo = rootCodeRepo.parent / 'enmap-box-documentation' + + os.makedirs(rootDocRepo, exist_ok=True) +assert rootDocRepo.is_dir() dryRun = False # a fast way to check if all parameters are documented def generateRST(): # create folder - rootCodeRepo = abspath(join(dirname(enmapbox.__file__), '..')) - rootDocRepo = abspath(join(dirname(enmapboxdocumentation.__file__), '..')) print(rootCodeRepo) print(rootDocRepo) @@ -92,9 +97,9 @@ def generateRST(): {} {} -{} -'''.format(akey, '*' * len(akey), akey, '*' * len(akey)) + +'''.format(akey, '*' * len(akey), akey) alg = groups[gkey][akey] print(alg) @@ -105,7 +110,6 @@ def generateRST(): else: print(f'skip {alg}') continue - # assert 0 filename = join(groupFolder, '{}.rst'.format(algoId)) for c in r'/()': @@ -125,24 +129,43 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): except Exception: assert 0 - text += injectGlossaryLinks(alg.shortDescription()) + '\n\n' + # Clear autogenerated title, removing any manual _Build 3D Cube before this block + title = alg.displayName() + title_underline = '*' * len(title) # This will ensure proper length of the underline + + # Generate title with proper format and markers + text = '''.. ## AUTOGENERATED TITLE START ## +.. _{}: + +{} +{} +.. ## AUTOGENERATED TITLE END ## +'''.format(title, title, title_underline) + + text += '\nHere I can add my manual defined rst code\n\n' - # include manual algo description - optionalRstFilename = join(groupFolder, algoId + '.rst').replace( - 'processing_algorithms', 'processing_algorithms_includes' - ) - if exists(optionalRstFilename): - text += f'.. include:: ../../processing_algorithms_includes/{basename(groupFolder)}/{algoId}.rst\n\n' + # Autogenerated Description + text += '''.. ## AUTOGENERATED DESCRIPTION START ## +{} +.. ## AUTOGENERATED DESCRIPTION END ## +'''.format(injectGlossaryLinks(alg.shortDescription())) + + # Manual content after description + text += '\nHere I can add more manual defined rst code\n\n' + + # Autogenerated Parameters Section + text += '\n**Parameters**\n\nHere I can add more manual content for Parameters.\n\n' + text += '''.. ## AUTOGENERATED PARAMETERS START ## +''' - text += '**Parameters**\n\n' outputsHeadingCreated = False for pd in alg.parameterDefinitions(): assert isinstance(pd, QgsProcessingParameterDefinition) pdhelp = helpParameters.get(pd.description(), 'undocumented') - if pdhelp == '': # an empty strings has to be set by the algo to actively hide an parameter + if pdhelp == '': continue - if pdhelp == 'undocumented': # 'undocumented' is the default and must be overwritten by the algo! + if pdhelp == 'undocumented': assert 0, pd.description() if not outputsHeadingCreated and isinstance(pd, QgsProcessingDestinationParameter): @@ -153,9 +176,6 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): pdhelp = injectGlossaryLinks(pdhelp) - if False: # todo pd.flags() auswerten - text += ' Optional\n' - for line in pdhelp.split('\n'): text += ' {}\n'.format(line) @@ -169,26 +189,27 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): else: text += ' Default: *{}*\n\n'.format(pd.defaultValue()) - if dryRun: - return '' + text += '''.. ## AUTOGENERATED PARAMETERS END ## +''' - # convert HTML weblinks into RST weblinks - htmlLinks = utilsFindHtmlWeblinks(text) - for htmlLink in htmlLinks: - rstLink = utilsHtmlWeblinkToRstWeblink(htmlLink) - text = text.replace(htmlLink, rstLink) + # Allow manual addition after Parameters section + text += '\nHere I can add even more manual defined rst code\n\n' - # add qgis_process help + # Add command-line usage algoId = 'enmapbox:' + alg.name() - print(algoId) result = subprocess.run(['qgis_process', 'help', algoId], stdout=subprocess.PIPE) - helptext = result.stdout.decode('cp1252') # use Windows codepage 1252 to avoid problems with special characters + helptext = result.stdout.decode('cp1252') helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] - helptext = '\n'.join([' ' + line for line in helptext.splitlines()]) - text += '**Command-line usage**\n\n' \ - f'``>qgis_process help {algoId}``::\n\n' - text += helptext + # Autogenerated Command-line Usage + text += '''.. ## AUTOGENERATED COMMAND USAGE START ## +**Command-line usage** + +``>qgis_process help {}``:: + +{} +.. ## AUTOGENERATED COMMAND USAGE END ## +'''.format(algoId, '\n'.join([' ' + line for line in helptext.splitlines()])) return text From 52a7828c4c80b1a8dd3b51076a847cf5fc796dc6 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 24 Sep 2024 23:26:51 +0200 Subject: [PATCH 02/16] issue_80: final editing, autogenerated sections defined by a separate function and style of comments corrected. --- scripts/create_processing_rst.py | 76 ++++++++++++-------------------- 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 9a12c8bb..c9b811ca 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -122,6 +122,10 @@ def generateRST(): f.write(textProcessingAlgorithmsRst) print('created RST file: ', filename) +def wrapAutoGenerated(rst_text: str, section: str) -> str: + """Wraps autogenerated content with start and end markers""" + return f"..\n ## AUTOGENERATED START {section}\n\n{rst_text}\n\n..\n ## AUTOGENERATED END {section}\n" + def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): try: @@ -129,35 +133,23 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): except Exception: assert 0 - # Clear autogenerated title, removing any manual _Build 3D Cube before this block + # Title Section title = alg.displayName() - title_underline = '*' * len(title) # This will ensure proper length of the underline - - # Generate title with proper format and markers - text = '''.. ## AUTOGENERATED TITLE START ## -.. _{}: + title_text = f".. _{title}:\n\n{title}\n{'*' * len(title)}\n" + text = wrapAutoGenerated(title_text, "TITLE") -{} -{} -.. ## AUTOGENERATED TITLE END ## -'''.format(title, title, title_underline) - - text += '\nHere I can add my manual defined rst code\n\n' - - # Autogenerated Description - text += '''.. ## AUTOGENERATED DESCRIPTION START ## -{} -.. ## AUTOGENERATED DESCRIPTION END ## -'''.format(injectGlossaryLinks(alg.shortDescription())) + # Manual section after the title + text += "\nHere I can add manually defined title-related content.\n\n" - # Manual content after description - text += '\nHere I can add more manual defined rst code\n\n' + # Description Section + description_text = injectGlossaryLinks(alg.shortDescription()) + text += wrapAutoGenerated(description_text, "DESCRIPTION") - # Autogenerated Parameters Section - text += '\n**Parameters**\n\nHere I can add more manual content for Parameters.\n\n' - text += '''.. ## AUTOGENERATED PARAMETERS START ## -''' + # Manual section after the description + text += "\nHere I can add manually defined description-related content.\n\n" + # Parameters Section + param_text = '' outputsHeadingCreated = False for pd in alg.parameterDefinitions(): assert isinstance(pd, QgsProcessingParameterDefinition) @@ -169,47 +161,37 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): assert 0, pd.description() if not outputsHeadingCreated and isinstance(pd, QgsProcessingDestinationParameter): - text += '**Outputs**\n\n' + param_text += '**Outputs**\n\n' outputsHeadingCreated = True - text += '\n:guilabel:`{}` [{}]\n'.format(pd.description(), pd.type()) - + param_text += f'\n:guilabel:`{pd.description()}` [{pd.type()}]\n' pdhelp = injectGlossaryLinks(pdhelp) - for line in pdhelp.split('\n'): - text += ' {}\n'.format(line) - - text += '\n' + param_text += f' {line}\n' if pd.defaultValue() is not None: if isinstance(pd.defaultValue(), str) and '\n' in pd.defaultValue(): - text += ' Default::\n\n' + param_text += ' Default::\n\n' for line in pd.defaultValue().split('\n'): - text += ' {}\n'.format(line) + param_text += f' {line}\n' else: - text += ' Default: *{}*\n\n'.format(pd.defaultValue()) + param_text += f' Default: *{pd.defaultValue()}*\n\n' - text += '''.. ## AUTOGENERATED PARAMETERS END ## -''' + text += wrapAutoGenerated(param_text, "PARAMETERS") - # Allow manual addition after Parameters section - text += '\nHere I can add even more manual defined rst code\n\n' + # Manual section after the parameters + text += "\nHere I can add manually defined parameters-related content.\n\n" - # Add command-line usage + # Command-line usage algoId = 'enmapbox:' + alg.name() result = subprocess.run(['qgis_process', 'help', algoId], stdout=subprocess.PIPE) helptext = result.stdout.decode('cp1252') helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] - # Autogenerated Command-line Usage - text += '''.. ## AUTOGENERATED COMMAND USAGE START ## -**Command-line usage** - -``>qgis_process help {}``:: + usage_text = f"**Command-line usage**\n\n``>qgis_process help {algoId}``::\n\n" + usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) -{} -.. ## AUTOGENERATED COMMAND USAGE END ## -'''.format(algoId, '\n'.join([' ' + line for line in helptext.splitlines()])) + text += wrapAutoGenerated(usage_text, "COMMAND USAGE") return text From bdb9b0fcd6f5dcee4d7fc8e80783d82a281e9439 Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 15 Nov 2024 02:07:33 +0530 Subject: [PATCH 03/16] issue_80 feedback corrections. Now the manually added sections are marked in the code as comments and not don't appear in the rst. --- scripts/create_processing_rst.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index c9b811ca..e3863823 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -138,15 +138,13 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): title_text = f".. _{title}:\n\n{title}\n{'*' * len(title)}\n" text = wrapAutoGenerated(title_text, "TITLE") - # Manual section after the title - text += "\nHere I can add manually defined title-related content.\n\n" + # Note: Add manually defined title-related content here if needed # Description Section description_text = injectGlossaryLinks(alg.shortDescription()) text += wrapAutoGenerated(description_text, "DESCRIPTION") - # Manual section after the description - text += "\nHere I can add manually defined description-related content.\n\n" + # Note: Add manually defined description-related content here if needed # Parameters Section param_text = '' @@ -179,8 +177,7 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): text += wrapAutoGenerated(param_text, "PARAMETERS") - # Manual section after the parameters - text += "\nHere I can add manually defined parameters-related content.\n\n" + # Note: Add manually defined parameters-related content here if needed # Command-line usage algoId = 'enmapbox:' + alg.name() From 1c6c651815dbb394f208a4ecdf60ce2dba8f6a2d Mon Sep 17 00:00:00 2001 From: Aryan Date: Wed, 20 Nov 2024 22:48:33 +0100 Subject: [PATCH 04/16] issue_80 feedback corrections. All links converted into rst style links . --- scripts/create_processing_rst.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index e3863823..9219b907 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -136,16 +136,14 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): # Title Section title = alg.displayName() title_text = f".. _{title}:\n\n{title}\n{'*' * len(title)}\n" + title_text = utilsConvertHtmlLinksToRstLinks(title_text) # Convert any HTML links text = wrapAutoGenerated(title_text, "TITLE") - # Note: Add manually defined title-related content here if needed - # Description Section description_text = injectGlossaryLinks(alg.shortDescription()) + description_text = utilsConvertHtmlLinksToRstLinks(description_text) # Convert HTML links text += wrapAutoGenerated(description_text, "DESCRIPTION") - # Note: Add manually defined description-related content here if needed - # Parameters Section param_text = '' outputsHeadingCreated = False @@ -164,6 +162,7 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): param_text += f'\n:guilabel:`{pd.description()}` [{pd.type()}]\n' pdhelp = injectGlossaryLinks(pdhelp) + pdhelp = utilsConvertHtmlLinksToRstLinks(pdhelp) # Convert HTML links in help text for line in pdhelp.split('\n'): param_text += f' {line}\n' @@ -177,13 +176,12 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): text += wrapAutoGenerated(param_text, "PARAMETERS") - # Note: Add manually defined parameters-related content here if needed - # Command-line usage algoId = 'enmapbox:' + alg.name() result = subprocess.run(['qgis_process', 'help', algoId], stdout=subprocess.PIPE) helptext = result.stdout.decode('cp1252') helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] + helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text usage_text = f"**Command-line usage**\n\n``>qgis_process help {algoId}``::\n\n" usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) @@ -193,6 +191,15 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): return text +def utilsConvertHtmlLinksToRstLinks(text: str) -> str: + """Convert all HTML-style links in the text to RST-style links.""" + links = utilsFindHtmlWeblinks(text) # Find all HTML links in the text + for html_link in links: + rst_link = utilsHtmlWeblinkToRstWeblink(html_link) + text = text.replace(html_link, rst_link) + return text + + def utilsFindHtmlWeblinks(text) -> List[str]: match_: re.Match starts = [match_.start() for match_ in re.finditer(' Date: Fri, 22 Nov 2024 14:19:29 +0100 Subject: [PATCH 05/16] refactoring --- scripts/create_processing_rst.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 9219b907..ec3a6cc4 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -1,6 +1,7 @@ import os import re import subprocess +import textwrap from os import makedirs from os.path import abspath, join, dirname, exists, basename from pathlib import Path @@ -41,7 +42,6 @@ def generateRST(): groups = dict() - nalg = 0 algs = algorithms() for alg in algs: # print(alg.displayName()) @@ -50,17 +50,16 @@ def generateRST(): if alg.group() not in groups: groups[alg.group()] = dict() groups[alg.group()][alg.displayName()] = alg - nalg += 1 - print(f'Found {nalg} algorithms.') + print(f'Found {len(algs)} algorithms.') - textProcessingAlgorithmsRst = '''Processing Algorithms -********************* - -.. toctree:: - :maxdepth: 1 - -''' + textProcessingAlgorithmsRst = textwrap.dedent( + """Processing Algorithms + ********************* + + .. toctree:: + :maxdepth: 1 + """) for gkey in sorted(groups.keys()): From 17bb0508e0f981ecf648f6c84302bd75a652ad18 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 08:22:15 +0100 Subject: [PATCH 06/16] moved EnMAPBoxProcessingProvider instance to EnMAPBoxProcessingProvider class Signed-off-by: jakimowb --- enmapbox/__init__.py | 11 +++-------- enmapbox/algorithmprovider.py | 9 +++++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/enmapbox/__init__.py b/enmapbox/__init__.py index af53b688..02477984 100644 --- a/enmapbox/__init__.py +++ b/enmapbox/__init__.py @@ -34,10 +34,9 @@ import warnings from osgeo import gdal - from qgis.PyQt.QtCore import QSettings from qgis.PyQt.QtGui import QIcon -from qgis.core import Qgis, QgsApplication, QgsProcessingRegistry, QgsProcessingProvider, QgsProcessingAlgorithm +from qgis.core import Qgis, QgsApplication, QgsProcessingAlgorithm, QgsProcessingProvider, QgsProcessingRegistry from qgis.gui import QgisInterface, QgsMapLayerConfigWidgetFactory # provide shortcuts @@ -96,7 +95,6 @@ ENMAP_BOX_KEY = 'EnMAP-Box' -_ENMAPBOX_PROCESSING_PROVIDER: QgsProcessingProvider = None _ENMAPBOX_MAPLAYER_CONFIG_WIDGET_FACTORIES: typing.List[QgsMapLayerConfigWidgetFactory] = [] gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', 'YES') @@ -262,13 +260,11 @@ def registerEnMAPBoxProcessingProvider(): assert isinstance(registry, QgsProcessingRegistry) provider = registry.providerById(ID) if not isinstance(provider, QgsProcessingProvider): - provider = EnMAPBoxProcessingProvider() + provider = EnMAPBoxProcessingProvider.instance() registry.addProvider(provider) assert isinstance(provider, EnMAPBoxProcessingProvider) assert id(registry.providerById(ID)) == id(provider) - global _ENMAPBOX_PROCESSING_PROVIDER - _ENMAPBOX_PROCESSING_PROVIDER = provider try: existingAlgNames = [a.name() for a in registry.algorithms() if a.groupId() == provider.id()] @@ -293,8 +289,7 @@ def unregisterEnMAPBoxProcessingProvider(): provider = registry.providerById(ID) if isinstance(provider, EnMAPBoxProcessingProvider): - global _ENMAPBOX_PROCESSING_PROVIDER - _ENMAPBOX_PROCESSING_PROVIDER = None + EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER = None # this deletes the C++ object registry.removeProvider(ID) diff --git a/enmapbox/algorithmprovider.py b/enmapbox/algorithmprovider.py index bc891459..e37f4cca 100644 --- a/enmapbox/algorithmprovider.py +++ b/enmapbox/algorithmprovider.py @@ -50,6 +50,15 @@ class EnMAPBoxProcessingProvider(QgsProcessingProvider): It enhances the "standard" processing.core.AlgorithmProvider by functionality to add and remove GeoAlgorithms during runtime. """ + _ENMAPBOX_PROCESSING_PROVIDER = None + + @staticmethod + def instance() -> 'EnMAPBoxProcessingProvider': + + if EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER is None: + EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER = EnMAPBoxProcessingProvider() + return EnMAPBoxProcessingProvider._ENMAPBOX_PROCESSING_PROVIDER + def __init__(self): super(EnMAPBoxProcessingProvider, self).__init__() # internal list of GeoAlgorithms. Is used on re-loads and can be manipulated From d1ec3f6b3fe975e5b5947febd61ee0629174a9a8 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 08:23:29 +0100 Subject: [PATCH 07/16] refactored create_processing_rst.py, can now be called with running QgsApplication instance Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 256 ++++++++++++++++++++----------- 1 file changed, 170 insertions(+), 86 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index ec3a6cc4..afb9c229 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -1,18 +1,20 @@ +import argparse import os import re import subprocess -import textwrap from os import makedirs -from os.path import abspath, join, dirname, exists, basename +from os.path import join from pathlib import Path -from shutil import rmtree -from typing import List +from typing import List, Union + +from qgis.core import QgsProcessingAlgorithm, QgsProcessingDestinationParameter, QgsProcessingParameterDefinition import enmapbox -from enmapboxprocessing.algorithm.algorithms import algorithms +from enmapbox.algorithmprovider import EnMAPBoxProcessingProvider +from enmapbox.qgispluginsupport.qps.utils import file_search +from enmapbox.testing import start_app from enmapboxprocessing.enmapalgorithm import EnMAPProcessingAlgorithm, Group from enmapboxprocessing.glossary import injectGlossaryLinks -from qgis.core import QgsProcessingParameterDefinition, QgsProcessingDestinationParameter rootCodeRepo = Path(__file__).parent.parent @@ -26,111 +28,162 @@ dryRun = False # a fast way to check if all parameters are documented +# environment to run qgis_process from python shell command +QGIS_PROCESS_ENV = os.environ.copy() +for k in ['QGIS_CUSTOM_CONFIG_PATH', 'QT3D_RENDERER']: + if k in QGIS_PROCESS_ENV: + QGIS_PROCESS_ENV.pop(k) -def generateRST(): - # create folder - print(rootCodeRepo) - print(rootDocRepo) - rootRst = join(rootDocRepo, 'source', 'usr_section', 'usr_manual', 'processing_algorithms') - print(rootRst) +def read_existing_rsts(rootRst: Union[str, Path]): + rootRst = Path(rootRst) - if exists(rootRst): - print('Delete root folder') - rmtree(rootRst) - makedirs(rootRst) + infos = dict() + if not rootRst.is_dir(): + return infos - groups = dict() + for p in file_search(rootRst, '*.rst', recursive=True): + with open(p, 'r') as f: + rst_test = f.read() - algs = algorithms() - for alg in algs: - # print(alg.displayName()) - if Group.Experimental.name in alg.group(): - raise RuntimeError('Remove experimental algorithms from final release!') - if alg.group() not in groups: - groups[alg.group()] = dict() - groups[alg.group()][alg.displayName()] = alg + infos[rootRst.as_posix()] = rst_test + return infos - print(f'Found {len(algs)} algorithms.') - textProcessingAlgorithmsRst = textwrap.dedent( - """Processing Algorithms - ********************* - - .. toctree:: - :maxdepth: 1 - """) +def write_and_update(file, text): + file = Path(file) + # todo: if file exists, keep none-autogenerated parts - for gkey in sorted(groups.keys()): + if isinstance(text, list): + text = '\n'.join(text) - # create group folder - groupId = gkey.lower() - for c in ' ,*': - groupId = groupId.replace(c, '_') - groupFolder = join(rootRst, groupId) - makedirs(groupFolder) + with open(file, 'w', encoding='utf-8') as f: + f.write(text) - textProcessingAlgorithmsRst += '\n {}/index.rst'.format(basename(groupFolder)) + print(f'Created {file}') - # create group index.rst - text = '''.. _{}:\n\n{} -{} -.. toctree:: - :maxdepth: 0 - :glob: +def generateAlgorithmRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List[str]: + rootRst = Path(rootRst) + if isinstance(algorithms, QgsProcessingAlgorithm): + algorithms = [algorithms] - * -'''.format(gkey, gkey, '=' * len(gkey)) - filename = join(groupFolder, 'index.rst') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(text) + for alg in algorithms: + name = alg.name() + algoId = name.lower() + for c in [' ']: + algoId = algoId.replace(c, '_') - for akey in groups[gkey]: + text = [ + f'.. _{algoId}\n', + '*' * len(name), + name, + '*' * len(name), + ] + text.append(v3(alg)) - algoId = akey.lower() - for c in [' ']: - algoId = algoId.replace(c, '_') + afilename = algoId + for c in r'/()': + afilename = afilename.replace(c, '_') - text = '''.. _{}: + filename = rootRst / groupFolderName(alg.group()) / '{}.rst'.format(afilename) + write_and_update(filename, text) -{} -{} +def groupFolderName(group: str) -> str: + name = group.lower() + for c in ' ,*': + name = name.replace(c, '_') + return name -'''.format(akey, '*' * len(akey), akey) - alg = groups[gkey][akey] - print(alg) +def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List[str]: + rootRst = Path(rootRst) - if isinstance(alg, EnMAPProcessingAlgorithm): - alg.initAlgorithm() - text = v3(alg, text, groupFolder, algoId) - else: - print(f'skip {alg}') - continue + groups = set([a.group() for a in algorithms]) + + index_files: List[str] = [] + for group in groups: + # create group folder + groupFolder = rootRst / groupFolderName(group) + makedirs(groupFolder, exist_ok=True) + + # create group index.rst + # create group index.rst + text = [ + f'.. _{group}:\n', + f'{group}', + f'{'=' * len(group)}', + '.. toctree::', + ' :maxdepth: 0', + ' :glob:\n', + ' *\n' + ] + + filename = groupFolder / 'index.rst' + write_and_update(filename, '\n'.join(text)) + index_files.append(f'{groupFolder.name}/{filename.name}') + + # write processing_algorithsm.rst + textProcessingAlgorithmsRst = [ + 'Processing Algorithms', + '*********************', + '', + '.. toctree::', + ' :maxdepth: 1\n' + ] + textProcessingAlgorithmsRst.extend([f' {f}' for f in index_files]) + + filename = rootRst / 'processing_algorithms.rst' + write_and_update(filename, '\n'.join(textProcessingAlgorithmsRst)) + + return index_files + + +def generateRST(rootRst=None, + algorithmIds: List[str] = None + ): + # create folder + print(rootCodeRepo) + + if rootRst is None: + print(rootDocRepo) + rootRst = join(rootDocRepo, 'source', 'usr_section', 'usr_manual', 'processing_algorithms') + + print(rootRst) + + assert isinstance(EnMAPBoxProcessingProvider.instance(), EnMAPBoxProcessingProvider) + makedirs(rootRst, exist_ok=True) + + # filter algorithms + algs: List[QgsProcessingAlgorithm] = [] + for alg in EnMAPBoxProcessingProvider.instance().algorithms(): + if Group.Experimental.name in alg.group(): + raise RuntimeError('Remove experimental algorithms from final release!') + algs.append(alg) - filename = join(groupFolder, '{}.rst'.format(algoId)) - for c in r'/()': - filename = filename.replace(c, '_') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(text) + # optional: filter algorithms (e.g. for testing) + if algorithmIds: + algs = [a for a in algs if a.id() in algorithmIds or a.name() in algorithmIds] + + # create group folders, /index.rst and processing_algorithms.rst + + generateGroupRSTs(rootRst, algs) + generateAlgorithmRSTs(rootRst, algs) + + s = "" - filename = join(rootRst, 'processing_algorithms.rst') - with open(filename, mode='w', encoding='utf-8') as f: - f.write(textProcessingAlgorithmsRst) - print('created RST file: ', filename) def wrapAutoGenerated(rst_text: str, section: str) -> str: """Wraps autogenerated content with start and end markers""" return f"..\n ## AUTOGENERATED START {section}\n\n{rst_text}\n\n..\n ## AUTOGENERATED END {section}\n" -def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): - try: +def v3(alg: QgsProcessingAlgorithm): + if isinstance(alg, EnMAPProcessingAlgorithm): helpParameters = {k: v for k, v in alg.helpParameters()} - except Exception: - assert 0 + else: + helpParameters = dict() # Title Section title = alg.displayName() @@ -176,13 +229,11 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): text += wrapAutoGenerated(param_text, "PARAMETERS") # Command-line usage - algoId = 'enmapbox:' + alg.name() - result = subprocess.run(['qgis_process', 'help', algoId], stdout=subprocess.PIPE) - helptext = result.stdout.decode('cp1252') + helptext = qgis_process_help(alg) helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text - usage_text = f"**Command-line usage**\n\n``>qgis_process help {algoId}``::\n\n" + usage_text = f"**Command-line usage**\n\n``>qgis_process help {alg.id()}``::\n\n" usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) text += wrapAutoGenerated(usage_text, "COMMAND USAGE") @@ -190,6 +241,16 @@ def v3(alg: EnMAPProcessingAlgorithm, text, groupFolder, algoId): return text +def qgis_process_help(algorithm: QgsProcessingAlgorithm) -> str: + result = subprocess.run(['qgis_process', 'help', algorithm.id()], + env=QGIS_PROCESS_ENV, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + assert result.returncode == 0 + helptext = result.stdout.decode('cp1252') + return helptext + + def utilsConvertHtmlLinksToRstLinks(text: str) -> str: """Convert all HTML-style links in the text to RST-style links.""" links = utilsFindHtmlWeblinks(text) # Find all HTML links in the text @@ -217,4 +278,27 @@ def utilsHtmlWeblinkToRstWeblink(htmlText: str) -> str: if __name__ == '__main__': - generateRST() + parser = argparse.ArgumentParser(description='Generates the documentation for EnMAPBox processing algorithms ', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('-r', '--rst_root', + required=False, + default=rootDocRepo.as_posix(), + help=f'Root folder to write the RST files. Defaults to {rootDocRepo}') + + parser.add_argument('-a', '--algs', + required=False, + nargs='*', + default=None, + help='List of algorithms ids to generate the documentation for') + + args = parser.parse_args() + + rootRst = Path(args.rst_root) + if not rootRst.is_absolute(): + rootRst = rootCodeRepo / rootRst + + algs = args.algs + + start_app() + enmapbox.initAll() + generateRST(rootRst, algs) From a5dda61e6c638d2ce52739f7936ba1982c6ca1cd Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 08:44:59 +0100 Subject: [PATCH 08/16] refactoring Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index afb9c229..1af8ff7b 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -69,21 +69,11 @@ def generateAlgorithmRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> algorithms = [algorithms] for alg in algorithms: - name = alg.name() - algoId = name.lower() - for c in [' ']: - algoId = algoId.replace(c, '_') - text = [ - f'.. _{algoId}\n', - '*' * len(name), - name, - '*' * len(name), - ] - text.append(v3(alg)) + text = v3(alg) - afilename = algoId - for c in r'/()': + afilename = alg.name() + for c in r'/() ': afilename = afilename.replace(c, '_') filename = rootRst / groupFolderName(alg.group()) / '{}.rst'.format(afilename) @@ -176,7 +166,7 @@ def generateRST(rootRst=None, def wrapAutoGenerated(rst_text: str, section: str) -> str: """Wraps autogenerated content with start and end markers""" - return f"..\n ## AUTOGENERATED START {section}\n\n{rst_text}\n\n..\n ## AUTOGENERATED END {section}\n" + return f"..\n ## AUTOGENERATED {section} START\n\n{rst_text}\n\n..\n ## AUTOGENERATED {section} END\n\n" def v3(alg: QgsProcessingAlgorithm): @@ -187,7 +177,8 @@ def v3(alg: QgsProcessingAlgorithm): # Title Section title = alg.displayName() - title_text = f".. _{title}:\n\n{title}\n{'*' * len(title)}\n" + dotline = '*' * len(title) + title_text = f".. _{title}:\n\n{dotline}\n{title}\n{dotline}\n" title_text = utilsConvertHtmlLinksToRstLinks(title_text) # Convert any HTML links text = wrapAutoGenerated(title_text, "TITLE") From f52d73322b02e4566235105c7d41a9b9cf2f2016 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 18:46:07 +0100 Subject: [PATCH 09/16] update autogenerated rst files, takes care of old-style include files addresses https://github.com/EnMAP-Box/enmap-box-documentation/issues/80 Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 261 ++++++++++++++++++++++--------- 1 file changed, 183 insertions(+), 78 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 1af8ff7b..d307aa27 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -2,32 +2,26 @@ import os import re import subprocess +import warnings from os import makedirs -from os.path import join from pathlib import Path -from typing import List, Union +from typing import Dict, List, Union -from qgis.core import QgsProcessingAlgorithm, QgsProcessingDestinationParameter, QgsProcessingParameterDefinition +from qgis.core import QgsApplication, QgsProcessingAlgorithm, QgsProcessingDestinationParameter, \ + QgsProcessingParameterDefinition import enmapbox from enmapbox.algorithmprovider import EnMAPBoxProcessingProvider -from enmapbox.qgispluginsupport.qps.utils import file_search from enmapbox.testing import start_app from enmapboxprocessing.enmapalgorithm import EnMAPProcessingAlgorithm, Group from enmapboxprocessing.glossary import injectGlossaryLinks rootCodeRepo = Path(__file__).parent.parent -if 'PATH_DOCU_REPO' in os.environ: - rootDocRepo = Path(os.environ['PATH_DOCU_REPO']) -else: - rootDocRepo = rootCodeRepo.parent / 'enmap-box-documentation' - - os.makedirs(rootDocRepo, exist_ok=True) -assert rootDocRepo.is_dir() - dryRun = False # a fast way to check if all parameters are documented +PREFIX_AUTOGEN = '..\n ## AUTOGENERATED ' + # environment to run qgis_process from python shell command QGIS_PROCESS_ENV = os.environ.copy() for k in ['QGIS_CUSTOM_CONFIG_PATH', 'QT3D_RENDERER']: @@ -35,49 +29,117 @@ QGIS_PROCESS_ENV.pop(k) -def read_existing_rsts(rootRst: Union[str, Path]): - rootRst = Path(rootRst) - - infos = dict() - if not rootRst.is_dir(): - return infos - - for p in file_search(rootRst, '*.rst', recursive=True): - with open(p, 'r') as f: - rst_test = f.read() - - infos[rootRst.as_posix()] = rst_test - return infos - - -def write_and_update(file, text): +def parse_text_sections(text: str) -> Dict[Union[str, int], str]: + """ + Splits the intput text into manually defined and autogenerated sections + key = int -> manually defined section + key = str -> name of autogenerated section + """ + rx = re.compile(r'(\.\.\n ## AUTOGENERATED (.*) START)') + matches = rx.findall(text) + if len(matches) == 0: + return {0: text} + + results = dict() + current_start = 0 + current_man_section = 0 + for match in matches: + section = match[1] + key_start = match[0] + key_end = key_start[0:-5] + 'END\n' + + i0 = text.find(key_start) + ie = text.find(key_end) + len(key_end) + + if i0 > current_start: + results[current_man_section] = text[current_start: i0] + + results[section] = text[i0:ie] + + current_start = ie + current_man_section += 1 + + if current_start < len(text): + results[current_man_section + 1] = text[current_start:] + return results + + +def update_rst(text_old: str, text_new: str) -> str: + """ + Updates the autogenerated sections in text_old by autogenerated sections in text_new + """ + old_sections = parse_text_sections(text_old) + new_sections = parse_text_sections(text_new) + + merges_section = old_sections.copy() + for k, text in new_sections.items(): + if isinstance(k, str): + merges_section[k] = new_sections[k] + + final = '' + for key, text in merges_section.items(): + if isinstance(key, str): # autogenerated section + if len(final) > 0 and not final[-1] in ['\n']: + final += '\n' + final += text + return final + + +def create_or_update_rst(file, text: str): + """ + Create or update a rst file that contains the text. + In case the rst file already exists, it will be updated by + autogenerated content in text. + """ file = Path(file) - # todo: if file exists, keep none-autogenerated parts - if isinstance(text, list): text = '\n'.join(text) + if file.is_file(): + # update text with existing, none-autogenerated filetext + with open(file, 'r') as f: + old_text = f.read() + + if PREFIX_AUTOGEN in old_text: + text = update_rst(old_text, text) + else: + warnings.warn(f'Overwrite none-autogenerated file:\n {file}') + with open(file, 'w', encoding='utf-8') as f: f.write(text) print(f'Created {file}') -def generateAlgorithmRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List[str]: +def generateAlgorithmRSTs(rootRst: Union[Path, str], + algorithms: List[QgsProcessingAlgorithm]) -> List[str]: + """ + Create an rst file for each provided QgsProcessingAlgorithm. + """ rootRst = Path(rootRst) if isinstance(algorithms, QgsProcessingAlgorithm): algorithms = [algorithms] for alg in algorithms: - text = v3(alg) - - afilename = alg.name() + afilename = alg.displayName().lower() for c in r'/() ': afilename = afilename.replace(c, '_') - filename = rootRst / groupFolderName(alg.group()) / '{}.rst'.format(afilename) - write_and_update(filename, text) + + section_adds = dict() + + # check for old-style processing_algorithms_includes files and include the rst code + filename_includes = rootRst.parent / f'{rootRst.name}_includes' / groupFolderName( + alg.group()) / '{}.rst'.format(afilename) + if filename_includes.is_file(): + with open(filename_includes) as f: + text = f.read() + section_adds['DESCRIPTION'] = text + + text = v3(alg, section_adds=section_adds) + + create_or_update_rst(filename, text) def groupFolderName(group: str) -> str: @@ -100,104 +162,142 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List # create group index.rst # create group index.rst - text = [ + title = [ f'.. _{group}:\n', f'{group}', f'{'=' * len(group)}', + ] + text = wrapAutoGenerated(title, 'TITLE') + + toc = [ '.. toctree::', ' :maxdepth: 0', ' :glob:\n', ' *\n' ] + text += wrapAutoGenerated(toc, 'TOC') filename = groupFolder / 'index.rst' - write_and_update(filename, '\n'.join(text)) + create_or_update_rst(filename, text) index_files.append(f'{groupFolder.name}/{filename.name}') - # write processing_algorithsm.rst - textProcessingAlgorithmsRst = [ - 'Processing Algorithms', - '*********************', - '', - '.. toctree::', - ' :maxdepth: 1\n' - ] - textProcessingAlgorithmsRst.extend([f' {f}' for f in index_files]) + # write processing_algorithm.rst + title = ['Processing Algorithms', + '*********************\n'] + toc = ['.. toctree::', + ' :maxdepth: 1\n'] + for f in index_files: + toc.append(f' {f}') + text = wrapAutoGenerated(title, 'TITLE') + wrapAutoGenerated(toc, 'TOC') filename = rootRst / 'processing_algorithms.rst' - write_and_update(filename, '\n'.join(textProcessingAlgorithmsRst)) + create_or_update_rst(filename, text) return index_files -def generateRST(rootRst=None, +def doc_repo_root() -> Path: + if 'PATH_DOCU_REPO' in os.environ: + rootDocRepo = Path(os.environ['PATH_DOCU_REPO']) + else: + rootDocRepo = rootCodeRepo.parent / 'enmap-box-documentation' + return rootDocRepo + + +def generateRST(rootRst: Union[Path, str], algorithmIds: List[str] = None ): - # create folder + rootRst = Path(rootRst) + assert rootRst.is_dir() print(rootCodeRepo) - - if rootRst is None: - print(rootDocRepo) - rootRst = join(rootDocRepo, 'source', 'usr_section', 'usr_manual', 'processing_algorithms') - print(rootRst) assert isinstance(EnMAPBoxProcessingProvider.instance(), EnMAPBoxProcessingProvider) makedirs(rootRst, exist_ok=True) # filter algorithms - algs: List[QgsProcessingAlgorithm] = [] - for alg in EnMAPBoxProcessingProvider.instance().algorithms(): - if Group.Experimental.name in alg.group(): - raise RuntimeError('Remove experimental algorithms from final release!') - algs.append(alg) + if algorithmIds is None: + algs = EnMAPBoxProcessingProvider.instance().algorithms() + for alg in EnMAPBoxProcessingProvider.instance().algorithms(): + if Group.Experimental.name in alg.group(): + raise RuntimeError('Remove experimental algorithms from final release!') - # optional: filter algorithms (e.g. for testing) - if algorithmIds: - algs = [a for a in algs if a.id() in algorithmIds or a.name() in algorithmIds] + else: + algs = QgsApplication.instance().processingRegistry().algorithms() - # create group folders, /index.rst and processing_algorithms.rst + algs = [a for a in algs if + a.id() in algorithmIds or a.name() in algorithmIds] + # create group folders, /index.rst and processing_algorithms.rst + print(f'Create *.rst files for {len(algs)} algorithms') generateGroupRSTs(rootRst, algs) generateAlgorithmRSTs(rootRst, algs) s = "" -def wrapAutoGenerated(rst_text: str, section: str) -> str: +def wrapWithNewLines(text: str) -> str: + """ + Ensures that the input text begins and ends with a newline character + In case it does not end with a newline, 2 empty lines will be appended + :param text: str + :return: str + """ + + if isinstance(text, str) and len(text) > 0: + if text[0] != '\n': + text = '\n' + text + if text[-1] != '\n': + text += '\n\n' + return text + else: + return '' + + +def wrapAutoGenerated(rst_text: Union[str, list], section: str) -> str: + if isinstance(rst_text, list): + rst_text = '\n'.join(rst_text) + """Wraps autogenerated content with start and end markers""" - return f"..\n ## AUTOGENERATED {section} START\n\n{rst_text}\n\n..\n ## AUTOGENERATED {section} END\n\n" + return f"{PREFIX_AUTOGEN}{section} START\n{wrapWithNewLines(rst_text)}\n{PREFIX_AUTOGEN}{section} END\n\n" + +def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): + """ -def v3(alg: QgsProcessingAlgorithm): + :param alg: QgsProcessingAlgorithm + :param section_adds: dictionary with rst code snippeds to append to an autogenerated section. + """ if isinstance(alg, EnMAPProcessingAlgorithm): helpParameters = {k: v for k, v in alg.helpParameters()} else: helpParameters = dict() + if section_adds is None: + section_adds = dict() + # Title Section title = alg.displayName() dotline = '*' * len(title) title_text = f".. _{title}:\n\n{dotline}\n{title}\n{dotline}\n" title_text = utilsConvertHtmlLinksToRstLinks(title_text) # Convert any HTML links + text = wrapAutoGenerated(title_text, "TITLE") + text += wrapWithNewLines(section_adds.get('TITLE')) # Description Section description_text = injectGlossaryLinks(alg.shortDescription()) - description_text = utilsConvertHtmlLinksToRstLinks(description_text) # Convert HTML links + description_text = utilsConvertHtmlLinksToRstLinks(description_text) text += wrapAutoGenerated(description_text, "DESCRIPTION") + text += wrapWithNewLines(section_adds.get('DESCRIPTION')) # Parameters Section - param_text = '' + param_text = '**Parameters**\n\n' outputsHeadingCreated = False for pd in alg.parameterDefinitions(): assert isinstance(pd, QgsProcessingParameterDefinition) - pdhelp = helpParameters.get(pd.description(), 'undocumented') - if pdhelp == '': - continue - if pdhelp == 'undocumented': - assert 0, pd.description() + pdhelp = helpParameters.get(pd.description(), pd.description()) if not outputsHeadingCreated and isinstance(pd, QgsProcessingDestinationParameter): param_text += '**Outputs**\n\n' @@ -218,6 +318,7 @@ def v3(alg: QgsProcessingAlgorithm): param_text += f' Default: *{pd.defaultValue()}*\n\n' text += wrapAutoGenerated(param_text, "PARAMETERS") + text += wrapWithNewLines(section_adds.get('PARAMETERS')) # Command-line usage helptext = qgis_process_help(alg) @@ -228,6 +329,7 @@ def v3(alg: QgsProcessingAlgorithm): usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) text += wrapAutoGenerated(usage_text, "COMMAND USAGE") + text += wrapWithNewLines(section_adds.get('COMMAND USAGE')) return text @@ -271,10 +373,13 @@ def utilsHtmlWeblinkToRstWeblink(htmlText: str) -> str: if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generates the documentation for EnMAPBox processing algorithms ', formatter_class=argparse.RawTextHelpFormatter) + + doc_root_default = doc_repo_root() / 'source' / 'usr_section' / 'usr_manual' / 'processing_algorithms' + parser.add_argument('-r', '--rst_root', required=False, - default=rootDocRepo.as_posix(), - help=f'Root folder to write the RST files. Defaults to {rootDocRepo}') + default=str(doc_root_default), + help=f'Root folder to write the RST files. Defaults to {doc_root_default}') parser.add_argument('-a', '--algs', required=False, @@ -288,8 +393,8 @@ def utilsHtmlWeblinkToRstWeblink(htmlText: str) -> str: if not rootRst.is_absolute(): rootRst = rootCodeRepo / rootRst - algs = args.algs + assert rootRst.is_dir(), f'Directory does not exists: {rootRst}. Use --rst_root to specify location.' start_app() enmapbox.initAll() - generateRST(rootRst, algs) + generateRST(rootRst, args.algs) From 1b7d2ca7c0a96f723964df695b87d42a0102d825 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 22:53:51 +0100 Subject: [PATCH 10/16] refactored shell outputs and rst newlines escape * in rst generated from html inputs added test_create_processing_rst.py addresses https://github.com/EnMAP-Box/enmap-box-documentation/issues/80 Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 60 ++++++--- .../scripts/test_create_processing_rst.py | 126 ++++++++++++++++++ 2 files changed, 167 insertions(+), 19 deletions(-) create mode 100644 tests/enmap-box/scripts/test_create_processing_rst.py diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index d307aa27..63f29e01 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -108,7 +108,7 @@ def create_or_update_rst(file, text: str): with open(file, 'w', encoding='utf-8') as f: f.write(text) - print(f'Created {file}') + return file def generateAlgorithmRSTs(rootRst: Union[Path, str], @@ -120,7 +120,9 @@ def generateAlgorithmRSTs(rootRst: Union[Path, str], if isinstance(algorithms, QgsProcessingAlgorithm): algorithms = [algorithms] - for alg in algorithms: + n = len(algorithms) + print(f'Create rst files for {n} algorithms...') + for i, alg in enumerate(algorithms): afilename = alg.displayName().lower() for c in r'/() ': @@ -139,7 +141,8 @@ def generateAlgorithmRSTs(rootRst: Union[Path, str], text = v3(alg, section_adds=section_adds) - create_or_update_rst(filename, text) + path = create_or_update_rst(filename, text) + print(f'Created {i + 1}/{n}: {path}') def groupFolderName(group: str) -> str: @@ -155,7 +158,9 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List groups = set([a.group() for a in algorithms]) index_files: List[str] = [] - for group in groups: + n = len(groups) + print(f'Create index.rst files for {n} groups...') + for i, group in enumerate(groups): # create group folder groupFolder = rootRst / groupFolderName(group) makedirs(groupFolder, exist_ok=True) @@ -178,8 +183,9 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List text += wrapAutoGenerated(toc, 'TOC') filename = groupFolder / 'index.rst' - create_or_update_rst(filename, text) + path = create_or_update_rst(filename, text) index_files.append(f'{groupFolder.name}/{filename.name}') + print(f'Created {i + 1}/{n}: {path}') # write processing_algorithm.rst title = ['Processing Algorithms', @@ -191,8 +197,8 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List text = wrapAutoGenerated(title, 'TITLE') + wrapAutoGenerated(toc, 'TOC') filename = rootRst / 'processing_algorithms.rst' - create_or_update_rst(filename, text) - + path = create_or_update_rst(filename, text) + print(f'Created {path}') return index_files @@ -259,7 +265,21 @@ def wrapAutoGenerated(rst_text: Union[str, list], section: str) -> str: rst_text = '\n'.join(rst_text) """Wraps autogenerated content with start and end markers""" - return f"{PREFIX_AUTOGEN}{section} START\n{wrapWithNewLines(rst_text)}\n{PREFIX_AUTOGEN}{section} END\n\n" + return f"\n{PREFIX_AUTOGEN}{section} START\n{wrapWithNewLines(rst_text)}\n{PREFIX_AUTOGEN}{section} END\n\n" + + +rx_needs_escape = re.compile(r'(?()\[\]])') + + +def escape_rst(text: str) -> str: + if text is None: + return '' + parts = re.findall('(`[^`]*`_|[^`]*)', text) + for i, part in enumerate(parts): + if not (part.startswith('`') and part.endswith('`_')): + parts[i] = rx_needs_escape.sub(r'\\\g<1>', part) + + return ''.join(parts) def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): @@ -288,6 +308,7 @@ def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): # Description Section description_text = injectGlossaryLinks(alg.shortDescription()) description_text = utilsConvertHtmlLinksToRstLinks(description_text) + description_text = escape_rst(description_text) text += wrapAutoGenerated(description_text, "DESCRIPTION") text += wrapWithNewLines(section_adds.get('DESCRIPTION')) @@ -300,36 +321,37 @@ def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): pdhelp = helpParameters.get(pd.description(), pd.description()) if not outputsHeadingCreated and isinstance(pd, QgsProcessingDestinationParameter): - param_text += '**Outputs**\n\n' + param_text += '\n\n**Outputs**\n\n' outputsHeadingCreated = True param_text += f'\n:guilabel:`{pd.description()}` [{pd.type()}]\n' pdhelp = injectGlossaryLinks(pdhelp) pdhelp = utilsConvertHtmlLinksToRstLinks(pdhelp) # Convert HTML links in help text for line in pdhelp.split('\n'): - param_text += f' {line}\n' + param_text += f' {escape_rst(line)}\n' if pd.defaultValue() is not None: if isinstance(pd.defaultValue(), str) and '\n' in pd.defaultValue(): param_text += ' Default::\n\n' for line in pd.defaultValue().split('\n'): - param_text += f' {line}\n' + param_text += f' {escape_rst(line)}\n' else: - param_text += f' Default: *{pd.defaultValue()}*\n\n' + param_text += f' Default: *{escape_rst(str(pd.defaultValue()))}*\n\n' text += wrapAutoGenerated(param_text, "PARAMETERS") text += wrapWithNewLines(section_adds.get('PARAMETERS')) # Command-line usage - helptext = qgis_process_help(alg) - helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] - helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text + if True: + helptext = qgis_process_help(alg) + helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] + helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text - usage_text = f"**Command-line usage**\n\n``>qgis_process help {alg.id()}``::\n\n" - usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) + usage_text = f"**Command-line usage**\n\n``>qgis_process help {alg.id()}``::\n\n" + usage_text += '\n'.join([f' {line}' for line in helptext.splitlines()]) - text += wrapAutoGenerated(usage_text, "COMMAND USAGE") - text += wrapWithNewLines(section_adds.get('COMMAND USAGE')) + text += wrapAutoGenerated(usage_text, "COMMAND USAGE") + text += wrapWithNewLines(section_adds.get('COMMAND USAGE')) return text diff --git a/tests/enmap-box/scripts/test_create_processing_rst.py b/tests/enmap-box/scripts/test_create_processing_rst.py new file mode 100644 index 00000000..3f4a29e5 --- /dev/null +++ b/tests/enmap-box/scripts/test_create_processing_rst.py @@ -0,0 +1,126 @@ +import os +import shutil +import subprocess +import unittest + +from enmapbox.testing import start_app, TestCase +from enmapbox import registerEnMAPBoxProcessingProvider +from scripts.create_processing_rst import __file__ as path_processing_script, escape_rst, generateRST, QGIS_PROCESS_ENV, \ + update_rst + +start_app() +registerEnMAPBoxProcessingProvider() + +RST_ORIGINAL = """ +before +.. + ## AUTOGENERATED FOO START + +blablala +.. + ## AUTOGENERATED FOO END +ending 1""" + +RST_UPDATE = """ + +.. + ## AUTOGENERATED FOO START +changed blablabla +.. + ## AUTOGENERATED FOO END + +DO_NOT_COPY_CODE_FROM_OUTSIDE_AUTOGENERATED_SECTIONS + +.. + ## AUTOGENERATED NEW START +new blablabla +.. + ## AUTOGENERATED NEW END +""" + +RST_RESULT = """ +before +.. + ## AUTOGENERATED FOO START +changed blablabla +.. + ## AUTOGENERATED FOO END +ending 1 +.. + ## AUTOGENERATED NEW START +new blablabla +.. + ## AUTOGENERATED NEW END +""" + + +class CreateProcessingRSTTestCases(TestCase): + + def test_update_rst(self): + self.assertEqual(update_rst(RST_ORIGINAL, ''), RST_ORIGINAL) + self.assertEqual(update_rst(RST_ORIGINAL, 'not in AUTOGENERATE section'), RST_ORIGINAL) + self.assertEqual(update_rst(RST_ORIGINAL, RST_UPDATE), RST_RESULT) + + def test_generateRST(self): + dir_tmp = self.createTestOutputDirectory() + dir_rst_root = dir_tmp / 'rst_root1' + generateRST(dir_rst_root, algorithmIds=['gdal:translate', 'enmapbox:Build3DCube']) + + expected_paths = [ + dir_rst_root / 'auxilliary' / 'build_3d_cube.rst', + dir_rst_root / 'raster_conversion' / 'translate__convert_format_.rst' + ] + for p in expected_paths: + self.assertTrue(p.is_file(), msg=f'File not created exist: {p}') + + @unittest.skipIf(TestCase.runsInCI(), 'Manual calls only') + def test_buildall(self): + + result = subprocess.run(['python', str(path_processing_script)], + env=QGIS_PROCESS_ENV, + stderr=subprocess.PIPE + ) + self.assertEqual(result.returncode, 0, msg=result.stderr.decode()) + + def test_escape_rst(self): + + examples = [ + ('* wor*d `line `_ ', '\\* wor\\*d `line `_ '), + ('a*c', 'a\\*c'), + ('abc', 'abc'), + ('*c', '\\*c'), + ('c*', 'c\\*'), + ('[a]', '\\[a\\]'), + (']a[', '\\]a\\['), + ] + for (text, expected) in examples: + result = escape_rst(text) + print(f'{text} -> {result} ({expected})') + self.assertEqual(result, expected) + + def test_script(self): + dir_tmp = self.createTestOutputDirectory() + dir_rst_root = dir_tmp / 'rst_root2' + if dir_rst_root.is_dir(): + shutil.rmtree(dir_rst_root) + os.makedirs(dir_rst_root) + + result = subprocess.run(['python', str(path_processing_script), + '-r', str(dir_rst_root), + '-a', 'Build3DCube'], + env=QGIS_PROCESS_ENV, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + self.assertEqual(result.returncode, 0, msg=result.stderr.decode()) + + expected_files = [ + dir_rst_root / 'processing_algorithms.rst', + dir_rst_root / 'auxilliary' / 'build_3d_cube.rst', + dir_rst_root / 'auxilliary' / 'index.rst', + ] + for p in expected_files: + self.assertTrue(p.is_file(), msg=f'Missing file: {p}') + + +if __name__ == '__main__': + unittest.main() From 5601cd3e5e76470de73e1389a87d148011bbca48 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Sun, 24 Nov 2024 22:55:41 +0100 Subject: [PATCH 11/16] flake8 (auto-format) change EnMAP-Box processing provider group 'Spectral library' to 'Spectral Library' (like 'Accuracy Assessment') Signed-off-by: jakimowb --- enmapboxprocessing/enmapalgorithm.py | 41 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/enmapboxprocessing/enmapalgorithm.py b/enmapboxprocessing/enmapalgorithm.py index 82bef099..68dee5d8 100644 --- a/enmapboxprocessing/enmapalgorithm.py +++ b/enmapboxprocessing/enmapalgorithm.py @@ -2,35 +2,34 @@ from enum import Enum from math import nan from os import makedirs -from os.path import isabs, join, dirname, exists, splitext, abspath +from os.path import abspath, dirname, exists, isabs, join, splitext from time import time -from typing import Any, Dict, Iterable, Optional, List, Tuple, TextIO +from typing import Any, Dict, Iterable, List, Optional, TextIO, Tuple import numpy as np from osgeo import gdal - import processing +from qgis.PyQt.QtGui import QIcon +from qgis.core import (Qgis, QgsCategorizedSymbolRenderer, QgsCoordinateReferenceSystem, QgsMapLayer, + QgsPalettedRasterRenderer, QgsProcessing, QgsProcessingAlgorithm, QgsProcessingContext, + QgsProcessingException, QgsProcessingFeedback, QgsProcessingOutputLayerDefinition, + QgsProcessingParameterBand, QgsProcessingParameterBoolean, QgsProcessingParameterCrs, + QgsProcessingParameterDefinition, QgsProcessingParameterEnum, QgsProcessingParameterExtent, + QgsProcessingParameterField, QgsProcessingParameterFile, QgsProcessingParameterFileDestination, + QgsProcessingParameterFolderDestination, QgsProcessingParameterMapLayer, + QgsProcessingParameterMatrix, QgsProcessingParameterMultipleLayers, QgsProcessingParameterNumber, + QgsProcessingParameterRange, QgsProcessingParameterRasterLayer, QgsProcessingParameterString, + QgsProcessingParameterVectorDestination, QgsProcessingParameterVectorLayer, QgsProcessingUtils, + QgsProject, QgsProperty, QgsRasterLayer, QgsRectangle, QgsVectorLayer) + from enmapbox.typeguard import typechecked from enmapboxprocessing.driver import Driver from enmapboxprocessing.glossary import injectGlossaryLinks from enmapboxprocessing.parameter.processingparameterrasterdestination import ProcessingParameterRasterDestination from enmapboxprocessing.processingfeedback import ProcessingFeedback -from enmapboxprocessing.typing import CreationOptions, GdalResamplingAlgorithm, ClassifierDump, \ - TransformerDump, RegressorDump, ClustererDump +from enmapboxprocessing.typing import ClassifierDump, ClustererDump, CreationOptions, GdalResamplingAlgorithm, \ + RegressorDump, TransformerDump from enmapboxprocessing.utils import Utils -from qgis.PyQt.QtGui import QIcon -from qgis.core import (QgsProcessingAlgorithm, QgsProcessingParameterRasterLayer, QgsProcessingParameterVectorLayer, - QgsProcessingContext, QgsProcessingFeedback, - QgsRasterLayer, QgsVectorLayer, QgsProcessingParameterNumber, QgsProcessingParameterDefinition, - QgsProcessingParameterField, QgsProcessingParameterBoolean, QgsProcessingParameterEnum, Qgis, - QgsProcessingParameterString, QgsProcessingParameterBand, QgsCategorizedSymbolRenderer, - QgsPalettedRasterRenderer, QgsProcessingParameterMapLayer, QgsMapLayer, - QgsProcessingParameterExtent, QgsCoordinateReferenceSystem, QgsRectangle, - QgsProcessingParameterFileDestination, QgsProcessingParameterFile, QgsProcessingParameterRange, - QgsProcessingParameterCrs, QgsProcessingParameterVectorDestination, QgsProcessing, - QgsProcessingUtils, QgsProcessingParameterMultipleLayers, QgsProcessingException, - QgsProcessingParameterFolderDestination, QgsProject, QgsProcessingOutputLayerDefinition, - QgsProperty, QgsProcessingParameterMatrix) class AlgorithmCanceledException(Exception): @@ -41,8 +40,8 @@ class AlgorithmCanceledException(Exception): class EnMAPProcessingAlgorithm(QgsProcessingAlgorithm): O_RESAMPLE_ALG = 'NearestNeighbour Bilinear Cubic CubicSpline Lanczos Average Mode Min Q1 Med Q3 Max'.split() NearestNeighbourResampleAlg, BilinearResampleAlg, CubicResampleAlg, CubicSplineResampleAlg, LanczosResampleAlg, \ - AverageResampleAlg, ModeResampleAlg, MinResampleAlg, Q1ResampleAlg, MedResampleAlg, Q3ResampleAlg, \ - MaxResampleAlg = range(12) + AverageResampleAlg, ModeResampleAlg, MinResampleAlg, Q1ResampleAlg, MedResampleAlg, Q3ResampleAlg, \ + MaxResampleAlg = range(12) O_DATA_TYPE = 'Byte Int16 UInt16 UInt32 Int32 Float32 Float64'.split() Byte, Int16, UInt16, Int32, UInt32, Float32, Float64 = range(len(O_DATA_TYPE)) PickleFileFilter = 'Pickle files (*.pkl)' @@ -1044,7 +1043,7 @@ class Group(Enum): RasterProjections = 'Raster projections' Regression = 'Regression' Sampling = 'Sampling' - SpectralLibrary = 'Spectral library' + SpectralLibrary = 'Spectral Library' SpectralResampling = 'Spectral resampling' Testdata = 'Testdata' Transformation = 'Transformation' From 82fab6990bd0c306d46eaf4cf7edc5cb93fe917c Mon Sep 17 00:00:00 2001 From: jakimowb Date: Mon, 25 Nov 2024 00:36:37 +0100 Subject: [PATCH 12/16] parallel collection of qgis processing help strings Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 63 ++++++++++++++----- .../scripts/test_create_processing_rst.py | 20 +++++- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 63f29e01..19f5f1b6 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -1,8 +1,10 @@ import argparse +import datetime import os import re import subprocess import warnings +from concurrent.futures.thread import ThreadPoolExecutor from os import makedirs from pathlib import Path from typing import Dict, List, Union @@ -29,7 +31,7 @@ QGIS_PROCESS_ENV.pop(k) -def parse_text_sections(text: str) -> Dict[Union[str, int], str]: +def parseRSTSections(text: str) -> Dict[Union[str, int], str]: """ Splits the intput text into manually defined and autogenerated sections key = int -> manually defined section @@ -68,8 +70,8 @@ def update_rst(text_old: str, text_new: str) -> str: """ Updates the autogenerated sections in text_old by autogenerated sections in text_new """ - old_sections = parse_text_sections(text_old) - new_sections = parse_text_sections(text_new) + old_sections = parseRSTSections(text_old) + new_sections = parseRSTSections(text_new) merges_section = old_sections.copy() for k, text in new_sections.items(): @@ -112,7 +114,8 @@ def create_or_update_rst(file, text: str): def generateAlgorithmRSTs(rootRst: Union[Path, str], - algorithms: List[QgsProcessingAlgorithm]) -> List[str]: + algorithms: List[QgsProcessingAlgorithm], + load_process_help: bool = True) -> List[str]: """ Create an rst file for each provided QgsProcessingAlgorithm. """ @@ -121,6 +124,14 @@ def generateAlgorithmRSTs(rootRst: Union[Path, str], algorithms = [algorithms] n = len(algorithms) + if load_process_help: + print(f'Collect qgis_process help string for {n} algorithms (takes a while)... ', end='') + t0 = datetime.datetime.now() + qgis_process_help = collectQgsProcessAlgorithmHelp(algorithms, run_async=True) + print(f'Done: {datetime.datetime.now() - t0}') + else: + qgis_process_help = dict() + print(f'Create rst files for {n} algorithms...') for i, alg in enumerate(algorithms): @@ -139,7 +150,7 @@ def generateAlgorithmRSTs(rootRst: Union[Path, str], text = f.read() section_adds['DESCRIPTION'] = text - text = v3(alg, section_adds=section_adds) + text = v3(alg, section_adds=section_adds, qgis_process_help=qgis_process_help) path = create_or_update_rst(filename, text) print(f'Created {i + 1}/{n}: {path}') @@ -188,8 +199,10 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List print(f'Created {i + 1}/{n}: {path}') # write processing_algorithm.rst - title = ['Processing Algorithms', - '*********************\n'] + title = [ + '.. _Processing Algorithms:\n', + 'Processing Algorithms', + '*********************\n'] toc = ['.. toctree::', ' :maxdepth: 1\n'] for f in index_files: @@ -239,8 +252,6 @@ def generateRST(rootRst: Union[Path, str], generateGroupRSTs(rootRst, algs) generateAlgorithmRSTs(rootRst, algs) - s = "" - def wrapWithNewLines(text: str) -> str: """ @@ -282,7 +293,27 @@ def escape_rst(text: str) -> str: return ''.join(parts) -def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): +def collectQgsProcessAlgorithmHelp(algorithms: List[QgsProcessingAlgorithm], run_async: bool = False) -> Dict[str, str]: + results = dict() + n = len(algorithms) + + if not run_async: + for alg in algorithms: + results[alg.id()] = qgisProcessHelp(alg) + else: + def process_algorithm(alg): + return alg.id(), qgisProcessHelp(alg) + + with ThreadPoolExecutor(max_workers=min(10, os.cpu_count())) as executor: + futures = executor.map(process_algorithm, algorithms) + + for alg_id, help_text in futures: + results[alg_id] = help_text + + return results + + +def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None, qgis_process_help: dict = None): """ :param alg: QgsProcessingAlgorithm @@ -295,6 +326,8 @@ def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): if section_adds is None: section_adds = dict() + if qgis_process_help is None: + qgis_process_help = dict() # Title Section title = alg.displayName() @@ -342,8 +375,8 @@ def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): text += wrapWithNewLines(section_adds.get('PARAMETERS')) # Command-line usage - if True: - helptext = qgis_process_help(alg) + if alg.id() in qgis_process_help: + helptext = qgis_process_help[alg.id()] helptext = helptext[helptext.find('----------------\nArguments\n----------------'):] helptext = utilsConvertHtmlLinksToRstLinks(helptext) # Convert HTML links in usage text @@ -356,12 +389,14 @@ def v3(alg: QgsProcessingAlgorithm, section_adds: dict = None): return text -def qgis_process_help(algorithm: QgsProcessingAlgorithm) -> str: +def qgisProcessHelp(algorithm: QgsProcessingAlgorithm) -> str: result = subprocess.run(['qgis_process', 'help', algorithm.id()], env=QGIS_PROCESS_ENV, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - assert result.returncode == 0 + if result.returncode != 0: + s = "" + assert result.returncode == 0, result.stderr.decode() helptext = result.stdout.decode('cp1252') return helptext diff --git a/tests/enmap-box/scripts/test_create_processing_rst.py b/tests/enmap-box/scripts/test_create_processing_rst.py index 3f4a29e5..5041d2d1 100644 --- a/tests/enmap-box/scripts/test_create_processing_rst.py +++ b/tests/enmap-box/scripts/test_create_processing_rst.py @@ -1,12 +1,15 @@ +import datetime import os import shutil import subprocess import unittest +from enmapbox.algorithmprovider import EnMAPBoxProcessingProvider from enmapbox.testing import start_app, TestCase from enmapbox import registerEnMAPBoxProcessingProvider -from scripts.create_processing_rst import __file__ as path_processing_script, escape_rst, generateRST, QGIS_PROCESS_ENV, \ - update_rst +from scripts.create_processing_rst import __file__ as path_processing_script, collectQgsProcessAlgorithmHelp, \ + escape_rst, \ + generateRST, QGIS_PROCESS_ENV, update_rst start_app() registerEnMAPBoxProcessingProvider() @@ -98,6 +101,19 @@ def test_escape_rst(self): print(f'{text} -> {result} ({expected})') self.assertEqual(result, expected) + @unittest.skipIf(TestCase.runsInCI(), 'Manual testing only') + def test_collect_algorithm_help(self): + + algs = EnMAPBoxProcessingProvider.instance().algorithms() + algs = algs[0:100] + t0 = datetime.datetime.now() + results1 = collectQgsProcessAlgorithmHelp(algs, run_async=True) + t1 = datetime.datetime.now() + results2 = collectQgsProcessAlgorithmHelp(algs, run_async=False) + t2 = datetime.datetime.now() + self.assertEqual(results1, results2) + print(f'Async: {t1 - t0}\nNormal: {t2 - t1}') + def test_script(self): dir_tmp = self.createTestOutputDirectory() dir_rst_root = dir_tmp / 'rst_root2' From c73fbc3d26a7c23f27c5f487ad21c99f4e2b4f2a Mon Sep 17 00:00:00 2001 From: jakimowb Date: Mon, 25 Nov 2024 01:06:31 +0100 Subject: [PATCH 13/16] test_create_processing_rst.py: faster testing Signed-off-by: jakimowb --- scripts/create_processing_rst.py | 6 +++--- tests/enmap-box/scripts/test_create_processing_rst.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 19f5f1b6..429fc5d8 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -224,8 +224,8 @@ def doc_repo_root() -> Path: def generateRST(rootRst: Union[Path, str], - algorithmIds: List[str] = None - ): + algorithmIds: List[str] = None, + load_process_help: bool = True): rootRst = Path(rootRst) assert rootRst.is_dir() print(rootCodeRepo) @@ -250,7 +250,7 @@ def generateRST(rootRst: Union[Path, str], # create group folders, /index.rst and processing_algorithms.rst print(f'Create *.rst files for {len(algs)} algorithms') generateGroupRSTs(rootRst, algs) - generateAlgorithmRSTs(rootRst, algs) + generateAlgorithmRSTs(rootRst, algs, load_process_help=load_process_help) def wrapWithNewLines(text: str) -> str: diff --git a/tests/enmap-box/scripts/test_create_processing_rst.py b/tests/enmap-box/scripts/test_create_processing_rst.py index 5041d2d1..4817e178 100644 --- a/tests/enmap-box/scripts/test_create_processing_rst.py +++ b/tests/enmap-box/scripts/test_create_processing_rst.py @@ -67,7 +67,9 @@ def test_update_rst(self): def test_generateRST(self): dir_tmp = self.createTestOutputDirectory() dir_rst_root = dir_tmp / 'rst_root1' - generateRST(dir_rst_root, algorithmIds=['gdal:translate', 'enmapbox:Build3DCube']) + generateRST(dir_rst_root, + algorithmIds=['gdal:translate', 'enmapbox:Build3DCube'], + load_process_help=False) expected_paths = [ dir_rst_root / 'auxilliary' / 'build_3d_cube.rst', From db1ceee765004d8e6a613f8e564cd798eb0231fb Mon Sep 17 00:00:00 2001 From: jakimowb Date: Mon, 25 Nov 2024 01:07:18 +0100 Subject: [PATCH 14/16] fixed import of QVariant Signed-off-by: jakimowb --- enmapboxprocessing/enmapalgorithm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/enmapboxprocessing/enmapalgorithm.py b/enmapboxprocessing/enmapalgorithm.py index fbe800a0..f3515816 100644 --- a/enmapboxprocessing/enmapalgorithm.py +++ b/enmapboxprocessing/enmapalgorithm.py @@ -9,6 +9,7 @@ import numpy as np from osgeo import gdal import processing +from qgis.PyQt.QtCore import QVariant from qgis.PyQt.QtGui import QIcon from qgis.core import (Qgis, QgsCategorizedSymbolRenderer, QgsCoordinateReferenceSystem, QgsMapLayer, QgsPalettedRasterRenderer, QgsProcessing, QgsProcessingAlgorithm, QgsProcessingContext, From 719b47be181f8ab118116c40fed333477aa12a9a Mon Sep 17 00:00:00 2001 From: jakimowb Date: Mon, 25 Nov 2024 01:15:07 +0100 Subject: [PATCH 15/16] flake8 Signed-off-by: jakimowb --- .flake8 | 2 +- scripts/EnFireMAP/sample_data.py | 10 +++++----- scripts/create_plugin.py | 13 ++++--------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.flake8 b/.flake8 index bdc269ef..e0f2fb22 100644 --- a/.flake8 +++ b/.flake8 @@ -59,7 +59,7 @@ per-file-ignores = # QGS104 Use 'import qgis.PyQt.pyrcc_main' instead of 'import PyQt5.pyrcc_main' # just not available in OSGeo4W installations # script should run immediately which site.addsitedir(REPO) before other imports - scripts/*.py : E402 + # scripts/*.py : E402 enmapbox/eo4qapps/geetimeseriesexplorerapp/*.py : F821 enmapbox/__main__.py: E402 enmapbox/gui/__init__.py: F401 diff --git a/scripts/EnFireMAP/sample_data.py b/scripts/EnFireMAP/sample_data.py index 9676b1ce..ed59e965 100644 --- a/scripts/EnFireMAP/sample_data.py +++ b/scripts/EnFireMAP/sample_data.py @@ -1,9 +1,12 @@ from os import listdir, makedirs -from os.path import join, exists, dirname +from os.path import dirname, exists, join +from qgis.core import QgsVectorFileWriter, QgsVectorLayer + +from enmapbox import initAll +from enmapbox.testing import start_app from enmapboxprocessing.algorithm.samplerastervaluesalgorithm import SampleRasterValuesAlgorithm from enmapboxprocessing.enmapalgorithm import EnMAPProcessingAlgorithm -from qgis.core import QgsVectorFileWriter, QgsVectorLayer rootCube = r'D:\data\EnFireMap\cube' tilingScheme = QgsVectorLayer(r'D:\data\EnFireMap\cube\shp\grid2.geojson') @@ -21,9 +24,6 @@ locations = QgsVectorLayer(r'D:\data\EnFireMap\lib_points\lib_points.shp') rootOutputSample = r'D:\data\EnFireMap\sample3' -from enmapbox import initAll -from enmapbox.testing import start_app - qgsApp = start_app() initAll() diff --git a/scripts/create_plugin.py b/scripts/create_plugin.py index 0c7ae409..017552d2 100644 --- a/scripts/create_plugin.py +++ b/scripts/create_plugin.py @@ -26,29 +26,24 @@ import pathlib import re import shutil -import site import sys import textwrap import typing import warnings -import markdown from typing import Union - -import docutils.core from os.path import exists -from qgis.core import QgsUserProfileManager, QgsUserProfile, QgsFileUtils - -site.addsitedir(pathlib.Path(__file__).parents[1]) # noqa - +import markdown +import docutils.core +from qgis.core import QgsFileUtils, QgsUserProfile, QgsUserProfileManager from qgis.testing import start_app -app = start_app() import enmapbox from enmapbox import DIR_REPO from enmapbox.qgispluginsupport.qps.make.deploy import QGISMetadataFileWriter, userProfileManager from enmapbox.qgispluginsupport.qps.utils import zipdir +app = start_app() # consider default Git location on Windows systems to avoid creating a Start-Up Script try: import git From 1a08340b2540b16aff610038d196af310371cc44 Mon Sep 17 00:00:00 2001 From: jakimowb Date: Mon, 25 Nov 2024 01:19:38 +0100 Subject: [PATCH 16/16] fix for flake8 E999 Signed-off-by: jakimowb --- .github/workflows/flake8.yml | 2 +- scripts/create_processing_rst.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index ed970c16..16f7ec31 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.12" - name: Run flake8 uses: julianwachholz/flake8-action@v2 with: diff --git a/scripts/create_processing_rst.py b/scripts/create_processing_rst.py index 429fc5d8..5514d25e 100644 --- a/scripts/create_processing_rst.py +++ b/scripts/create_processing_rst.py @@ -176,12 +176,11 @@ def generateGroupRSTs(rootRst, algorithms: List[QgsProcessingAlgorithm]) -> List groupFolder = rootRst / groupFolderName(group) makedirs(groupFolder, exist_ok=True) - # create group index.rst - # create group index.rst + group_points = '=' * len(group) title = [ f'.. _{group}:\n', f'{group}', - f'{'=' * len(group)}', + f'{group_points}', ] text = wrapAutoGenerated(title, 'TITLE')