HTML - 16968 - Initial updates for fedramp compliance | Updated SDK t…

…o the latest version (#2837)

plugins/html/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "f5482a1cef5e1328c18792ff5c50d80d",
-	"manifest": "ff24d95d8572ec0b30dc435956525e48",
-	"setup": "e17dd73a961bd7a3bbdabe362a23d382",
+	"spec": "1c682e5751de39f77df69ff820118640",
+	"manifest": "f8496c6f37c90b1549c58634683b21a8",
+	"setup": "8f8d60fa7c130285701aae5de46f6632",
 	"schemas": [
 		{
 			"identifier": "docx/schema.py",

plugins/html/Dockerfile

@@ -1,10 +1,10 @@
-FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.0.1
+FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.1.3
 
 LABEL organization=rapid7
 LABEL sdk=python
 LABEL type=plugin
 
-RUN apt-get update && apt-get install -y pandoc texlive lmodern --no-install-recommends
+RUN apt-get update && apt-get install -y texlive lmodern --no-install-recommends
 
 WORKDIR /python/src
 

plugins/html/bin/icon_html

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "HTML"
 Vendor = "rapid7"
-Version = "1.2.6"
+Version = "1.2.7"
 Description = "Hypertext Markup Language (HTML) is the standard markup language for documents designed to be displayed in a web browser. This plugin provides the ability to convert an HTML document into a variety of formats using [pypandoc](https://pypi.python.org/pypi/pypandoc). Supported formats are: DOCX, EPUB, Markdown, PDF, HTML5, Plain Text"
 
 

plugins/html/help.md

@@ -12,7 +12,7 @@ Hypertext Markup Language (HTML) is the standard markup language for documents d
 
 # Supported Product Versions
 
-* 2023-08-30
+* 2024-09-30
 
 # Documentation
 
@@ -271,6 +271,7 @@ Example output:
 
 # Version History
 
+* 1.2.7 - Initial updates for fedramp compliance | Updated SDK to the latest version
 * 1.2.6 - SDK Bump | Addressing Snyk vulnerabilities | Fixing Unit Tests | Dockerfile USER permission updated
 * 1.2.5 - Update requirements for pypandoc
 * 1.2.4 - Actions modified in order to implement PluginExceptions

plugins/html/icon_html/actions/docx/action.py

@@ -1,10 +1,7 @@
 import insightconnect_plugin_runtime
-import base64
-import pypandoc
-import re
-
-from insightconnect_plugin_runtime.exceptions import PluginException
 from .schema import DocxInput, DocxOutput, Output, Input, Component
+from icon_html.util.api import HTMLConverter
+from icon_html.util.strategies import ConvertToDocx
 
 
 class Docx(insightconnect_plugin_runtime.Action):
@@ -14,18 +11,8 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        temp_file = "temp_html_2_docx.docx"
-        tag_parser = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"  # noqa: W605
-        doc = params.get(Input.DOC)
-        tags = re.findall(tag_parser, doc)
-
-        if not tags:
-            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
+        # END INPUT BINDING - DO NOT REMOVE
 
-        try:
-            pypandoc.convert_text(doc, "docx", outputfile=temp_file, format="html")
-        except RuntimeError as error:
-            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
-        with open(temp_file, "rb") as output:
-            # Reading the output and sending it in base64
-            return {Output.DOCX: base64.b64encode(output.read()).decode("utf-8")}
+        return {Output.DOCX: HTMLConverter(ConvertToDocx()).convert(document)}

plugins/html/icon_html/actions/epub/action.py

@@ -1,10 +1,7 @@
 import insightconnect_plugin_runtime
-import base64
-import pypandoc
-import re
-
-from insightconnect_plugin_runtime.exceptions import PluginException
 from .schema import EpubInput, EpubOutput, Input, Output, Component
+from icon_html.util.api import HTMLConverter
+from icon_html.util.strategies import ConvertToEpub
 
 
 class Epub(insightconnect_plugin_runtime.Action):
@@ -14,18 +11,8 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        temp_file = "temp_html_3_epub.epub"
-        tag_parser = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"  # noqa: W605
-        doc = params.get(Input.DOC)
-        tags = re.findall(tag_parser, doc)
-
-        if not tags:
-            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
+        # END INPUT BINDING - DO NOT REMOVE
 
-        try:
-            pypandoc.convert_text(doc, "epub", outputfile=temp_file, format="html")
-        except RuntimeError as error:
-            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
-        with open(temp_file, "rb") as output:
-            # Reading the output and sending it in base64
-            return {Output.EPUB: base64.b64encode(output.read()).decode("utf-8")}
+        return {Output.EPUB: HTMLConverter(ConvertToEpub()).convert(document)}

plugins/html/icon_html/actions/html5/action.py

@@ -1,9 +1,8 @@
 import insightconnect_plugin_runtime
-from insightconnect_plugin_runtime.exceptions import PluginException
-import pypandoc
-import base64
-import re
 from .schema import Html5Input, Html5Output, Input, Output, Component
+from icon_html.util.api import HTMLConverter
+from icon_html.util.helpers import encode_to_base64
+from icon_html.util.strategies import ConvertToHTML, ConvertToHTML5
 
 
 class Html5(insightconnect_plugin_runtime.Action):
@@ -16,18 +15,13 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        tag_parser = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"  # noqa: W605
-        doc = params.get(Input.DOC)
-        tags = re.findall(tag_parser, doc)
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
+        # END INPUT BINDING - DO NOT REMOVE
 
-        if not tags:
-            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")
-
-        try:
-            output = pypandoc.convert_text(doc, "html", format="md")
-            new_output = pypandoc.convert_text(output, "html5", format="md")
-        except RuntimeError as error:
-            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
-
-        file_ = base64.b64encode(new_output.encode("utf-8")).decode()
-        return {Output.HTML5_CONTENTS: output, Output.HTML5_FILE: file_}
+        converted_html = HTMLConverter(ConvertToHTML()).convert(document)
+        converted_html5 = HTMLConverter(ConvertToHTML5()).convert(converted_html)
+        return {
+            Output.HTML5_CONTENTS: converted_html,
+            Output.HTML5_FILE: encode_to_base64(converted_html5),
+        }

plugins/html/icon_html/actions/markdown/action.py

@@ -1,9 +1,8 @@
 import insightconnect_plugin_runtime
-import pypandoc
-import base64
-import re
 from .schema import MarkdownInput, MarkdownOutput, Input, Output, Component
-from insightconnect_plugin_runtime.exceptions import PluginException
+from icon_html.util.api import HTMLConverter
+from icon_html.util.strategies import ConvertToMarkdown
+from icon_html.util.helpers import encode_to_base64
 
 
 class Markdown(insightconnect_plugin_runtime.Action):
@@ -16,16 +15,12 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        tag_parser = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"  # noqa: W605
-        doc = params.get(Input.DOC)
-        tags = re.findall(tag_parser, doc)
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
+        # END INPUT BINDING - DO NOT REMOVE
 
-        if not tags:
-            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")
-
-        try:
-            output = pypandoc.convert_text(doc, "md", format="html")
-        except RuntimeError as error:
-            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
-        file_ = base64.b64encode(output.encode("ascii")).decode()
-        return {Output.MARKDOWN_CONTENTS: output, Output.MARKDOWN_FILE: file_}
+        converted_markdown = HTMLConverter(ConvertToMarkdown()).convert(document)
+        return {
+            Output.MARKDOWN_CONTENTS: converted_markdown,
+            Output.MARKDOWN_FILE: encode_to_base64(converted_markdown),
+        }

plugins/html/icon_html/actions/pdf/action.py

@@ -1,10 +1,7 @@
 import insightconnect_plugin_runtime
-import base64
-
-from insightconnect_plugin_runtime.exceptions import PluginException
-import pypandoc
-import re
 from .schema import PdfInput, PdfOutput, Input, Output, Component
+from icon_html.util.api import HTMLConverter
+from icon_html.util.strategies import ConvertToPDF
 
 
 class Pdf(insightconnect_plugin_runtime.Action):
@@ -14,18 +11,8 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        temp_file = "temp_html_2_pdf.pdf"
-        tag_parser = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"  # noqa: W605
-        doc = params.get(Input.DOC)
-        tags = re.findall(tag_parser, doc)
-
-        if not tags:
-            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
+        # END INPUT BINDING - DO NOT REMOVE
 
-        try:
-            pypandoc.convert_text(doc, "pdf", outputfile=temp_file, format="html")
-        except RuntimeError as error:
-            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
-        with open(temp_file, "rb") as output:
-            # Reading the output and sending it in base64
-            return {Output.PDF: base64.b64encode(output.read()).decode("utf-8")}
+        return {Output.PDF: HTMLConverter(ConvertToPDF()).convert(document)}

plugins/html/icon_html/actions/text/action.py

@@ -12,14 +12,16 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        in_text = params.get(Input.DOC)
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        document = params.get(Input.DOC, "")
         remove_scripts = params.get(Input.REMOVE_SCRIPTS, False)
-        if in_text:  # BeautifulSoup will bomb on null text
-            soup = BeautifulSoup(in_text, features="html.parser")
-            if remove_scripts:
-                for script in soup(["script", "style"]):  # remove all javascript and stylesheet code
-                    script.extract()
-            output = soup.get_text()
-            return {Output.TEXT: output}
-        else:
+        # END INPUT BINDING - DO NOT REMOVE
+
+        if not document:
             return {Output.TEXT: ""}
+
+        soup = BeautifulSoup(document, features="html.parser")
+        if remove_scripts:
+            for script in soup(["script", "style"]):  # remove all javascript and stylesheet code
+                script.extract()
+        return {Output.TEXT: soup.get_text()}

plugins/html/icon_html/actions/validate/action.py

@@ -2,6 +2,7 @@
 import requests
 from .schema import ValidateOutput, ValidateInput, Input, Output, Component
 from insightconnect_plugin_runtime.exceptions import PluginException
+from icon_html.util.constants import REQUESTS_TIMEOUT
 
 
 class Validate(insightconnect_plugin_runtime.Action):
@@ -14,18 +15,24 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        # Configure requests
+        # START INPUT BINDING - DO NOT REMOVE - ANY INPUTS BELOW WILL UPDATE WITH YOUR PLUGIN SPEC AFTER REGENERATION
+        html_contents = params.get(Input.HTML_CONTENTS, "").encode()
+        # END INPUT BINDING - DO NOT REMOVE
 
-        headers = {"Content-Type": "text/html; charset=utf-8"}
-        api_call = "https://validator.w3.org/nu/?out=json"
-        html_data = params.get(Input.HTML_CONTENTS).encode()
         try:
-            response = requests.post(api_call, headers=headers, data=html_data, timeout=10)
-            msgs = response.json()["messages"]
-            if len(msgs) == 0:
+            response = requests.post(
+                "https://validator.w3.org/nu/?out=json",
+                headers={"Content-Type": "text/html; charset=utf-8"},
+                data=html_contents,
+                timeout=REQUESTS_TIMEOUT,
+            )
+            messages = response.json()["messages"]
+            if not messages:
                 self.logger.info("Run: No response from web service, can't determine validity")
                 return {Output.VALIDATED: False}
-            status = msgs[0]["type"]
+            status = messages[0]["type"]
             return {Output.VALIDATED: (not status == "error")}
-        except requests.exceptions.RequestException:
-            return PluginException(cause="Error validating input. ", assistance="Please check logs.")
+        except requests.exceptions.RequestException as error:
+            raise PluginException(cause="Error validating input.", assistance="Please check logs.", data=error)
+        except Exception as error:
+            raise PluginException(preset=PluginException.Preset.UNKNOWN, data=error)

plugins/html/icon_html/connection/connection.py

@@ -2,11 +2,16 @@
 from .schema import ConnectionSchema
 
 # Custom imports below
+from typing import Dict, Any
 
 
 class Connection(insightconnect_plugin_runtime.Connection):
     def __init__(self):
         super(self.__class__, self).__init__(input=ConnectionSchema())
 
-    def connect(self, params={}):
+    def connect(self, params={}) -> None:
         pass
+
+    @staticmethod
+    def test() -> Dict[str, Any]:
+        return {"success": True}

plugins/html/icon_html/util/api.py

@@ -0,0 +1,24 @@
+import re
+
+from icon_html.util.constants import HTML_TAG_PARSER
+from icon_html.util.strategies import HTMLConverterStrategy
+from insightconnect_plugin_runtime.exceptions import PluginException
+
+
+class HTMLConverter:
+    """HTMLConverter context"""
+
+    def __init__(self, strategy: HTMLConverterStrategy) -> None:
+        self._strategy = strategy
+
+    def convert(self, input_html_string: str) -> str:
+        self._validate_input_html(input_html_string)
+        try:
+            return self._strategy.convert(input_html_string)
+        except Exception as error:
+            raise PluginException(cause="Error converting doc file. ", assistance="Check stack trace log.", data=error)
+
+    @staticmethod
+    def _validate_input_html(input_html_string: str) -> None:
+        if not re.findall(HTML_TAG_PARSER, input_html_string):
+            raise PluginException(cause="Invalid input.", assistance="Input must be of type HTML.")

plugins/html/icon_html/util/constants.py

@@ -0,0 +1,3 @@
+REQUESTS_TIMEOUT = 10
+DEFAULT_ENCODING = "utf-8"
+HTML_TAG_PARSER = "(?i)<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[^'\">\s]+))?)+\s*|\s*)\/?>"

plugins/html/icon_html/util/helpers.py

@@ -0,0 +1,24 @@
+import base64
+from typing import Union
+
+import pypandoc
+
+from icon_html.util.constants import DEFAULT_ENCODING
+
+
+def encode_to_base64(content: Union[str, bytes]) -> str:
+    if isinstance(content, str):
+        content = content.encode(DEFAULT_ENCODING)
+    return base64.b64encode(content).decode(DEFAULT_ENCODING)
+
+
+def read_file_content(filename: str) -> str:
+    with open(filename, "rb") as file_:
+        return encode_to_base64(file_.read())
+
+
+def convert_with_temporary_file(
+    input_html_string: str, to_format: str, from_format: str, temporary_filename: str, *args, **kwargs
+) -> str:
+    pypandoc.convert_text(input_html_string, to_format, from_format, outputfile=temporary_filename, *args, **kwargs)
+    return read_file_content(temporary_filename)