Merge pull request #35 from sw360/rework-project-bom-code

Rework some project and bom methods

capycli/bom/download_sources.py

@@ -18,7 +18,6 @@
 import requests
 from cyclonedx.model import ExternalReference, ExternalReferenceType, HashAlgorithm, HashType
 from cyclonedx.model.bom import Bom
-from cyclonedx.model.component import Component
 
 import capycli.common.json_support
 import capycli.common.script_base
@@ -122,35 +121,20 @@ def download_sources(self, sbom: Bom, source_folder: str) -> None:
                 if new:
                     component.external_references.add(ext_ref)
 
-    def have_relative_source_file_path(self, component: Component, bompath: str):
-        ext_ref = CycloneDxSupport.get_ext_ref(
-            component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT)
-        if not ext_ref:
-            return
-
-        bip = pathlib.PurePath(ext_ref.url)
-        try:
-            CycloneDxSupport.update_or_set_property(
-                component,
-                CycloneDxSupport.CDX_PROP_FILENAME,
-                bip.name)
-            file = bip.as_posix()
-            if os.path.isfile(file):
-                CycloneDxSupport.update_or_set_ext_ref(
-                    component,
-                    ExternalReferenceType.DISTRIBUTION,
-                    CaPyCliBom.SOURCE_FILE_COMMENT,
-                    "file://" + bip.relative_to(bompath).as_posix())
-        except ValueError:
-            print_yellow(
-                "  SBOM file is not relative to source file " + ext_ref.url)
-        # .relative_to
-        pass
-
     def update_local_path(self, sbom: Bom, bomfile: str):
         bompath = pathlib.Path(bomfile).parent
         for component in sbom.components:
-            self.have_relative_source_file_path(component, bompath)
+            ext_ref = CycloneDxSupport.get_ext_ref(
+                component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT)
+            if ext_ref:
+                try:
+                    name = CycloneDxSupport.have_relative_ext_ref_path(ext_ref, bompath)
+                    CycloneDxSupport.update_or_set_property(
+                        component,
+                        CycloneDxSupport.CDX_PROP_FILENAME,
+                        name)
+                except ValueError:
+                    print_yellow("  SBOM file is not relative to source file " + ext_ref.url)
 
     def run(self, args):
         """Main method

capycli/common/capycli_bom_support.py

@@ -9,6 +9,7 @@
 import json
 import os
 import tempfile
+import pathlib
 import uuid
 from datetime import datetime
 from enum import Enum
@@ -370,6 +371,14 @@ def update_or_set_ext_ref(comp: Component, type: ExternalReferenceType, comment:
         else:
             CycloneDxSupport.set_ext_ref(comp, type, comment, value)
 
+    @staticmethod
+    def have_relative_ext_ref_path(ext_ref: ExternalReference, rel_to: str):
+        bip = pathlib.PurePath(ext_ref.url)
+        file = bip.as_posix()
+        if os.path.isfile(file):
+            ext_ref.url = "file://" + bip.relative_to(rel_to).as_posix()
+        return bip.name
+
     @staticmethod
     def get_ext_ref_by_comment(comp: Component, comment: str) -> Any:
         for ext_ref in comp.external_references:

capycli/common/script_base.py

@@ -10,6 +10,8 @@
 Base class for python scripts.
 """
 
+from typing import List, Tuple
+
 import json
 import os
 import sys
@@ -111,6 +113,32 @@ def get_error_message(self, swex: sw360.sw360_api.SW360Error) -> str:
                     str(jcontent["status"]) + "): " + jcontent["message"]
                 return text
 
+    @staticmethod
+    def get_release_attachments(release_details: dict, att_types: Tuple[str] = None) -> List[dict]:
+        """Returns the attachments with the given types from a release. Use empty att_types
+        to get all attachments."""
+        if "_embedded" not in release_details:
+            return None
+
+        if "sw360:attachments" not in release_details["_embedded"]:
+            return None
+
+        found = []
+        attachments = release_details["_embedded"]["sw360:attachments"]
+        if not att_types:
+            return attachments
+
+        for attachment in attachments:
+            if attachment["attachmentType"] in att_types:
+                found.append(attachment)
+
+        return found
+
+    def release_web_url(self, release_id) -> str:
+        """Returns the HTML URL for a given release_id."""
+        return (self.sw360_url + "group/guest/components/-/component/release/detailRelease/"
+                + release_id)
+
     def find_project(self, name: str, version: str, show_results: bool = False) -> str:
         """Find the project with the matching name and version on SW360"""
         print_text("  Searching for project...")

capycli/project/create_bom.py

@@ -8,7 +8,6 @@
 
 import logging
 import sys
-from typing import List, Tuple
 
 import sw360
 from cyclonedx.model import ExternalReferenceType, HashAlgorithm
@@ -34,22 +33,6 @@ def get_external_id(self, name: str, release_details: dict):
 
         return release_details["externalIds"].get(name, "")
 
-    def get_attachments(self, att_types: Tuple[str], release_details: dict) -> List[dict]:
-        """Returns the attachments with the given types or empty list."""
-        if "_embedded" not in release_details:
-            return None
-
-        if "sw360:attachments" not in release_details["_embedded"]:
-            return None
-
-        found = []
-        attachments = release_details["_embedded"]["sw360:attachments"]
-        for attachment in attachments:
-            if attachment["attachmentType"] in att_types:
-                found.append(attachment)
-
-        return found
-
     def get_clearing_state(self, proj, href) -> str:
         """Returns the clearing state of the given component/release"""
         rel = proj["linkedReleases"]
@@ -103,7 +86,7 @@ def create_project_bom(self, project) -> list:
 
                 for at_type, comment in (("SOURCE", CaPyCliBom.SOURCE_FILE_COMMENT),
                                          ("BINARY", CaPyCliBom.BINARY_FILE_COMMENT)):
-                    attachments = self.get_attachments((at_type, at_type + "_SELF"), release_details)
+                    attachments = self.get_release_attachments(release_details, (at_type, at_type + "_SELF"))
                     for attachment in attachments:
                         CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.DISTRIBUTION,
                                                      comment, attachment["filename"],
@@ -123,9 +106,7 @@ def create_project_bom(self, project) -> list:
             CycloneDxSupport.set_property(
                 rel_item,
                 CycloneDxSupport.CDX_PROP_SW360_URL,
-                self.sw360_url
-                + "group/guest/components/-/component/release/detailRelease/"
-                + sw360_id)
+                self.release_web_url(sw360_id))
 
             bom.append(rel_item)
 

capycli/project/show_ecc.py

@@ -114,10 +114,8 @@ def get_project_status(self, project_id: str):
                 rel_item["Id"] = self.client.get_id_from_href(href)
                 rel_item["S360Id"] = rel_item["Id"]
                 rel_item["Href"] = href
-                rel_item["Url"] = (
-                    self.sw360_url
-                    + "group/guest/components/-/component/release/detailRelease/"
-                    + self.client.get_id_from_href(href))
+                rel_item["Url"] = self.release_web_url(
+                    self.client.get_id_from_href(href))
 
                 try:
                     release_details = self.client.get_release_by_url(href)

tests/test_base.py

@@ -84,16 +84,13 @@ def dump_textfile(text: str, filename: str) -> None:
             outfile.write(text)
 
     @staticmethod
-    def capture_stderr(func: Any, args: Any = None) -> str:
+    def capture_stderr(func: Any, *args, **kwargs) -> str:
         """Capture stderr for the given function and return result as string"""
         # setup the environment
         old_stderr = sys.stderr
         sys.stderr = TextIOWrapper(BytesIO(), sys.stderr.encoding)
 
-        if args:
-            func(args)
-        else:
-            func()
+        func(*args, **kwargs)
 
         # get output
         sys.stderr.seek(0)       # jump to the start
@@ -106,33 +103,13 @@ def capture_stderr(func: Any, args: Any = None) -> str:
         return out
 
     @staticmethod
-    def capture_stdout(func: Any, args: Any = None) -> str:
+    def capture_stdout(func: Any, *args, **kwargs) -> str:
         """Capture stdout for the given function and return result as string"""
         # setup the environment
         old_stdout = sys.stdout
         sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding)
 
-        func(args)
-
-        # get output
-        sys.stdout.seek(0)       # jump to the start
-        out = sys.stdout.read()  # read output
-
-        # restore stdout
-        sys.stdout.close()
-        sys.stdout = old_stdout
-
-        return out
-
-    @staticmethod
-    def capture_stdout_no_args(func: Any) -> str:
-        """Capture stdout for the given function and return result as string"""
-        # setup the environment
-        old_stdout = sys.stdout
-        sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding)
-
-        func()
-
+        func(*args, **kwargs)
         # get output
         sys.stdout.seek(0)       # jump to the start
         out = sys.stdout.read()  # read output

tests/test_bom_downloadsources.py

@@ -11,12 +11,15 @@
 
 import responses
 
+from cyclonedx.model import ExternalReferenceType
+
+from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
 from capycli.bom.download_sources import BomDownloadSources
 from capycli.main.result_codes import ResultCode
 from tests.test_base import AppArguments, TestBase
 
 
-class TestShowBom(TestBase):
+class TestBomDownloadsources(TestBase):
     INPUTFILE = "sbom_for_download.json"
     INPUTERROR = "plaintext.txt"
     OUTPUTFILE = "output.json"
@@ -74,7 +77,7 @@ def test_error_loading_file(self) -> None:
             args.command = []
             args.command.append("bom")
             args.command.append("downloadsources")
-            args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTERROR)
+            args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTERROR)
 
             sut.run(args)
             self.assertTrue(False, "Failed to report invalid file")
@@ -90,7 +93,7 @@ def test_source_folder_does_not_exist(self) -> None:
             args.command = []
             args.command.append("bom")
             args.command.append("downloadsources")
-            args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE)
+            args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE)
             args.source = "XXX"
 
             sut.run(args)
@@ -107,8 +110,8 @@ def test_simple_bom(self) -> None:
         args.command = []
         args.command.append("bom")
         args.command.append("downloadsources")
-        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE)
-        args.outputfile = TestShowBom.OUTPUTFILE
+        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE)
+        args.outputfile = TestBomDownloadsources.OUTPUTFILE
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             args.source = tmpdirname
@@ -126,20 +129,68 @@ def test_simple_bom(self) -> None:
 
             try:
                 out = self.capture_stdout(sut.run, args)
+                out_bom = CaPyCliBom.read_sbom(args.outputfile)
                 # capycli.common.json_support.write_json_to_file(out, "STDOUT.TXT")
                 self.assertTrue("Loading SBOM file" in out)
                 self.assertTrue("sbom_for_download.json" in out)  # path may vary
+                self.assertIn("SBOM file is not relative to", out)
                 self.assertTrue("Downloading source files to folder" in out)
                 self.assertTrue("Downloading file certifi-2022.12.7.tar.gz" in out)
 
                 resultfile = os.path.join(tmpdirname, "certifi-2022.12.7.tar.gz")
                 self.assertTrue(os.path.isfile(resultfile))
 
+                ext_ref = CycloneDxSupport.get_ext_ref(
+                    out_bom.components[0], ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT)
+                self.assertEqual(ext_ref.url, resultfile)
+
                 self.delete_file(args.outputfile)
                 return
-            except:  # noqa
+            except Exception as e:  # noqa
                 # catch all exception to let Python cleanup the temp folder
-                pass
+                print(e)
+
+        self.assertTrue(False, "Error: we must never arrive here")
+
+    @responses.activate
+    def test_simple_bom_relative_path(self) -> None:
+        sut = BomDownloadSources()
+
+        # create argparse command line argument object
+        args = AppArguments()
+        args.command = []
+        args.command.append("bom")
+        args.command.append("downloadsources")
+        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE)
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            args.source = tmpdirname
+            args.outputfile = os.path.join(tmpdirname, TestBomDownloadsources.OUTPUTFILE)
+
+            # fake file content
+            responses.add(
+                responses.GET,
+                url="https://files.pythonhosted.org/packages/37/f7/2b1b/certifi-2022.12.7.tar.gz",
+                body="""
+                SOME DUMMY DATA
+                """,
+                status=200,
+                content_type="application/json",
+            )
+
+            try:
+                sut.run(args)
+                out_bom = CaPyCliBom.read_sbom(args.outputfile)
+
+                ext_ref = CycloneDxSupport.get_ext_ref(
+                    out_bom.components[0], ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT)
+                self.assertEqual(ext_ref.url, "file://certifi-2022.12.7.tar.gz")
+
+                self.delete_file(args.outputfile)
+                return
+            except Exception as e:  # noqa
+                # catch all exception to let Python cleanup the temp folder
+                print(e)
 
         self.assertTrue(False, "Error: we must never arrive here")
 
@@ -152,8 +203,8 @@ def test_simple_bom_error_download(self) -> None:
         args.command = []
         args.command.append("bom")
         args.command.append("downloadsources")
-        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE)
-        args.outputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.OUTPUTFILE)
+        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE)
+        args.outputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.OUTPUTFILE)
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             args.source = tmpdirname

tests/test_logging.py

@@ -35,7 +35,7 @@ def test_error_critical(self) -> None:
         self.assertFalse(self.DEBUG_MSG in out)
 
     def test_info_debug(self) -> None:
-        out = self.capture_stdout_no_args(self.create_output)
+        out = self.capture_stdout(self.create_output)
         # self.dump_textfile(out, "dump.txt")
         self.assertFalse(self.CRITICAL_MSG in out)
         self.assertFalse(self.ERROR_MSG in out)