From c79d5e3aa4e48ee818a6bf4056bad56b4d2951a1 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Tue, 18 Jul 2023 16:06:07 +0200 Subject: [PATCH 1/5] refactor(project): move generic helpers to base class Move some reusable code from project classes to base class. --- capycli/common/script_base.py | 28 ++++++++++++++++++++++++++++ capycli/project/create_bom.py | 23 ++--------------------- capycli/project/show_ecc.py | 6 ++---- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/capycli/common/script_base.py b/capycli/common/script_base.py index c566132..bd613d1 100644 --- a/capycli/common/script_base.py +++ b/capycli/common/script_base.py @@ -10,6 +10,8 @@ Base class for python scripts. """ +from typing import List, Tuple + import json import os import sys @@ -111,6 +113,32 @@ def get_error_message(self, swex: sw360.sw360_api.SW360Error) -> str: str(jcontent["status"]) + "): " + jcontent["message"] return text + @staticmethod + def get_release_attachments(release_details: dict, att_types: Tuple[str] = None) -> List[dict]: + """Returns the attachments with the given types from a release. Use empty att_types + to get all attachments.""" + if "_embedded" not in release_details: + return None + + if "sw360:attachments" not in release_details["_embedded"]: + return None + + found = [] + attachments = release_details["_embedded"]["sw360:attachments"] + if not att_types: + return attachments + + for attachment in attachments: + if attachment["attachmentType"] in att_types: + found.append(attachment) + + return found + + def release_web_url(self, release_id) -> str: + """Returns the HTML URL for a given release_id.""" + return (self.sw360_url + "group/guest/components/-/component/release/detailRelease/" + + release_id) + def find_project(self, name: str, version: str, show_results: bool = False) -> str: """Find the project with the matching name and version on SW360""" print_text(" Searching for project...") diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 9941aea..c22817b 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -8,7 +8,6 @@ import logging import sys -from typing import List, Tuple import sw360 from cyclonedx.model import ExternalReferenceType, HashAlgorithm @@ -34,22 +33,6 @@ def get_external_id(self, name: str, release_details: dict): return release_details["externalIds"].get(name, "") - def get_attachments(self, att_types: Tuple[str], release_details: dict) -> List[dict]: - """Returns the attachments with the given types or empty list.""" - if "_embedded" not in release_details: - return None - - if "sw360:attachments" not in release_details["_embedded"]: - return None - - found = [] - attachments = release_details["_embedded"]["sw360:attachments"] - for attachment in attachments: - if attachment["attachmentType"] in att_types: - found.append(attachment) - - return found - def get_clearing_state(self, proj, href) -> str: """Returns the clearing state of the given component/release""" rel = proj["linkedReleases"] @@ -103,7 +86,7 @@ def create_project_bom(self, project) -> list: for at_type, comment in (("SOURCE", CaPyCliBom.SOURCE_FILE_COMMENT), ("BINARY", CaPyCliBom.BINARY_FILE_COMMENT)): - attachments = self.get_attachments((at_type, at_type + "_SELF"), release_details) + attachments = self.get_release_attachments(release_details, (at_type, at_type + "_SELF")) for attachment in attachments: CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.DISTRIBUTION, comment, attachment["filename"], @@ -123,9 +106,7 @@ def create_project_bom(self, project) -> list: CycloneDxSupport.set_property( rel_item, CycloneDxSupport.CDX_PROP_SW360_URL, - self.sw360_url - + "group/guest/components/-/component/release/detailRelease/" - + sw360_id) + self.release_web_url(sw360_id)) bom.append(rel_item) diff --git a/capycli/project/show_ecc.py b/capycli/project/show_ecc.py index cebfbc8..644c8c7 100644 --- a/capycli/project/show_ecc.py +++ b/capycli/project/show_ecc.py @@ -114,10 +114,8 @@ def get_project_status(self, project_id: str): rel_item["Id"] = self.client.get_id_from_href(href) rel_item["S360Id"] = rel_item["Id"] rel_item["Href"] = href - rel_item["Url"] = ( - self.sw360_url - + "group/guest/components/-/component/release/detailRelease/" - + self.client.get_id_from_href(href)) + rel_item["Url"] = self.release_web_url( + self.client.get_id_from_href(href)) try: release_details = self.client.get_release_by_url(href) From b7e156841ac8dc69938c15e6b9433fdb247ddf80 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Wed, 19 Jul 2023 08:40:26 +0200 Subject: [PATCH 2/5] test(bom_downloadsources): fix copy-paste error in class name --- tests/test_bom_downloadsources.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_bom_downloadsources.py b/tests/test_bom_downloadsources.py index 1d9cde4..ef03c1e 100644 --- a/tests/test_bom_downloadsources.py +++ b/tests/test_bom_downloadsources.py @@ -16,7 +16,7 @@ from tests.test_base import AppArguments, TestBase -class TestShowBom(TestBase): +class TestBomDownloadsources(TestBase): INPUTFILE = "sbom_for_download.json" INPUTERROR = "plaintext.txt" OUTPUTFILE = "output.json" @@ -74,7 +74,7 @@ def test_error_loading_file(self) -> None: args.command = [] args.command.append("bom") args.command.append("downloadsources") - args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTERROR) + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTERROR) sut.run(args) self.assertTrue(False, "Failed to report invalid file") @@ -90,7 +90,7 @@ def test_source_folder_does_not_exist(self) -> None: args.command = [] args.command.append("bom") args.command.append("downloadsources") - args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE) + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE) args.source = "XXX" sut.run(args) @@ -107,8 +107,8 @@ def test_simple_bom(self) -> None: args.command = [] args.command.append("bom") args.command.append("downloadsources") - args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE) - args.outputfile = TestShowBom.OUTPUTFILE + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE) + args.outputfile = TestBomDownloadsources.OUTPUTFILE with tempfile.TemporaryDirectory() as tmpdirname: args.source = tmpdirname @@ -152,8 +152,8 @@ def test_simple_bom_error_download(self) -> None: args.command = [] args.command.append("bom") args.command.append("downloadsources") - args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.INPUTFILE) - args.outputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestShowBom.OUTPUTFILE) + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE) + args.outputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.OUTPUTFILE) with tempfile.TemporaryDirectory() as tmpdirname: args.source = tmpdirname From c3171b50f4cc6c43d1caf9a82c7e0faaef86206d Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sun, 30 Jul 2023 08:17:39 +0200 Subject: [PATCH 3/5] test(base): refactor output capture methods They now allow an arbitrary number of arguments, so we also don't neet the _no_args variant any more. --- tests/test_base.py | 31 ++++--------------------------- tests/test_logging.py | 2 +- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/tests/test_base.py b/tests/test_base.py index 5b9c5ef..a1306a8 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -84,16 +84,13 @@ def dump_textfile(text: str, filename: str) -> None: outfile.write(text) @staticmethod - def capture_stderr(func: Any, args: Any = None) -> str: + def capture_stderr(func: Any, *args, **kwargs) -> str: """Capture stderr for the given function and return result as string""" # setup the environment old_stderr = sys.stderr sys.stderr = TextIOWrapper(BytesIO(), sys.stderr.encoding) - if args: - func(args) - else: - func() + func(*args, **kwargs) # get output sys.stderr.seek(0) # jump to the start @@ -106,33 +103,13 @@ def capture_stderr(func: Any, args: Any = None) -> str: return out @staticmethod - def capture_stdout(func: Any, args: Any = None) -> str: + def capture_stdout(func: Any, *args, **kwargs) -> str: """Capture stdout for the given function and return result as string""" # setup the environment old_stdout = sys.stdout sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding) - func(args) - - # get output - sys.stdout.seek(0) # jump to the start - out = sys.stdout.read() # read output - - # restore stdout - sys.stdout.close() - sys.stdout = old_stdout - - return out - - @staticmethod - def capture_stdout_no_args(func: Any) -> str: - """Capture stdout for the given function and return result as string""" - # setup the environment - old_stdout = sys.stdout - sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding) - - func() - + func(*args, **kwargs) # get output sys.stdout.seek(0) # jump to the start out = sys.stdout.read() # read output diff --git a/tests/test_logging.py b/tests/test_logging.py index 87a0909..26f45ab 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -35,7 +35,7 @@ def test_error_critical(self) -> None: self.assertFalse(self.DEBUG_MSG in out) def test_info_debug(self) -> None: - out = self.capture_stdout_no_args(self.create_output) + out = self.capture_stdout(self.create_output) # self.dump_textfile(out, "dump.txt") self.assertFalse(self.CRITICAL_MSG in out) self.assertFalse(self.ERROR_MSG in out) From 0e453aded5963a326f3210a86b19ef3801c6899a Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sat, 5 Aug 2023 08:05:28 +0200 Subject: [PATCH 4/5] refactor(bom downloadsources): narrow relpath helper Replace have_relative_source_file_path() helper function by a generic have_relative_ext_ref_path() which can be re-used for other tasks. To be sure we do the right thing, also add test cases for absolute and relative paths. --- capycli/bom/download_sources.py | 31 +++++++---------- tests/test_bom_downloadsources.py | 55 +++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/capycli/bom/download_sources.py b/capycli/bom/download_sources.py index e0a34e8..f9f9639 100644 --- a/capycli/bom/download_sources.py +++ b/capycli/bom/download_sources.py @@ -18,7 +18,6 @@ import requests from cyclonedx.model import ExternalReference, ExternalReferenceType, HashAlgorithm, HashType from cyclonedx.model.bom import Bom -from cyclonedx.model.component import Component import capycli.common.json_support import capycli.common.script_base @@ -122,35 +121,29 @@ def download_sources(self, sbom: Bom, source_folder: str) -> None: if new: component.external_references.add(ext_ref) - def have_relative_source_file_path(self, component: Component, bompath: str): - ext_ref = CycloneDxSupport.get_ext_ref( - component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) - if not ext_ref: - return - + def have_relative_ext_ref_path(self, ext_ref: ExternalReference, bompath: str): bip = pathlib.PurePath(ext_ref.url) try: - CycloneDxSupport.update_or_set_property( - component, - CycloneDxSupport.CDX_PROP_FILENAME, - bip.name) file = bip.as_posix() if os.path.isfile(file): - CycloneDxSupport.update_or_set_ext_ref( - component, - ExternalReferenceType.DISTRIBUTION, - CaPyCliBom.SOURCE_FILE_COMMENT, - "file://" + bip.relative_to(bompath).as_posix()) + ext_ref.url = "file://" + bip.relative_to(bompath).as_posix() except ValueError: print_yellow( " SBOM file is not relative to source file " + ext_ref.url) - # .relative_to - pass + + return bip.name def update_local_path(self, sbom: Bom, bomfile: str): bompath = pathlib.Path(bomfile).parent for component in sbom.components: - self.have_relative_source_file_path(component, bompath) + ext_ref = CycloneDxSupport.get_ext_ref( + component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) + if ext_ref: + name = self.have_relative_ext_ref_path(ext_ref, bompath) + CycloneDxSupport.update_or_set_property( + component, + CycloneDxSupport.CDX_PROP_FILENAME, + name) def run(self, args): """Main method diff --git a/tests/test_bom_downloadsources.py b/tests/test_bom_downloadsources.py index ef03c1e..34487b2 100644 --- a/tests/test_bom_downloadsources.py +++ b/tests/test_bom_downloadsources.py @@ -11,6 +11,9 @@ import responses +from cyclonedx.model import ExternalReferenceType + +from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport from capycli.bom.download_sources import BomDownloadSources from capycli.main.result_codes import ResultCode from tests.test_base import AppArguments, TestBase @@ -126,20 +129,68 @@ def test_simple_bom(self) -> None: try: out = self.capture_stdout(sut.run, args) + out_bom = CaPyCliBom.read_sbom(args.outputfile) # capycli.common.json_support.write_json_to_file(out, "STDOUT.TXT") self.assertTrue("Loading SBOM file" in out) self.assertTrue("sbom_for_download.json" in out) # path may vary + self.assertIn("SBOM file is not relative to", out) self.assertTrue("Downloading source files to folder" in out) self.assertTrue("Downloading file certifi-2022.12.7.tar.gz" in out) resultfile = os.path.join(tmpdirname, "certifi-2022.12.7.tar.gz") self.assertTrue(os.path.isfile(resultfile)) + ext_ref = CycloneDxSupport.get_ext_ref( + out_bom.components[0], ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) + self.assertEqual(ext_ref.url, resultfile) + self.delete_file(args.outputfile) return - except: # noqa + except Exception as e: # noqa # catch all exception to let Python cleanup the temp folder - pass + print(e) + + self.assertTrue(False, "Error: we must never arrive here") + + @responses.activate + def test_simple_bom_relative_path(self) -> None: + sut = BomDownloadSources() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("downloadsources") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadsources.INPUTFILE) + + with tempfile.TemporaryDirectory() as tmpdirname: + args.source = tmpdirname + args.outputfile = os.path.join(tmpdirname, TestBomDownloadsources.OUTPUTFILE) + + # fake file content + responses.add( + responses.GET, + url="https://files.pythonhosted.org/packages/37/f7/2b1b/certifi-2022.12.7.tar.gz", + body=""" + SOME DUMMY DATA + """, + status=200, + content_type="application/json", + ) + + try: + sut.run(args) + out_bom = CaPyCliBom.read_sbom(args.outputfile) + + ext_ref = CycloneDxSupport.get_ext_ref( + out_bom.components[0], ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) + self.assertEqual(ext_ref.url, "file://certifi-2022.12.7.tar.gz") + + self.delete_file(args.outputfile) + return + except Exception as e: # noqa + # catch all exception to let Python cleanup the temp folder + print(e) self.assertTrue(False, "Error: we must never arrive here") From 2a2c2bf5c2bcdef4b152c651e1e2e2c0277f71c7 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Sat, 5 Aug 2023 08:24:05 +0200 Subject: [PATCH 5/5] refactor: move ext_ref relpath helper to CycloneDxSupport --- capycli/bom/download_sources.py | 25 ++++++++----------------- capycli/common/capycli_bom_support.py | 9 +++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/capycli/bom/download_sources.py b/capycli/bom/download_sources.py index f9f9639..0fee855 100644 --- a/capycli/bom/download_sources.py +++ b/capycli/bom/download_sources.py @@ -121,29 +121,20 @@ def download_sources(self, sbom: Bom, source_folder: str) -> None: if new: component.external_references.add(ext_ref) - def have_relative_ext_ref_path(self, ext_ref: ExternalReference, bompath: str): - bip = pathlib.PurePath(ext_ref.url) - try: - file = bip.as_posix() - if os.path.isfile(file): - ext_ref.url = "file://" + bip.relative_to(bompath).as_posix() - except ValueError: - print_yellow( - " SBOM file is not relative to source file " + ext_ref.url) - - return bip.name - def update_local_path(self, sbom: Bom, bomfile: str): bompath = pathlib.Path(bomfile).parent for component in sbom.components: ext_ref = CycloneDxSupport.get_ext_ref( component, ExternalReferenceType.DISTRIBUTION, CaPyCliBom.SOURCE_FILE_COMMENT) if ext_ref: - name = self.have_relative_ext_ref_path(ext_ref, bompath) - CycloneDxSupport.update_or_set_property( - component, - CycloneDxSupport.CDX_PROP_FILENAME, - name) + try: + name = CycloneDxSupport.have_relative_ext_ref_path(ext_ref, bompath) + CycloneDxSupport.update_or_set_property( + component, + CycloneDxSupport.CDX_PROP_FILENAME, + name) + except ValueError: + print_yellow(" SBOM file is not relative to source file " + ext_ref.url) def run(self, args): """Main method diff --git a/capycli/common/capycli_bom_support.py b/capycli/common/capycli_bom_support.py index c5b8e8d..751a761 100644 --- a/capycli/common/capycli_bom_support.py +++ b/capycli/common/capycli_bom_support.py @@ -9,6 +9,7 @@ import json import os import tempfile +import pathlib import uuid from datetime import datetime from enum import Enum @@ -370,6 +371,14 @@ def update_or_set_ext_ref(comp: Component, type: ExternalReferenceType, comment: else: CycloneDxSupport.set_ext_ref(comp, type, comment, value) + @staticmethod + def have_relative_ext_ref_path(ext_ref: ExternalReference, rel_to: str): + bip = pathlib.PurePath(ext_ref.url) + file = bip.as_posix() + if os.path.isfile(file): + ext_ref.url = "file://" + bip.relative_to(rel_to).as_posix() + return bip.name + @staticmethod def get_ext_ref_by_comment(comp: Component, comment: str) -> Any: for ext_ref in comp.external_references: