From 1d1fefb4b8a2807a0c04496e23673330a5242553 Mon Sep 17 00:00:00 2001 From: Gernot Hillier Date: Wed, 9 Aug 2023 17:11:04 +0200 Subject: [PATCH] feat(bom downloadAttachments): read attachment id from control file --- capycli/bom/download_attachments.py | 43 +++-- tests/fixtures/sbom_for_download-control.json | 24 +++ tests/test_bom_downloadattachments.py | 152 ++++-------------- 3 files changed, 81 insertions(+), 138 deletions(-) create mode 100644 tests/fixtures/sbom_for_download-control.json diff --git a/capycli/bom/download_attachments.py b/capycli/bom/download_attachments.py index 091b8db..cf1dcdc 100644 --- a/capycli/bom/download_attachments.py +++ b/capycli/bom/download_attachments.py @@ -19,6 +19,7 @@ from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomWriter from capycli.common.print import print_red, print_text, print_yellow from capycli.common.script_support import ScriptSupport +from capycli.common.json_support import load_json_file from capycli.main.result_codes import ResultCode LOG = capycli.get_logger(__name__) @@ -29,7 +30,7 @@ class BomDownloadAttachments(capycli.common.script_base.ScriptBase): Download SW360 attachments as specified in the SBOM. """ - def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = None, + def download_attachments(self, sbom: Bom, control_components: list, source_folder: str, bompath: str = None, attachment_types: Tuple[str] = ("COMPONENT_LICENSE_INFO_XML", "CLEARING_REPORT")) -> Bom: for component in sbom.components: @@ -46,26 +47,23 @@ def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = Non if not found: continue - attachment_id = ext_ref.comment.split(", sw360Id: ") - if len(attachment_id) != 2: - print_red(" No sw360Id for attachment!") - continue - attachment_id = attachment_id[1] - release_id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID) if not release_id: print_red(" No sw360Id for release!") continue - print(" ", ext_ref.url, release_id, attachment_id) filename = os.path.join(source_folder, ext_ref.url) - try: - at_info = self.client.get_attachment(attachment_id) - at_info = {k: v for k, v in at_info.items() - if k.startswith("check") - or k.startswith("created")} - print(at_info) + details = [e for e in control_components + if e["Sw360Id"] == release_id and ( + e.get("CliFile", "") == ext_ref.url + or e.get("ReportFile", "") == ext_ref.url)] + if len(details) != 1: + print_red(" ERROR: Found", len(details), "entries for attachment", + ext_ref.url, "of", item_name, "in control file!") + continue + attachment_id = details[0]["Sw360AttachmentId"] + try: self.client.download_release_attachment(filename, release_id, attachment_id) ext_ref.url = filename try: @@ -103,6 +101,7 @@ def run(self, args): print("optional arguments:") print(" -h, --help show this help message and exit") print(" -i INPUTFILE, input SBOM to read from, e.g. created by \"project CreateBom\"") + print(" -ct CONTROLFILE, control file to read from as created by \"project CreateBom\"") print(" -source SOURCE source folder or additional source file") print(" -o OUTPUTFILE output file to write to") print(" -v be verbose") @@ -112,6 +111,10 @@ def run(self, args): print_red("No input file specified!") sys.exit(ResultCode.RESULT_COMMAND_ERROR) + if not args.controlfile: + print_red("No control file specified!") + sys.exit(ResultCode.RESULT_COMMAND_ERROR) + if not os.path.isfile(args.inputfile): print_red("Input file not found!") sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) @@ -126,6 +129,16 @@ def run(self, args): if args.verbose: print_text(" " + str(len(bom.components)) + "components read from SBOM file") + print_text("Loading control file " + args.controlfile) + try: + control = load_json_file(args.controlfile) + except Exception as ex: + print_red("JSON error reading control file: " + repr(ex)) + sys.exit(ResultCode.RESULT_ERROR_READING_BOM) + if "Components" not in control: + print_red("missing Components in control file") + sys.exit(ResultCode.RESULT_ERROR_READING_BOM) + source_folder = "./" if args.source: source_folder = args.source @@ -143,7 +156,7 @@ def run(self, args): print_text("Downloading source files to folder " + source_folder + " ...") - self.download_attachments(bom, source_folder, os.path.dirname(args.outputfile)) + self.download_attachments(bom, control["Components"], source_folder, os.path.dirname(args.outputfile)) if args.outputfile: print_text("Updating path information") diff --git a/tests/fixtures/sbom_for_download-control.json b/tests/fixtures/sbom_for_download-control.json new file mode 100644 index 0000000..dbedf07 --- /dev/null +++ b/tests/fixtures/sbom_for_download-control.json @@ -0,0 +1,24 @@ +{ + "ProjectName": "CaPyCLI, 2.0.0-dev1", + "Components": [ + { + "ComponentName": "certifi 2022.12.7", + "Sw360Id": "ae8c7ed", + "Sw360AttachmentId": "794446", + "CreatedBy": "user1@siemens.com", + "CreatedTeam": "AA", + "CreatedOn": "2020-10-23", + "CheckStatus": "ACCEPTED", + "CheckedBy": "user2@siemens.com", + "CheckedTeam": "BB", + "CheckedOn": "2020-10-30", + "CliFile": "CLIXML_certifi-2022.12.7.xml" + }, + { + "ComponentName": "certifi 2022.12.7", + "Sw360Id": "ae8c7ed", + "Sw360AttachmentId": "63b368", + "ReportFile": "certifi-2022.12.7_clearing_report.docx" + } + ] +} diff --git a/tests/test_bom_downloadattachments.py b/tests/test_bom_downloadattachments.py index 890c2f9..4de2820 100644 --- a/tests/test_bom_downloadattachments.py +++ b/tests/test_bom_downloadattachments.py @@ -11,15 +11,16 @@ import responses -from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport +from capycli.common.capycli_bom_support import CaPyCliBom +from capycli.common.json_support import load_json_file from capycli.bom.download_attachments import BomDownloadAttachments from capycli.main.result_codes import ResultCode -from cyclonedx.model import ExternalReferenceType, HashAlgorithm from tests.test_base import AppArguments, TestBase class TestBomDownloadAttachments(TestBase): INPUTFILE = "sbom_for_download.json" + CONTROLFILE = "sbom_for_download-control.json" INPUTERROR = "plaintext.txt" OUTPUTFILE = "output.json" @@ -69,6 +70,8 @@ def test_file_not_found(self) -> None: args.command.append("bom") args.command.append("downloadattachments") args.inputfile = "DOESNOTEXIST" + args.controlfile = os.path.join(os.path.dirname(__file__), + "fixtures", TestBomDownloadAttachments.CONTROLFILE) sut.run(args) self.assertTrue(False, "Failed to report missing file") @@ -85,6 +88,8 @@ def test_error_loading_file(self) -> None: args.command.append("bom") args.command.append("downloadattachments") args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTERROR) + args.controlfile = os.path.join(os.path.dirname(__file__), + "fixtures", TestBomDownloadAttachments.CONTROLFILE) sut.run(args) self.assertTrue(False, "Failed to report invalid file") @@ -103,6 +108,8 @@ def test_source_folder_does_not_exist(self) -> None: args.command.append("downloadattachments") args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) + args.controlfile = os.path.join(os.path.dirname(__file__), + "fixtures", TestBomDownloadAttachments.CONTROLFILE) args.source = "XXX" sut.run(args) @@ -113,32 +120,10 @@ def test_source_folder_does_not_exist(self) -> None: @responses.activate def test_simple_bom(self) -> None: bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) - bom = CaPyCliBom.read_sbom(bom) + controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE) - # attachment info - CLI - responses.add( - method=responses.GET, - url=self.MYURL + "resource/api/attachments/794446", - body=""" - { - "filename": "CLIXML_certifi-2022.12.7.xml", - "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", - "attachmentType": "COMPONENT_LICENSE_INFO_XML", - "createdBy": "thomas.graf@siemens.com", - "createdTeam": "AA", - "createdComment": "comment1", - "createdOn": "2020-10-08", - "checkStatus": "NOTCHECKED", - "_links": { - "self": { - "href": "https://my.server.com/resource/api/attachments/794446" - } - } - }""", - status=200, - content_type="application/json", - adding_headers={"Authorization": "Token " + self.MYTOKEN}, - ) + bom = CaPyCliBom.read_sbom(bom) + controlfile = load_json_file(controlfile) # get attachment - CLI cli_file = self.get_cli_file_mit() @@ -150,35 +135,6 @@ def test_simple_bom(self) -> None: content_type="application/text", adding_headers={"Authorization": "Token " + self.MYTOKEN}, ) - - # attachment info - report - responses.add( - method=responses.GET, - url=self.MYURL + "resource/api/attachments/63b368", - body=""" - { - "filename": "certifi-2022.12.7_clearing_report.docx", - "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", - "attachmentType": "CLEARING_REPORT", - "createdBy": "gernot.hillier@siemens.com", - "createdTeam": "BB", - "createdComment": "comment3", - "createdOn": "2020-10-08", - "checkedBy": "thomas.graf@siemens.com", - "checkedOn" : "2021-01-18", - "checkedComment": "comment4", - "checkStatus": "ACCEPTED", - "_links": { - "self": { - "href": "https://my.server.com/resource/api/attachments/63b368" - } - } - }""", - status=200, - content_type="application/json", - adding_headers={"Authorization": "Token " + self.MYTOKEN}, - ) - # get attachment - report responses.add( method=responses.GET, @@ -191,7 +147,7 @@ def test_simple_bom(self) -> None: with tempfile.TemporaryDirectory() as tmpdirname: try: - bom = self.app.download_attachments(bom, tmpdirname) + bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname) resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml") self.assertEqual(bom.components[0].external_references[5].url, resultfile) self.assertTrue(os.path.isfile(resultfile), "CLI file missing") @@ -211,25 +167,8 @@ def test_simple_bom_relpath(self) -> None: bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) bom = CaPyCliBom.read_sbom(bom) - # attachment info - CLI - responses.add( - method=responses.GET, - url=self.MYURL + "resource/api/attachments/794446", - body=""" - { - "filename": "CLIXML_certifi-2022.12.7.xml", - "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", - "attachmentType": "COMPONENT_LICENSE_INFO_XML", - "_links": { - "self": { - "href": "https://my.server.com/resource/api/attachments/794446" - } - } - }""", - status=200, - content_type="application/json", - adding_headers={"Authorization": "Token " + self.MYTOKEN}, - ) + controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE) + controlfile = load_json_file(controlfile) # get attachment - CLI cli_file = self.get_cli_file_mit() @@ -244,7 +183,8 @@ def test_simple_bom_relpath(self) -> None: with tempfile.TemporaryDirectory() as tmpdirname: try: - bom = self.app.download_attachments(bom, tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",)) + bom = self.app.download_attachments(bom, controlfile["Components"], + tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",)) resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml") self.assertEqual(bom.components[0].external_references[5].url, "file://CLIXML_certifi-2022.12.7.xml") @@ -263,59 +203,29 @@ def test_simple_bom_download_errors(self) -> None: bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) bom = CaPyCliBom.read_sbom(bom) - # attachment info - CLI, ok - responses.add( - method=responses.GET, - url=self.MYURL + "resource/api/attachments/794446", - body=""" - { - "filename": "CLIXML_certifi-2022.12.7.xml", - "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", - "attachmentType": "COMPONENT_LICENSE_INFO_XML", - "_links": { - "self": { - "href": "https://my.server.com/resource/api/attachments/794446" - } - } - }""", - status=200, - content_type="application/json", - adding_headers={"Authorization": "Token " + self.MYTOKEN}, - ) + controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE) + controlfile = load_json_file(controlfile) # get attachment - CLI, error responses.add( method=responses.GET, url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/794446", - body="cli_file", status=500, content_type="application/text", adding_headers={"Authorization": "Token " + self.MYTOKEN}, ) - - # attachment info - report, error + # get attachment - CLI, error responses.add( method=responses.GET, - url=self.MYURL + "resource/api/attachments/63b368", - body=""" - { - "filename": "certifi-2022.12.7_clearing_report.docx", - "sha1": "3cd24769fa3da4af74d0118433619a130da091b0", - "attachmentType": "CLEARING_REPORT", - "_links": { - "self": { - "href": "https://my.server.com/resource/api/attachments/63b368" - } - } - }""", - status=404, - content_type="application/json", + url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/63b368", + status=403, + content_type="application/text", adding_headers={"Authorization": "Token " + self.MYTOKEN}, ) with tempfile.TemporaryDirectory() as tmpdirname: try: - bom = self.app.download_attachments(bom, tmpdirname) + bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname) resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml") self.assertFalse(os.path.isfile(resultfile), "CLI created despite HTTP 500") @@ -335,8 +245,8 @@ def test_simple_bom_no_release_id(self) -> None: bom.components[0].properties = [] with tempfile.TemporaryDirectory() as tmpdirname: try: - err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname) - assert "No sw360Id for release" in err + err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname) + self.assertIn("No sw360Id for release", err) return except Exception as e: # noqa @@ -346,18 +256,14 @@ def test_simple_bom_no_release_id(self) -> None: self.assertTrue(False, "Error: we must never arrive here") @responses.activate - def test_simple_bom_no_attachment_id(self) -> None: + def test_simple_bom_no_ctrl_file_entry(self) -> None: bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE) bom = CaPyCliBom.read_sbom(bom) - bom.components[0].external_references = [] - CycloneDxSupport.set_ext_ref(bom.components[0], ExternalReferenceType.OTHER, - CaPyCliBom.CLI_FILE_COMMENT, "CLIXML_foo.xml", - HashAlgorithm.SHA_1, "123") with tempfile.TemporaryDirectory() as tmpdirname: try: - err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname) - assert "No sw360Id for attachment" in err + err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname) + assert "Found 0 entries for attachment CLIXML_certifi-2022.12.7.xml" in err return except Exception as e: # noqa