From 76272c47035475126bf3252a2d72d100eff372d7 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 10 Nov 2021 15:57:32 -0500 Subject: [PATCH 1/2] support spdx 2.2 compliant json Signed-off-by: Daniel Holth --- spdx/creationinfo.py | 4 +- spdx/file.py | 14 +- spdx/package.py | 14 +- spdx/parsers/jsonyamlxml.py | 35 ++- spdx/writers/json.py | 4 +- spdx/writers/jsonyamlxml.py | 55 ++++- spdx/writers/rdf.py | 3 +- spdx/writers/xml.py | 15 +- spdx/writers/yaml.py | 4 +- tests/data/doc_write/json-simple-plus.json | 13 +- .../data/doc_write/json-simple-plus.new.json | 59 +++++ tests/data/doc_write/json-simple.json | 13 +- tests/data/doc_write/json-simple.new.json | 59 +++++ .../data/doc_write/yaml-simple-plus.new.yaml | 38 +++ tests/data/doc_write/yaml-simple-plus.yaml | 13 +- tests/data/doc_write/yaml-simple.new.yaml | 38 +++ tests/data/doc_write/yaml-simple.yaml | 13 +- tests/data/formats/SPDXJsonExample2.2.json | 231 ++++++++++++++++++ tests/test_document.py | 8 +- tests/test_write_anything.py | 6 +- tests/utils_test.py | 4 +- 21 files changed, 607 insertions(+), 36 deletions(-) create mode 100644 tests/data/doc_write/json-simple-plus.new.json create mode 100644 tests/data/doc_write/json-simple.new.json create mode 100644 tests/data/doc_write/yaml-simple-plus.new.yaml create mode 100644 tests/data/doc_write/yaml-simple.new.yaml create mode 100644 tests/data/formats/SPDXJsonExample2.2.json diff --git a/spdx/creationinfo.py b/spdx/creationinfo.py index 1545a8ab7..54a99e9d3 100644 --- a/spdx/creationinfo.py +++ b/spdx/creationinfo.py @@ -44,7 +44,7 @@ class Organization(Creator): - email: Org's email address. Optional. Type: str. """ - def __init__(self, name, email): + def __init__(self, name, email=None): super(Organization, self).__init__(name) self.email = email @@ -80,7 +80,7 @@ class Person(Creator): - email: person's email address. Optional. Type: str. """ - def __init__(self, name, email): + def __init__(self, name, email=None): super(Person, self).__init__(name) self.email = email diff --git a/spdx/file.py b/spdx/file.py index f35a7e2aa..0ae9f4549 100644 --- a/spdx/file.py +++ b/spdx/file.py @@ -63,7 +63,7 @@ def __init__(self, name, spdx_id=None, chk_sum=None): self.spdx_id = spdx_id self.comment = None self.type = None - self.chk_sum = chk_sum + self.checksums = [None] self.conc_lics = None self.licenses_in_file = [] self.license_comment = None @@ -82,6 +82,18 @@ def __eq__(self, other): def __lt__(self, other): return self.name < other.name + @property + def chk_sum(self): + """ + Backwards compatibility, return first checksum. + """ + # NOTE Package.check_sum but File.chk_sum + return self.checksums[0] + + @chk_sum.setter + def chk_sum(self, value): + self.checksums[0] = value + def add_lics(self, lics): self.licenses_in_file.append(lics) diff --git a/spdx/package.py b/spdx/package.py index 32279d2d5..12036b928 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -82,7 +82,7 @@ def __init__( self.files_analyzed = None self.homepage = None self.verif_code = None - self.check_sum = None + self.checksums = [None] self.source_info = None self.conc_lics = None self.license_declared = None @@ -103,6 +103,18 @@ def are_files_analyzed(self): # as default None Value is False, previous line is simplification of # return self.files_analyzed or self.files_analyzed is None + @property + def check_sum(self): + """ + Backwards compatibility, return first checksum. + """ + # NOTE Package.check_sum but File.chk_sum + return self.checksums[0] + + @check_sum.setter + def check_sum(self, value): + self.checksums[0] = value + def add_file(self, fil): self.files.append(fil) diff --git a/spdx/parsers/jsonyamlxml.py b/spdx/parsers/jsonyamlxml.py index b4f254240..c6a981d30 100644 --- a/spdx/parsers/jsonyamlxml.py +++ b/spdx/parsers/jsonyamlxml.py @@ -1486,6 +1486,39 @@ def parse_pkg_chksum(self, pkg_chksum): self.value_error("PKG_CHECKSUM", pkg_chksum) +def unflatten_document(document): + """ + Inverse of spdx.writers.jsonyamlxml.flatten_document + """ + files_by_id = {} + if "files" in document: + for f in document.pop("files"): + f["name"] = f.pop("fileName") + # XXX must downstream rely on "sha1" property? + for checksum in f["checksums"]: + if checksum["algorithm"] == "SHA1": + f["sha1"] = checksum["checksumValue"] + break + if "licenseInfoInFiles" in f: + f["licenseInfoFromFiles"] = f.pop("licenseInfoInFiles") + files_by_id[f["SPDXID"]] = f + + packages = document.pop("packages") + for package in packages: + if "hasFiles" in package: + package["files"] = [{ + "File": files_by_id[spdxid]} for spdxid in package["hasFiles"] + ] + # XXX must downstream rely on "sha1" property? + for checksum in package.get("checksums", []): + if checksum["algorithm"] == "SHA1": + package["sha1"] = checksum["checksumValue"] + break + + document["documentDescribes"] = [{ "Package": package} for package in packages ] + + return document + class Parser( CreationInfoParser, ExternalDocumentRefsParser, @@ -1503,7 +1536,7 @@ def __init__(self, builder, logger): def json_yaml_set_document(self, data): # we could verify that the spdxVersion >= 2.2, but we try to be resilient in parsing if data.get("spdxVersion"): - self.document_object = data + self.document_object = unflatten_document(data) return self.document_object = data.get("Document") diff --git a/spdx/writers/json.py b/spdx/writers/json.py index 0177cd104..a587a650b 100644 --- a/spdx/writers/json.py +++ b/spdx/writers/json.py @@ -12,7 +12,7 @@ import json from spdx.writers.tagvalue import InvalidDocumentError -from spdx.writers.jsonyamlxml import Writer +from spdx.writers.jsonyamlxml import JsonYamlWriter from spdx.parsers.loggers import ErrorMessages @@ -24,6 +24,6 @@ def write_document(document, out, validate=True): if messages: raise InvalidDocumentError(messages) - writer = Writer(document) + writer = JsonYamlWriter(document) document_object = writer.create_document() json.dump(document_object, out, indent=4) diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index faac513dd..9b06b3319 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -47,7 +47,7 @@ def checksum(self, checksum_field): """ checksum_object = dict() checksum_object["algorithm"] = ( - "checksumAlgorithm_" + checksum_field.identifier.lower() + checksum_field.identifier.upper() ) checksum_object["checksumValue"] = checksum_field.value return checksum_object @@ -138,13 +138,14 @@ def create_package_info(self, package): package_object["packageFileName"] = package.file_name if package.has_optional_field("supplier"): - package_object["supplier"] = package.supplier.__str__() + package_object["supplier"] = package.supplier.to_value() if package.has_optional_field("originator"): - package_object["originator"] = package.originator.__str__() + package_object["originator"] = package.originator.to_value() if package.has_optional_field("check_sum"): - package_object["checksums"] = [self.checksum(package.check_sum)] + package_object["checksums"] = [self.checksum(checksum) for checksum in package.checksums if checksum] + assert package.check_sum.identifier == "SHA1", "First checksum must be SHA1" package_object["sha1"] = package.check_sum.value if package.has_optional_field("description"): @@ -197,12 +198,14 @@ def create_file_info(self, package): file_object["name"] = file.name file_object["SPDXID"] = self.spdx_id(file.spdx_id) - file_object["checksums"] = [self.checksum(file.chk_sum)] + file_object["checksums"] = [self.checksum(checksum) for checksum in file.checksums if checksum] file_object["licenseConcluded"] = self.license(file.conc_lics) file_object["licenseInfoFromFiles"] = list( map(self.license, file.licenses_in_file) ) file_object["copyrightText"] = file.copyright.__str__() + + assert file.chk_sum.identifier == "SHA1", "First checksum must be SHA1" file_object["sha1"] = file.chk_sum.value if file.has_optional_field("comment"): @@ -505,3 +508,45 @@ def create_document(self): self.document_object["relationships"] = self.create_relationship_info() return {"Document": self.document_object} + + +def flatten_document(document_object): + """ + Move nested Package -> Files to top level to conform with schema. + """ + + document = document_object["Document"] + + # replace documentDescribes with SPDXID references + package_objects = document["documentDescribes"] + + document["documentDescribes"] = [package["Package"]["SPDXID"] for package in package_objects] + + document["packages"] = [package["Package"] for package in package_objects] + + file_objects = [] + + for package_info_object in document.get("packages", []): + if not "files" in package_info_object: + continue + if "sha1" in package_info_object: + del package_info_object["sha1"] + package_info_object["hasFiles"] = [file_object["File"]["SPDXID"] for file_object in package_info_object["files"]] + file_objects.extend(file_object["File"] for file_object in package_info_object.pop("files")) + + for file_object in file_objects: + file_object["fileName"] = file_object.pop("name") + if "licenseInfoFromFiles" in file_object: + file_object["licenseInfoInFiles"] = file_object.pop("licenseInfoFromFiles") + del file_object["sha1"] + + document["files"] = file_objects + + return document + + +class JsonYamlWriter(Writer): + + def create_document(self): + document_object = super().create_document() + return flatten_document(document_object) diff --git a/spdx/writers/rdf.py b/spdx/writers/rdf.py index b5bb6a458..8112b4a59 100644 --- a/spdx/writers/rdf.py +++ b/spdx/writers/rdf.py @@ -22,6 +22,7 @@ from spdx import document from spdx import config from spdx import utils +from spdx.package import Package from spdx.parsers.loggers import ErrorMessages from spdx.writers.tagvalue import InvalidDocumentError @@ -709,7 +710,7 @@ def handle_package_literal_optional(self, package, package_node, predicate, fiel triple = (package_node, predicate, value_node) self.graph.add(triple) - def handle_pkg_optional_fields(self, package, package_node): + def handle_pkg_optional_fields(self, package: Package, package_node): """ Write package optional fields. """ diff --git a/spdx/writers/xml.py b/spdx/writers/xml.py index 43e28f8cf..9e4cb9b09 100644 --- a/spdx/writers/xml.py +++ b/spdx/writers/xml.py @@ -16,6 +16,19 @@ from spdx.parsers.loggers import ErrorMessages +class XMLWriter(Writer): + def checksum(self, checksum_field): + """ + Return a dictionary representation of a spdx.checksum.Algorithm object + """ + checksum_object = dict() + checksum_object["algorithm"] = ( + "checksumAlgorithm_" + checksum_field.identifier.lower() + ) + checksum_object["checksumValue"] = checksum_field.value + return checksum_object + + def write_document(document, out, validate=True): if validate: @@ -24,7 +37,7 @@ def write_document(document, out, validate=True): if messages: raise InvalidDocumentError(messages) - writer = Writer(document) + writer = XMLWriter(document) document_object = {"SpdxDocument": writer.create_document()} xmltodict.unparse(document_object, out, encoding="utf-8", pretty=True) diff --git a/spdx/writers/yaml.py b/spdx/writers/yaml.py index 24600d8a7..6fd0135e6 100644 --- a/spdx/writers/yaml.py +++ b/spdx/writers/yaml.py @@ -12,7 +12,7 @@ import yaml from spdx.writers.tagvalue import InvalidDocumentError -from spdx.writers.jsonyamlxml import Writer +from spdx.writers.jsonyamlxml import JsonYamlWriter from spdx.parsers.loggers import ErrorMessages @@ -24,7 +24,7 @@ def write_document(document, out, validate=True): if messages: raise InvalidDocumentError(messages) - writer = Writer(document) + writer = JsonYamlWriter(document) document_object = writer.create_document() yaml.safe_dump(document_object, out, indent=2, explicit_start=True) diff --git a/tests/data/doc_write/json-simple-plus.json b/tests/data/doc_write/json-simple-plus.json index 11e0929e4..d034ecf50 100644 --- a/tests/data/doc_write/json-simple-plus.json +++ b/tests/data/doc_write/json-simple-plus.json @@ -5,10 +5,17 @@ "name": "Sample_Document-V2.1", "SPDXID": "SPDXRef-DOCUMENT", "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "creationInfo": { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + }, "documentDescribes": [ { "Package": { - "SPDXID": "SPDXRef-Package", + "SPDXID": "SPDXRef-Package", "name": "some/path", "downloadLocation": "NOASSERTION", "copyrightText": "Some copyrught", @@ -17,7 +24,7 @@ }, "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" } ], @@ -30,7 +37,7 @@ "SPDXID": "SPDXRef-File", "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" } ], diff --git a/tests/data/doc_write/json-simple-plus.new.json b/tests/data/doc_write/json-simple-plus.new.json new file mode 100644 index 000000000..a67c6e324 --- /dev/null +++ b/tests/data/doc_write/json-simple-plus.new.json @@ -0,0 +1,59 @@ +{ + "spdxVersion": "SPDX-2.1", + "dataLicense": "CC0-1.0", + "name": "Sample_Document-V2.1", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "creationInfo": { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + }, + "documentDescribes": [ + "SPDXRef-Package" + ], + "packages": [ + { + "SPDXID": "SPDXRef-Package", + "name": "some/path", + "downloadLocation": "NOASSERTION", + "copyrightText": "Some copyrught", + "packageVerificationCode": { + "packageVerificationCodeValue": "SOME code" + }, + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "SOME-SHA1" + } + ], + "licenseDeclared": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "licenseInfoFromFiles": [ + "LGPL-2.1-or-later" + ], + "hasFiles": [ + "SPDXRef-File" + ] + } + ], + "files": [ + { + "SPDXID": "SPDXRef-File", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "SOME-SHA1" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "fileName": "./some/path/tofile", + "licenseInfoInFiles": [ + "LGPL-2.1-or-later" + ] + } + ] +} \ No newline at end of file diff --git a/tests/data/doc_write/json-simple.json b/tests/data/doc_write/json-simple.json index 246a0cacb..619821b40 100644 --- a/tests/data/doc_write/json-simple.json +++ b/tests/data/doc_write/json-simple.json @@ -5,10 +5,17 @@ "name": "Sample_Document-V2.1", "SPDXID": "SPDXRef-DOCUMENT", "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "creationInfo": { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + }, "documentDescribes": [ { "Package": { - "SPDXID": "SPDXRef-Package", + "SPDXID": "SPDXRef-Package", "name": "some/path", "downloadLocation": "NOASSERTION", "copyrightText": "Some copyrught", @@ -17,7 +24,7 @@ }, "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" } ], @@ -30,7 +37,7 @@ "SPDXID": "SPDXRef-File", "checksums": [ { - "algorithm": "checksumAlgorithm_sha1", + "algorithm": "SHA1", "checksumValue": "SOME-SHA1" } ], diff --git a/tests/data/doc_write/json-simple.new.json b/tests/data/doc_write/json-simple.new.json new file mode 100644 index 000000000..187895621 --- /dev/null +++ b/tests/data/doc_write/json-simple.new.json @@ -0,0 +1,59 @@ +{ + "spdxVersion": "SPDX-2.1", + "dataLicense": "CC0-1.0", + "name": "Sample_Document-V2.1", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "creationInfo": { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + }, + "documentDescribes": [ + "SPDXRef-Package" + ], + "packages": [ + { + "SPDXID": "SPDXRef-Package", + "name": "some/path", + "downloadLocation": "NOASSERTION", + "copyrightText": "Some copyrught", + "packageVerificationCode": { + "packageVerificationCodeValue": "SOME code" + }, + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "SOME-SHA1" + } + ], + "licenseDeclared": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "licenseInfoFromFiles": [ + "LGPL-2.1-only" + ], + "hasFiles": [ + "SPDXRef-File" + ] + } + ], + "files": [ + { + "SPDXID": "SPDXRef-File", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "SOME-SHA1" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "fileName": "./some/path/tofile", + "licenseInfoInFiles": [ + "LGPL-2.1-only" + ] + } + ] +} \ No newline at end of file diff --git a/tests/data/doc_write/yaml-simple-plus.new.yaml b/tests/data/doc_write/yaml-simple-plus.new.yaml new file mode 100644 index 000000000..1c20a50c4 --- /dev/null +++ b/tests/data/doc_write/yaml-simple-plus.new.yaml @@ -0,0 +1,38 @@ +SPDXID: SPDXRef-DOCUMENT +creationInfo: + created: '2021-11-15T00:00:00Z' + creators: + - 'Organization: SPDX' + licenseListVersion: '3.6' +dataLicense: CC0-1.0 +documentDescribes: +- SPDXRef-Package +documentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 +files: +- SPDXID: SPDXRef-File + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 + copyrightText: NOASSERTION + fileName: ./some/path/tofile + licenseConcluded: NOASSERTION + licenseInfoInFiles: + - LGPL-2.1-or-later +name: Sample_Document-V2.1 +packages: +- SPDXID: SPDXRef-Package + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 + copyrightText: Some copyrught + downloadLocation: NOASSERTION + hasFiles: + - SPDXRef-File + licenseConcluded: NOASSERTION + licenseDeclared: NOASSERTION + licenseInfoFromFiles: + - LGPL-2.1-or-later + name: some/path + packageVerificationCode: + packageVerificationCodeValue: SOME code +spdxVersion: SPDX-2.1 diff --git a/tests/data/doc_write/yaml-simple-plus.yaml b/tests/data/doc_write/yaml-simple-plus.yaml index 9858c8167..2a58e6d40 100644 --- a/tests/data/doc_write/yaml-simple-plus.yaml +++ b/tests/data/doc_write/yaml-simple-plus.yaml @@ -5,6 +5,13 @@ Document: name: "Sample_Document-V2.1" SPDXID: "SPDXRef-DOCUMENT" documentNamespace: "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301" + creationInfo: { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + } documentDescribes: - Package: SPDXID: "SPDXRef-Package" @@ -14,7 +21,7 @@ Document: packageVerificationCode: packageVerificationCodeValue: "SOME code" checksums: - - algorithm: "checksumAlgorithm_sha1" + - algorithm: "SHA1" checksumValue: "SOME-SHA1" licenseDeclared: "NOASSERTION" licenseConcluded: "NOASSERTION" @@ -22,8 +29,8 @@ Document: - File: name: "./some/path/tofile" SPDXID: "SPDXRef-File" - checksums: - - algorithm: "checksumAlgorithm_sha1" + checksums: + - algorithm: "SHA1" checksumValue: "SOME-SHA1" licenseConcluded: "NOASSERTION" copyrightText: "NOASSERTION" diff --git a/tests/data/doc_write/yaml-simple.new.yaml b/tests/data/doc_write/yaml-simple.new.yaml new file mode 100644 index 000000000..9cbdc783c --- /dev/null +++ b/tests/data/doc_write/yaml-simple.new.yaml @@ -0,0 +1,38 @@ +SPDXID: SPDXRef-DOCUMENT +creationInfo: + created: '2021-11-15T00:00:00Z' + creators: + - 'Organization: SPDX' + licenseListVersion: '3.6' +dataLicense: CC0-1.0 +documentDescribes: +- SPDXRef-Package +documentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 +files: +- SPDXID: SPDXRef-File + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 + copyrightText: NOASSERTION + fileName: ./some/path/tofile + licenseConcluded: NOASSERTION + licenseInfoInFiles: + - LGPL-2.1-only +name: Sample_Document-V2.1 +packages: +- SPDXID: SPDXRef-Package + checksums: + - algorithm: SHA1 + checksumValue: SOME-SHA1 + copyrightText: Some copyrught + downloadLocation: NOASSERTION + hasFiles: + - SPDXRef-File + licenseConcluded: NOASSERTION + licenseDeclared: NOASSERTION + licenseInfoFromFiles: + - LGPL-2.1-only + name: some/path + packageVerificationCode: + packageVerificationCodeValue: SOME code +spdxVersion: SPDX-2.1 diff --git a/tests/data/doc_write/yaml-simple.yaml b/tests/data/doc_write/yaml-simple.yaml index b4946b70f..449a2cc15 100644 --- a/tests/data/doc_write/yaml-simple.yaml +++ b/tests/data/doc_write/yaml-simple.yaml @@ -5,6 +5,13 @@ Document: name: "Sample_Document-V2.1" SPDXID: "SPDXRef-DOCUMENT" documentNamespace: "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301" + creationInfo: { + "creators": [ + "Organization: SPDX" + ], + "created": "2021-11-15T00:00:00Z", + "licenseListVersion": "3.6" + } documentDescribes: - Package: SPDXID: "SPDXRef-Package" @@ -14,7 +21,7 @@ Document: packageVerificationCode: packageVerificationCodeValue: "SOME code" checksums: - - algorithm: "checksumAlgorithm_sha1" + - algorithm: "SHA1" checksumValue: "SOME-SHA1" licenseDeclared: "NOASSERTION" licenseConcluded: "NOASSERTION" @@ -22,8 +29,8 @@ Document: - File: name: "./some/path/tofile" SPDXID: "SPDXRef-File" - checksums: - - algorithm: "checksumAlgorithm_sha1" + checksums: + - algorithm: "SHA1" checksumValue: "SOME-SHA1" licenseConcluded: "NOASSERTION" copyrightText: "NOASSERTION" diff --git a/tests/data/formats/SPDXJsonExample2.2.json b/tests/data/formats/SPDXJsonExample2.2.json new file mode 100644 index 000000000..33b6f93b7 --- /dev/null +++ b/tests/data/formats/SPDXJsonExample2.2.json @@ -0,0 +1,231 @@ +{ + "comment": "This is a sample spreadsheet", + "name": "Sample_Document-V2.1", + "documentDescribes": [ + "SPDXRef-Package" + ], + "creationInfo": { + "comment": "This is an example of an SPDX spreadsheet format", + "creators": [ + "Tool: SourceAuditor-V1.2", + "Person: Gary O'Neall", + "Organization: Source Auditor Inc." + ], + "licenseListVersion": "3.6", + "created": "2010-02-03T00:00:00Z" + }, + "externalDocumentRefs": [ + { + "checksum": { + "checksumValue": "d6a770ba38583ed4bb4525bd96e50461655d2759", + "algorithm": "SHA1" + }, + "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", + "externalDocumentId": "DocumentRef-spdx-tool-2.1" + } + ], + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "annotations": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "annotationType": "REVIEW", + "SPDXID": "SPDXRef-45", + "annotationDate": "2012-06-13T00:00:00Z", + "annotator": "Person: Jim Reviewer" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-Package", + "relationshipType": "CONTAINS" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-File", + "relationshipType": "DESCRIBES" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "DocumentRef-spdx-tool-1.2:SPDXRef-ToolsElement", + "relationshipType": "COPY_OF" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-Package", + "relationshipType": "DESCRIBES" + }, + { + "spdxElementId": "SPDXRef-Package", + "relatedSpdxElement": "SPDXRef-Saxon", + "relationshipType": "DYNAMIC_LINK" + }, + { + "spdxElementId": "SPDXRef-Package", + "relatedSpdxElement": "SPDXRef-JenaLib", + "relationshipType": "CONTAINS" + }, + { + "spdxElementId": "SPDXRef-CommonsLangSrc", + "relatedSpdxElement": "NOASSERTION", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-JenaLib", + "relatedSpdxElement": "SPDXRef-Package", + "relationshipType": "CONTAINS" + }, + { + "spdxElementId": "SPDXRef-File", + "relatedSpdxElement": "SPDXRef-fromDoap-0", + "relationshipType": "GENERATED_FROM" + } + ], + "dataLicense": "CC0-1.0", + "reviewers": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "reviewer": "Person: Joe Reviewer", + "reviewDate": "2010-02-10T00:00:00Z" + }, + { + "comment": "Another example reviewer.", + "reviewer": "Person: Suzanne Reviewer", + "reviewDate": "2011-03-13T00:00:00Z" + } + ], + "hasExtractedLicensingInfos": [ + { + "extractedText": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "licenseId": "LicenseRef-2" + }, + { + "extractedText": "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "comment": "This is tye CyperNeko License", + "licenseId": "LicenseRef-3", + "name": "CyberNeko License", + "seeAlso": [ + "http://justasample.url.com", + "http://people.apache.org/~andyc/neko/LICENSE" + ] + }, + { + "extractedText": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", + "licenseId": "LicenseRef-4" + }, + { + "extractedText": "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */", + "licenseId": "LicenseRef-1" + } + ], + "spdxVersion": "SPDX-2.1", + "SPDXID": "SPDXRef-DOCUMENT", + "snippets": [ + { + "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", + "name": "from linux kernel", + "copyrightText": "Copyright 2008-2010 John Smith", + "licenseConcluded": "Apache-2.0", + "licenseInfoFromSnippet": [ + "Apache-2.0" + ], + "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "SPDXID": "SPDXRef-Snippet", + "fileId": "SPDXRef-DoapSource" + } + ], + "packages": [ + { + "SPDXID": "SPDXRef-Package", + "originator": "Organization: SPDX", + "licenseInfoFromFiles": [ + "Apache-1.0", + "LicenseRef-3", + "MPL-1.1", + "LicenseRef-2", + "LicenseRef-4", + "Apache-2.0", + "LicenseRef-1" + ], + "name": "SPDX Translator", + "packageFileName": "spdxtranslator-1.0.zip", + "licenseComments": "The declared license information can be found in the NOTICE file at the root of the archive file", + "summary": "SPDX Translator utility", + "sourceInfo": "Version 1.0 of the SPDX Translator application", + "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", + "packageVerificationCode": { + "packageVerificationCodeValue": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "packageVerificationCodeExcludedFiles": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + }, + "licenseConcluded": "(Apache-1.0 AND LicenseRef-2 AND MPL-1.1 AND LicenseRef-3 AND LicenseRef-4 AND Apache-2.0 AND LicenseRef-1)", + "supplier": "Organization: Linux Foundation", + "attributionTexts": [ + "The GNU C Library is free software. See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed. License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually." + ], + "checksums": [ + { + "checksumValue": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", + "algorithm": "SHA1" + } + ], + "versionInfo": "Version 0.9.2", + "licenseDeclared": "(LicenseRef-4 AND LicenseRef-3 AND Apache-2.0 AND LicenseRef-2 AND MPL-1.1 AND LicenseRef-1)", + "downloadLocation": "http://www.spdx.org/tools", + "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", + "hasFiles": [ + "SPDXRef-File1", + "SPDXRef-File2" + ] + } + ], + "files": [ + { + "comment": "This file belongs to Jena", + "copyrightText": "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", + "artifactOf": [ + { + "name": "Jena", + "homePage": "http://www.openjena.org/", + "projectUri": "http://subversion.apache.org/doap.rdf" + } + ], + "licenseConcluded": "LicenseRef-1", + "licenseComments": "This license is used by Jena", + "checksums": [ + { + "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", + "algorithm": "SHA1" + } + ], + "fileTypes": [ + "fileType_archive" + ], + "SPDXID": "SPDXRef-File1", + "fileName": "Jenna-2.6.3/jena-2.6.3-sources.jar", + "licenseInfoInFiles": [ + "LicenseRef-1" + ] + }, + { + "copyrightText": "Copyright 2010, 2011 Source Auditor Inc.", + "licenseConcluded": "Apache-2.0", + "checksums": [ + { + "checksumValue": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", + "algorithm": "SHA1" + } + ], + "fileTypes": [ + "fileType_source" + ], + "SPDXID": "SPDXRef-File2", + "fileName": "src/org/spdx/parser/DOAPProject.java", + "licenseInfoInFiles": [ + "Apache-2.0" + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_document.py b/tests/test_document.py index 53adad6cd..376ed6f90 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -290,7 +290,7 @@ def test_write_document_json_with_validate(self): write_document(doc, output, validate=True) expected_file = utils_test.get_test_loc( - 'doc_write/json-simple.json', + 'doc_write/json-simple.new.json', test_data_dir=utils_test.test_data_dir) utils_test.check_json_scan(expected_file, result_file, regen=False) @@ -310,7 +310,7 @@ def test_write_document_json_with_or_later_with_validate(self): write_document(doc, output, validate=True) expected_file = utils_test.get_test_loc( - 'doc_write/json-simple-plus.json', + 'doc_write/json-simple-plus.new.json', test_data_dir=utils_test.test_data_dir) utils_test.check_json_scan(expected_file, result_file, regen=False) @@ -330,7 +330,7 @@ def test_write_document_yaml_with_validate(self): write_document(doc, output, validate=True) expected_file = utils_test.get_test_loc( - 'doc_write/yaml-simple.yaml', + 'doc_write/yaml-simple.new.yaml', test_data_dir=utils_test.test_data_dir) utils_test.check_yaml_scan(expected_file, result_file, regen=False) @@ -350,7 +350,7 @@ def test_write_document_yaml_with_or_later_with_validate(self): write_document(doc, output, validate=True) expected_file = utils_test.get_test_loc( - 'doc_write/yaml-simple-plus.yaml', + 'doc_write/yaml-simple-plus.new.yaml', test_data_dir=utils_test.test_data_dir) utils_test.check_yaml_scan(expected_file, result_file, regen=False) diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index 9dceaffcb..457aad140 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -35,7 +35,9 @@ "SPDXRdfExample.rdf-yaml", "SPDXRdfExample.rdf-xml", "SPDXRdfExample.rdf-json", - "SPDXRdfExample.rdf-tag" + "SPDXRdfExample.rdf-tag", + "SPDXJsonExample2.2.json-rdf", + "SPDXJsonExample2.2.json-tag", } @pytest.mark.parametrize("out_format", ['rdf', 'yaml', 'xml', 'json', 'tag']) @@ -56,7 +58,7 @@ def test_write_anything(in_file, out_format, tmpdir): doc2, error2 = parse_anything.parse_file(out_fn) result2 = utils_test.TestParserUtils.to_dict(doc2) assert not error2 - + test = in_basename + "-" + out_format if test not in UNSTABLE_CONVERSIONS: assert result==result2 diff --git a/tests/utils_test.py b/tests/utils_test.py index 92840f108..5adcb9949 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -173,7 +173,7 @@ def load_and_clean_json(location): """ with io.open(location, encoding='utf-8') as l: content = l.read() - data = json.loads(content) + data = {'Document': json.loads(content)} if 'creationInfo' in data['Document']: del(data['Document']['creationInfo']) @@ -203,7 +203,7 @@ def load_and_clean_yaml(location): """ with io.open(location, encoding='utf-8') as l: content = l.read() - data = yaml.safe_load(content) + data = {'Document': yaml.safe_load(content)} if 'creationInfo' in data['Document']: del(data['Document']['creationInfo']) From d800302d88beba987be0a985b3bf0a341b9aa768 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Mon, 22 Nov 2021 14:41:51 -0500 Subject: [PATCH 2/2] allow package.verif_code to be 0..1 in json (spdx 2.2.1) Signed-off-by: Daniel Holth --- spdx/package.py | 2 +- spdx/writers/jsonyamlxml.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spdx/package.py b/spdx/package.py index 12036b928..f776f65e5 100644 --- a/spdx/package.py +++ b/spdx/package.py @@ -39,7 +39,7 @@ class Package(object): If set to "false", the package must not contain any files. Optional, boolean. - homepage: Optional, URL as string or NONE or NO_ASSERTION. - - verif_code: string. Mandatory if files_analyzed is True or None (omitted) + - verif_code: string. 0..1 if files_analyzed is True or None (omitted) Must be None (omitted) if files_analyzed is False - check_sum: Optional , spdx.checksum.Algorithm. - source_info: Optional string. diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index 9b06b3319..d7217f1ac 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -112,9 +112,10 @@ def create_package_info(self, package): package_object["SPDXID"] = self.spdx_id(package.spdx_id) package_object["name"] = package.name package_object["downloadLocation"] = package.download_location.__str__() - package_object["packageVerificationCode"] = self.package_verification_code( - package - ) + if package.verif_code is not None: + package_object["packageVerificationCode"] = self.package_verification_code( + package + ) package_object["licenseConcluded"] = self.license(package.conc_lics) package_object["licenseInfoFromFiles"] = list( map(self.license, package.licenses_from_files)