Skip to content

Commit

Permalink
feat: Manifest schema tests (#224)
Browse files Browse the repository at this point in the history
* manifest validity tests

* use manifest schema from go repo

* call the function correctly

* linting

* linting

* set the schema file correctly

* rename schema file

* dont require mimetype

* remove mimetype from required, set file

* use manifest schema from platform repo
  • Loading branch information
elizabethhealy authored Nov 25, 2024
1 parent 450ca7f commit 5694c3c
Show file tree
Hide file tree
Showing 7 changed files with 325 additions and 6 deletions.
1 change: 0 additions & 1 deletion .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,4 @@ jobs:
pip install black
ruff check
black --check .
pytest test_nano.py
working-directory: xtest
6 changes: 6 additions & 0 deletions .github/workflows/xtest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ jobs:
run: |-
pip install -r requirements.txt
working-directory: otdftests/xtest
# When the schema gets merged into the spec repo, we can just rely on that as a source of truth
- name: Get manifest schema from platform repo
run: |-
curl -L -o manifest.schema.json https://raw.githubusercontent.com/opentdf/platform/main/sdk/schema/manifest.schema.json
working-directory: otdftests/xtest
- name: Validate xtests
if: ${{ !inputs }}
run: |-
Expand All @@ -149,6 +154,7 @@ jobs:
working-directory: otdftests/xtest
env:
PLATFORM_DIR: '../../${{ steps.run-platform.outputs.platform-working-dir }}'
SCHEMA_FILE: 'manifest.schema.json'

######## ATTRIBUTE BASED CONFIGURATION #############
- name: Start additional kas
Expand Down
4 changes: 2 additions & 2 deletions xtest/assertions.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from pydantic import BaseModel
from typing import Literal
from typing import Literal, Union


Type = Literal["handling", "other"]
Expand All @@ -11,7 +11,7 @@
class Statement(BaseModel):
format: str
schema: str
value: str
value: Union[str, dict]


class Binding(BaseModel):
Expand Down
238 changes: 238 additions & 0 deletions xtest/manifest.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://example.com/manifest.schema.json",
"title": "manifest",
"description": "TDF manifest in JSON",
"type": "object",
"properties": {
"payload": {
"type": "object",
"description": "An object which contains information describing the payload.",
"properties": {
"type": {
"description": "Describes the type of payload is associated with the TDF.",
"type": "string"
},
"url": {
"description": "URL which points to payload. For reference types, with the default ZIP protocol, the URL would point to a local file within the zip.",
"type": "string"
},
"protocol": {
"description": "The payload protocol. Default is zip."
},
"isEncrypted": {
"description": "Designates whether or not the payload is encrypted, or cleartext.",
"type": "boolean"
},
"mimeType": {
"description": "Specifies the type of file that is encrypted. Default is `application/octet-stream`.",
"type": "string"
},
"tdf_spec_version": {
"description": "Semver version number of the TDF spec.",
"type": "string"
}
},
"required": ["type", "url", "protocol", "isEncrypted"]
},
"encryptionInformation": {
"type": "object",
"properties": {
"type": {
"description": "Designates the type of key access protocol was used. Default, is split.",
"type": "string"
},
"keyAccess": {
"description": "An array of keyAccess objects which are used to retrieve keys from one, or more Key Access Services",
"type": "array",
"items": {
"description": "A key access object",
"type": "object",
"properties": {
"type": {
"description": "The type of key access object.",
"type": "string",
"enum": ["wrapped", "remote"]
},
"url": {
"description": "A fully qualified URL pointing to a key access service responsible for managing access to the encryption keys.",
"type": "string"
},
"protocol": {
"description": "The protocol to be used for managing key access.",
"type": "string",
"enum": ["kas"]
},
"wrappedKey": {
"description": "The symmetric key used to encrypt the payload. It has been encrypted using the public key of the KAS, then base64 encoded.",
"type": "string"
},
"sid": {
"description": "A unique identifier for a single key split. In some complex policies, multiple key access objects may exist that share a specific key split. Using a splitId allows software to more efficiently operate by not reusing key material unnecessarily. ",
"type": "string"
},
"kid": {
"description": "A UUID for the specific keypair used for wrapping the symmetric key.",
"type": "string"
},
"policyBinding": {
"description": "Object describing the policyBinding. Contains a hash, and an algorithm used. May also be a string, with just the hash. In that case default to HS256.",
"oneOf": [
{
"type": "string"
},{
"type": "object",
"properties": {
"alg": {
"description": "The policy binding algorithm used to generate the hash.",
"type": "string"
},
"hash": {
"description": "This contains a keyed hash that will provide cryptographic integrity on the policy object, such that it cannot be modified or copied to another TDF, without invalidating the binding. Specifically, you would have to have access to the key in order to overwrite the policy.",
"type": "string"
}
}
,"required": ["alg", "hash"]
}
]
},
"encryptedMetadata": {
"description": "Metadata associated with the TDF, and the request. The contents of the metadata are freeform, and are used to pass information from the client, and any plugins that may be in use by the KAS. The metadata stored here should not be used for primary access decisions. Base64.",
"type": "string"
}
}
},
"required": ["type", "url", "protocol", "wrappedKey","sid", "kid", "policyBinding"]
},
"method": {
"type": "object",
"properties": {
"algorithm": {
"description": "Algorithm used to encrypt the payload",
"type": "string"
},
"isStreamable": {
"description": "Designates whether or not the payload is streamable.",
"type": "boolean"
}
},
"required": ["algorithm", "isStreamable"]
},
"integrityInformation": {
"type": "object",
"properties": {
"rootSignature": {
"type": "object",
"properties": {
"alg": {
"description": "Algorithm used to generate the root signature of the payload",
"type": "string"
},
"sig": {
"description": "The payload signature",
"type": "string"
}
}
},
"segmentSizeDefault": {
"description": "Default size of a encryption segment",
"type": "number"
},
"segmentHashAlg": {
"description": "Algorithm used to generate segment hashes",
"type": "string"
},
"segments": {
"description": "An array of segment objects. Allows for the possibility of assuring integrity over file segments, in addition to the entire payload. Useful for streaming.",
"type": "array",
"items": {
"description": "Segment object. Contains information necessary to validate integrity over a specific byte range of a payload.",
"type": "object",
"properties": {
"hash": {
"description": "Generated hash using the segment hashing algorithm specified in the parent object.",
"type": "string"
},
"segmentSize": {
"description": "The size of the segment prior to its encryption. Optional field only specified if it differs from the 'segmentSizeDefault', specified above.",
"type": "number"
},
"encryptedSegmentSize": {
"description": "The size of the segment once it has been encrypted.",
"type": "number"
}
}
}
},
"encryptedSegmentSizeDefault": {
"description": "Default size of an encrypted segment. TODO: Is this necessary??",
"type": "number"
}
},
"required": ["rootSignature", "segmentSizeDefault", "segments", "encryptedSegmentSizeDefault"]
},
"policy": {
"description": "Base64 encoded policy object",
"type": "string"
}
}
},
"assertions": {
"type": "array",
"description": "An array of objects used to express metadata about the objects in the scope attribute of the assertion. An assertion also supports metadata about the assertion statement for the purposes of indicating any handling instructions pertinent to the statement itself. Also supports encrypted statements and binding the statement with objects in its scope.",
"items": {
"type": "object",
"description": "A single assertion",
"properties": {
"id": {
"description": "A unique local identifier used for binding and signing purposes. Not guaranteed to be unique across multiple TDOs but must be unique within a single instance.",
"type": "string"
},
"type": {
"description": "Describes the type of assertion ('handling' or 'other').",
"type": "string"
},
"scope": {
"description": "An enumeration of the object to which the assertion applies ('tdo' or 'payload').",
"type": "string"
},
"appliesToState": {
"description": "Used to indicate if the statement metadata applies to 'encrypted' or 'unencrypted' data.",
"type": "string"
},
"statement": {
"description": "Intended for access, rights, and/or handling instructions that apply to the scope of the assertion.",
"type": "object",
"properties": {
"format": {
"description": "Describes the payload content encoding format ('xml-structured', 'base64binary', 'string').",
"type": "string"
},
"value": {
"description": "Payload content encoded in the format specified.",
"type": ["string", "object"]
}
}
},
"binding": {
"description": "Object describing the assertionBinding. Contains a hash, and an algorithm used.",
"type": "object",
"properties": {
"method": {
"description": "The assertion binding method used encode the signature. Default is 'jws'",
"type": "string"
},
"signature": {
"description": "This contains a keyed hash that will provide cryptographic integrity on the assertion object, such that it cannot be modified or copied to another TDF, without invalidating the binding. Specifically, you would have to have access to the key in order to overwrite the policy.",
"type": "string"
}
},
"required": ["method", "signature"]
}
},
"required": ["id", "type", "scope", "appliesToState", "statement"]
}
}
},
"required": ["payload", "encryptionInformation"]
}
1 change: 1 addition & 0 deletions xtest/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ construct-typing==0.6.2
cryptography==43.0.1
idna==3.8
iniconfig==2.0.0
jsonschema==4.23.0
opentdf==1.5.5
packaging==24.1
pluggy==1.5.0
Expand Down
32 changes: 29 additions & 3 deletions xtest/tdfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@
import os
import subprocess
import zipfile
import jsonschema

from pydantic import BaseModel
from pydantic_core import to_jsonable_python
from typing import Literal
from typing import Literal, Optional, List, Union

logger = logging.getLogger("xtest")
logging.basicConfig()
Expand Down Expand Up @@ -56,7 +57,7 @@ class KeyAccessObject(BaseModel):
url: str
protocol: str
wrappedKey: str
policyBinding: str | PolicyBinding
policyBinding: Union[str, PolicyBinding]
encryptedMetadata: str | None = None
kid: str | None = None
sid: str | None = None
Expand Down Expand Up @@ -118,7 +119,7 @@ def policy_object(self, value: Policy):
class Manifest(BaseModel):
encryptionInformation: EncryptionInformation
payload: PayloadReference
assertions: list[tdfassertions.Assertion] | None = None
assertions: Optional[List[tdfassertions.Assertion]] = []


def manifest(tdf_file: str) -> Manifest:
Expand Down Expand Up @@ -151,6 +152,31 @@ def update_manifest(
return outfile


def validate_manifest_schema(tdf_file: str):
## Unzip the tdf
tmp_dir = os.path.dirname(tdf_file)
fname = os.path.basename(tdf_file).split(".")[0]
unzipped_dir = os.path.join(tmp_dir, f"{fname}-manifest-validation-unzipped")
with zipfile.ZipFile(tdf_file, "r") as zipped:
zipped.extractall(unzipped_dir)

## Get the schema file
schema_file_path = os.getenv("SCHEMA_FILE")
if not schema_file_path:
raise ValueError("SCHEMA_FILE environment variable is not set or is empty.")
elif not os.path.isfile(schema_file_path):
raise FileNotFoundError(f"Schema file '{schema_file_path}' not found.")
with open(schema_file_path, "r") as schema_file:
schema = json.load(schema_file)

## Get the manifest file
with open(os.path.join(unzipped_dir, "0.manifest.json"), "r") as manifest_file:
manifest = json.load(manifest_file)

## Validate
jsonschema.validate(instance=manifest, schema=schema)


def encrypt(
sdk,
pt_file,
Expand Down
Loading

0 comments on commit 5694c3c

Please sign in to comment.