From 9d6543a7450c030dbd5ce6b767a387c4ee0112ab Mon Sep 17 00:00:00 2001 From: mferrera Date: Tue, 2 Jul 2024 09:52:32 +0200 Subject: [PATCH] DOC: Complete `meta` docstrings --- schema/definitions/0.8.0/schema/fmu_meta.json | 146 +++++----- src/fmu/dataio/_metadata.py | 6 +- src/fmu/dataio/datastructure/meta/meta.py | 271 ++++++++++++------ tests/test_units/test_metadata_class.py | 4 +- 4 files changed, 267 insertions(+), 160 deletions(-) diff --git a/schema/definitions/0.8.0/schema/fmu_meta.json b/schema/definitions/0.8.0/schema/fmu_meta.json index 4961764c8..0a07e878d 100644 --- a/schema/definitions/0.8.0/schema/fmu_meta.json +++ b/schema/definitions/0.8.0/schema/fmu_meta.json @@ -41,6 +41,7 @@ ], "$defs": { "Access": { + "description": "The ``access`` block contains information related to access control for\nthis data object.", "properties": { "asset": { "$ref": "#/$defs/Asset" @@ -64,9 +65,9 @@ "type": "object" }, "Aggregation": { + "description": "The ``fmu.aggregation`` block contains information about an aggregation\nperformed over an ensemble.", "properties": { "id": { - "description": "The ID of this aggregation", "examples": [ "15ce3b84-766f-4c93-9050-b154861f9100" ], @@ -75,7 +76,6 @@ "type": "string" }, "operation": { - "description": "The aggregation performed", "title": "Operation", "type": "string" }, @@ -88,11 +88,9 @@ "type": "null" } ], - "default": null, - "description": "Parameters for this realization" + "default": null }, "realization_ids": { - "description": "Array of realization ids included in this aggregation", "items": { "type": "integer" }, @@ -204,6 +202,7 @@ "title": "AnyContent" }, "Asset": { + "description": "A block containing information about the owner asset of these data.", "properties": { "name": { "examples": [ @@ -494,7 +493,7 @@ "type": "string" }, "Context": { - "description": "The internal FMU context in which this data object was produced", + "description": "The ``fmu.context`` block contains the FMU context in which this data object\nwas produced.", "properties": { "stage": { "$ref": "#/$defs/FmuContext" @@ -981,6 +980,7 @@ "type": "object" }, "Display": { + "description": "The ``display`` block contains information related to how this data object\nshould/could be displayed. As a general rule, the consumer of data is responsible\nfor figuring out how a specific data object shall be displayed. However, we use\nthis block to communicate preferences from the data producers perspective.\n\nWe also maintain this block due to legacy reasons. No data filtering logic should\nbe placed on the ``display`` block.", "properties": { "name": { "anyOf": [ @@ -2725,7 +2725,7 @@ "type": "object" }, "File": { - "description": "Block describing the file as the data appear in FMU context", + "description": "The ``file`` block contains references to this data object as a file on a disk.\nA filename in this context can be actual, or abstract. Particularly the\n``relative_path`` is, and will most likely remain, an important identifier for\nindividual file objects within an FMU case - irrespective of the existance of an\nactual file system. For this reason, the ``relative_path`` - as well as the\n``checksum_md5`` will be generated even if a file is not saved to disk. The\n``absolute_path`` will only be generated in the case of actually creating a file on\ndisk and is not required under this schema.", "properties": { "absolute_path": { "anyOf": [ @@ -2737,7 +2737,6 @@ } ], "default": null, - "description": "The absolute file path", "examples": [ "/abs/path/share/results/maps/volantis_gp_base--depth.gri" ], @@ -2764,14 +2763,12 @@ "type": "null" } ], - "description": "md5 checksum of the file or bytestring", "examples": [ "kjhsdfvsdlfk23knerknvk23" ], "title": "Checksum Md5" }, "relative_path": { - "description": "The file path relative to RUNPATH", "examples": [ "share/results/maps/volantis_gp_base--depth.gri" ], @@ -2800,7 +2797,6 @@ } ], "default": null, - "description": "Size of file object in bytes", "title": "Size Bytes" } }, @@ -4279,6 +4275,55 @@ "title": "ObjectMetadata", "type": "object" }, + "OperatingSystem": { + "description": "The ``operating_system`` block contains information about the OS on which the\nensemble was run.", + "properties": { + "hostname": { + "examples": [ + "st-123.equinor.com" + ], + "title": "Hostname", + "type": "string" + }, + "operating_system": { + "examples": [ + "Darwin-18.7.0-x86_64-i386-64bit" + ], + "title": "Operating System", + "type": "string" + }, + "release": { + "examples": [ + "18.7.0" + ], + "title": "Release", + "type": "string" + }, + "system": { + "examples": [ + "GNU/Linux" + ], + "title": "System", + "type": "string" + }, + "version": { + "examples": [ + "#1 SMP Tue Aug 27 21:37:59 PDT 2019" + ], + "title": "Version", + "type": "string" + } + }, + "required": [ + "hostname", + "operating_system", + "release", + "system", + "version" + ], + "title": "OperatingSystem", + "type": "object" + }, "PVTContent": { "properties": { "alias": { @@ -4589,6 +4634,7 @@ } ] }, + "description": "The ``parameters`` block contains the parameters used in a realization. It is a\ndirect pass of ``parameters.txt`` and will contain key:value pairs representing the\nparameters.", "title": "Parameters", "type": "object" }, @@ -6879,7 +6925,7 @@ "type": "object" }, "Ssdl": { - "description": "Sub-Surface Data Lake", + "description": "A block containing information related to SSDL. Note that this is kept due to\nlegacy.", "properties": { "access_level": { "$ref": "#/$defs/Classification" @@ -6897,6 +6943,7 @@ "type": "object" }, "SsdlAccess": { + "description": "The ``access`` block contains information related to access control for\nthis data object, with legacy SSDL settings..", "properties": { "asset": { "$ref": "#/$defs/Asset" @@ -7309,11 +7356,12 @@ "type": "object" }, "SystemInformation": { + "description": "The ```sysinfo`` block contains information about the system upon which these\ndata were exported from.", "properties": { "fmu-dataio": { "anyOf": [ { - "$ref": "#/$defs/VersionInformation" + "$ref": "#/$defs/Version" }, { "type": "null" @@ -7327,7 +7375,7 @@ "komodo": { "anyOf": [ { - "$ref": "#/$defs/VersionInformation" + "$ref": "#/$defs/Version" }, { "type": "null" @@ -7341,7 +7389,7 @@ "operating_system": { "anyOf": [ { - "$ref": "#/$defs/SystemInformationOperatingSystem" + "$ref": "#/$defs/OperatingSystem" }, { "type": "null" @@ -7353,60 +7401,6 @@ "title": "SystemInformation", "type": "object" }, - "SystemInformationOperatingSystem": { - "description": "This model encapsulates various pieces of system-related information using Python's\nplatform module. It provides a structured way to access details about the system's\nhardware, operating system, and interpreter version information.", - "properties": { - "hostname": { - "description": "The network name of the computer, possibly not fully qualified.", - "examples": [ - "Johns-MacBook-Pro.local" - ], - "title": "Hostname", - "type": "string" - }, - "operating_system": { - "description": "A detailed string identifying the underlying platform with as much useful information as possible.", - "examples": [ - "Darwin-18.7.0-x86_64-i386-64bit" - ], - "title": "Platform", - "type": "string" - }, - "release": { - "description": "The system's release version, such as a version number or a name.", - "examples": [ - "18.7.0" - ], - "title": "Release", - "type": "string" - }, - "system": { - "description": "The name of the operating system.", - "examples": [ - "Darwin" - ], - "title": "System", - "type": "string" - }, - "version": { - "description": "The specific release version of the system.", - "examples": [ - "#1 SMP Tue Aug 27 21:37:59 PDT 2019" - ], - "title": "Version", - "type": "string" - } - }, - "required": [ - "hostname", - "operating_system", - "release", - "system", - "version" - ], - "title": "SystemInformationOperatingSystem", - "type": "object" - }, "TableSpecification": { "description": "Specifies relevant values describing a generic tabular data object.", "properties": { @@ -8385,13 +8379,14 @@ "type": "object" }, "User": { + "description": "The ``user`` block holds information about the user.", "properties": { "id": { "examples": [ "peesv", - "jlov" + "jriv" ], - "title": "User ID", + "title": "Id", "type": "string" } }, @@ -8694,7 +8689,8 @@ "title": "VelocityContent", "type": "object" }, - "VersionInformation": { + "Version": { + "description": "A generic block that contains a string representing the version of\nsomething.", "properties": { "version": { "title": "Version", @@ -8704,7 +8700,7 @@ "required": [ "version" ], - "title": "VersionInformation", + "title": "Version", "type": "object" }, "VolumesContent": { @@ -9294,9 +9290,9 @@ "type": "object" }, "Workflow": { + "description": "The ``fmu.workflow`` block refers to specific subworkflows within the large\nFMU workflow being ran. This has not been standardized, mainly due to the lack of\nprogrammatic access to the workflows being run in important software within FMU.\n\n.. note:: A key usage of ``fmu.workflow.reference`` is related to ensuring\n uniqueness of data objects.", "properties": { "reference": { - "description": "Reference to the part of the FMU workflow that produced this", "title": "Reference", "type": "string" } @@ -9341,4 +9337,4 @@ } }, "title": "Root" -} +} \ No newline at end of file diff --git a/src/fmu/dataio/_metadata.py b/src/fmu/dataio/_metadata.py index 9c0302539..5fa215e1f 100644 --- a/src/fmu/dataio/_metadata.py +++ b/src/fmu/dataio/_metadata.py @@ -42,11 +42,11 @@ def generate_meta_tracklog( event=event, user=meta.User.model_construct(id=getpass.getuser()), sysinfo=meta.SystemInformation.model_construct( - fmu_dataio=meta.VersionInformation.model_construct(version=__version__), - komodo=meta.VersionInformation.model_construct(version=kr) + fmu_dataio=meta.Version.model_construct(version=__version__), + komodo=meta.Version.model_construct(version=kr) if (kr := os.environ.get("KOMODO_RELEASE")) else None, - operating_system=meta.SystemInformationOperatingSystem.model_construct( + operating_system=meta.OperatingSystem.model_construct( hostname=platform.node(), operating_system=platform.platform(), release=platform.release(), diff --git a/src/fmu/dataio/datastructure/meta/meta.py b/src/fmu/dataio/datastructure/meta/meta.py index f45a88bc9..2852d7a2a 100644 --- a/src/fmu/dataio/datastructure/meta/meta.py +++ b/src/fmu/dataio/datastructure/meta/meta.py @@ -23,52 +23,83 @@ class Asset(BaseModel): + """A block containing information about the owner asset of these data.""" + name: str = Field(examples=["Drogon"]) + """A string referring to a known asset name.""" class Ssdl(BaseModel): """ - Sub-Surface Data Lake + A block containing information related to SSDL. Note that this is kept due to + legacy. """ access_level: enums.Classification + """The SSDL access level. See :class:`enums.Classification`.""" + rep_include: bool + """Flag if this data is to be shown in REP or not.""" class Access(BaseModel): + """ + The ``access`` block contains information related to access control for + this data object. + """ + asset: Asset + """A block containing information about the owner asset of these data. + See :class:`Asset`.""" + classification: Optional[enums.Classification] = Field(default=None) + """The access classification level. See :class:`enums.Classification`.""" class SsdlAccess(Access): + """ + The ``access`` block contains information related to access control for + this data object, with legacy SSDL settings.. + """ + ssdl: Ssdl + """A block containing information related to SSDL. See :class:`Ssdl`.""" class File(BaseModel): """ - Block describing the file as the data appear in FMU context + The ``file`` block contains references to this data object as a file on a disk. + A filename in this context can be actual, or abstract. Particularly the + ``relative_path`` is, and will most likely remain, an important identifier for + individual file objects within an FMU case - irrespective of the existance of an + actual file system. For this reason, the ``relative_path`` - as well as the + ``checksum_md5`` will be generated even if a file is not saved to disk. The + ``absolute_path`` will only be generated in the case of actually creating a file on + disk and is not required under this schema. """ absolute_path: Optional[Path] = Field( default=None, - description="The absolute file path", examples=["/abs/path/share/results/maps/volantis_gp_base--depth.gri"], ) + """The absolute path of a file, e.g. /scratch/field/user/case/etc.""" + relative_path: Path = Field( - description="The file path relative to RUNPATH", examples=["share/results/maps/volantis_gp_base--depth.gri"], ) - checksum_md5: Optional[str] = Field( - description="md5 checksum of the file or bytestring", - examples=["kjhsdfvsdlfk23knerknvk23"], - ) - size_bytes: Optional[int] = Field( - default=None, - description="Size of file object in bytes", - ) + """The path of a file relative to the case root.""" + + checksum_md5: Optional[str] = Field(examples=["kjhsdfvsdlfk23knerknvk23"]) + """A valid MD5 checksum of the file.""" + + size_bytes: Optional[int] = Field(default=None) + """Size of file object in bytes""" relative_path_symlink: Optional[Path] = Field(default=None) + """The path to a symlink of the relative path.""" + absolute_path_symlink: Optional[Path] = Field(default=None) + """The path to a symlink of the absolute path.""" @model_validator(mode="before") @classmethod @@ -81,41 +112,59 @@ def _check_for_non_ascii_in_path(cls, values: Dict) -> Dict: class Parameters(RootModel): + """ + The ``parameters`` block contains the parameters used in a realization. It is a + direct pass of ``parameters.txt`` and will contain key:value pairs representing the + parameters. + """ + root: Dict[str, Union[Parameters, int, float, str]] + """A dictionary representing parameters as-is from parameters.txt.""" class Aggregation(BaseModel): - id: UUID = Field( - description="The ID of this aggregation", - examples=["15ce3b84-766f-4c93-9050-b154861f9100"], - ) - operation: str = Field( - description="The aggregation performed", - ) - realization_ids: List[int] = Field( - description="Array of realization ids included in this aggregation" - ) - parameters: Optional[Parameters] = Field( - default=None, - description="Parameters for this realization", - ) + """ + The ``fmu.aggregation`` block contains information about an aggregation + performed over an ensemble. + """ + + id: UUID = Field(examples=["15ce3b84-766f-4c93-9050-b154861f9100"]) + """The unique identifier of an aggregation.""" + + operation: str + """A string representing the type of aggregation performed.""" + + realization_ids: List[int] + """An array of realization ids included in this aggregation.""" + + parameters: Optional[Parameters] = Field(default=None) + """Parameters for this realization. See :class:`Parameters`.""" class Workflow(BaseModel): - reference: str = Field( - description="Reference to the part of the FMU workflow that produced this" - ) + """ + The ``fmu.workflow`` block refers to specific subworkflows within the large + FMU workflow being ran. This has not been standardized, mainly due to the lack of + programmatic access to the workflows being run in important software within FMU. + + .. note:: A key usage of ``fmu.workflow.reference`` is related to ensuring + uniqueness of data objects. + """ + + reference: str + """A string referring to which workflow this data object was exported by.""" class User(BaseModel): - id: str = Field( - examples=["peesv", "jlov"], - title="User ID", - ) + """The ``user`` block holds information about the user.""" + + id: str = Field(examples=["peesv", "jriv"]) + """A user identity reference..""" class Case(BaseModel): - """The ``fmu.case`` block contains information about the case from which this data + """ + The ``fmu.case`` block contains information about the case from which this data object was exported. A case represent a set of iterations that belong together, either by being part of @@ -141,8 +190,10 @@ class Case(BaseModel): class Iteration(BaseModel): - """The ``fmu.iteration`` block contains information about the iteration this data - object belongs to.""" + """ + The ``fmu.iteration`` block contains information about the iteration this data + object belongs to. + """ id: Optional[int] = Field(default=None) """The internal identification of this iteration, typically represented by an @@ -184,6 +235,11 @@ class Model(BaseModel): class RealizationJobListing(BaseModel): + """ + Directly pass "jobs.json". Temporarily deactivated in fmu-dataio pending + further alignment with ERT. + """ + arg_types: List[str] argList: List[Path] error_file: Optional[Path] @@ -202,8 +258,10 @@ class RealizationJobListing(BaseModel): class Realization(BaseModel): - """The ``fmu.realization`` block contains information about the realization this - data object belongs to.""" + """ + The ``fmu.realization`` block contains information about the realization this + data object belongs to. + """ id: int """The internal ID of the realization, typically represented by an integer.""" @@ -310,64 +368,60 @@ class Masterdata(BaseModel): See :class:`Smda`.""" -class VersionInformation(BaseModel): +class Version(BaseModel): + """ + A generic block that contains a string representing the version of + something. + """ + version: str + """A string representing the version.""" -class SystemInformationOperatingSystem(BaseModel): +class OperatingSystem(BaseModel): """ - This model encapsulates various pieces of system-related information using Python's - platform module. It provides a structured way to access details about the system's - hardware, operating system, and interpreter version information. + The ``operating_system`` block contains information about the OS on which the + ensemble was run. """ - hostname: str = Field( - title="Hostname", - description="The network name of the computer, possibly not fully qualified.", - examples=["Johns-MacBook-Pro.local"], - ) + hostname: str = Field(examples=["st-123.equinor.com"]) + """A string containing the network name of the machine.""" - operating_system: str = Field( - title="Platform", - description=( - "A detailed string identifying the underlying platform " - "with as much useful information as possible." - ), - examples=["Darwin-18.7.0-x86_64-i386-64bit"], - ) + operating_system: str = Field(examples=["Darwin-18.7.0-x86_64-i386-64bit"]) + """A string containing the name of the operating system implementation.""" - release: str = Field( - title="Release", - description="The system's release version, such as a version number or a name.", - examples=["18.7.0"], - ) + release: str = Field(examples=["18.7.0"]) + """A string containing the level of the operating system.""" - system: str = Field( - title="System", - description="The name of the operating system.", - examples=["Darwin"], - ) + system: str = Field(examples=["GNU/Linux"]) + """A string containing the name of the operating system kernel.""" - version: str = Field( - title="Version", - description="The specific release version of the system.", - examples=["#1 SMP Tue Aug 27 21:37:59 PDT 2019"], - ) + version: str = Field(examples=["#1 SMP Tue Aug 27 21:37:59 PDT 2019"]) + """The specific release version of the system.""" class SystemInformation(BaseModel): - fmu_dataio: Optional[VersionInformation] = Field( + """ + The ```sysinfo`` block contains information about the system upon which these + data were exported from. + """ + + fmu_dataio: Optional[Version] = Field( alias="fmu-dataio", default=None, examples=["1.2.3"], ) - komodo: Optional[VersionInformation] = Field( + """The version of fmu-dataio used to export the data. See :class:`Version`.""" + + komodo: Optional[Version] = Field( default=None, examples=["2023.12.05-py38"], ) - operating_system: Optional[SystemInformationOperatingSystem] = Field( - default=None, - ) + """The version of Komodo in which the the ensemble was run from.""" + + operating_system: Optional[OperatingSystem] = Field(default=None) + """The operating system from which the ensemble was started from. + See :class:`OperatingSystem`.""" class TracklogEvent(BaseModel): @@ -382,23 +436,45 @@ class TracklogEvent(BaseModel): datetime: Union[NaiveDatetime, AwareDatetime] = Field( examples=["2020-10-28T14:28:02"], ) - event: str = Field( - examples=["created", "updated", "merged"], - ) + """A datetime representation recording when the event occurred.""" + + event: str = Field(examples=["created", "updated", "merged"]) + """A string containing a reference to the type of event being logged.""" + user: User + """The user who caused the event to happen. See :class:`User`.""" + sysinfo: Optional[SystemInformation] = Field( default_factory=SystemInformation, ) + """Information about the system on which the event occurred. + See :class:`SystemInformation`.""" class Display(BaseModel): + """ + The ``display`` block contains information related to how this data object + should/could be displayed. As a general rule, the consumer of data is responsible + for figuring out how a specific data object shall be displayed. However, we use + this block to communicate preferences from the data producers perspective. + + We also maintain this block due to legacy reasons. No data filtering logic should + be placed on the ``display`` block. + """ + name: Optional[str] = Field(default=None) + """A display-friendly version of ``data.name``.""" class Context(BaseModel): - """The internal FMU context in which this data object was produced""" + """ + The ``fmu.context`` block contains the FMU context in which this data object + was produced. + """ stage: enums.FmuContext + """The stage of an FMU experiment in which this data was produced. + See :class:`enums.FmuContext`.""" class FMUCaseAttributes(BaseModel): @@ -409,7 +485,12 @@ class FMUCaseAttributes(BaseModel): """ case: Case + """The ``fmu.case`` block contains information about the case from which this data + object was exported. See :class:`Case`.""" + model: Model + """The ``fmu.model`` block contains information about the model used. + See :class:`Model`.""" class FMUAttributes(FMUCaseAttributes): @@ -420,10 +501,24 @@ class FMUAttributes(FMUCaseAttributes): """ context: Context + """The ``fmu.context`` block contains the FMU context in which this data object + was produced. See :class:`Context`. """ + iteration: Optional[Iteration] = Field(default=None) + """The ``fmu.iteration`` block contains information about the iteration this data + object belongs to. See :class:`Iteration`. """ + workflow: Optional[Workflow] = Field(default=None) + """The ``fmu.workflow`` block refers to specific subworkflows within the large + FMU workflow being ran. See :class:`Workflow`.""" + aggregation: Optional[Aggregation] = Field(default=None) + """The ``fmu.aggregation`` block contains information about an aggregation + performed over an ensemble. See :class:`Aggregation`.""" + realization: Optional[Realization] = Field(default=None) + """The ``fmu.realization`` block contains information about the realization this + data object belongs to. See :class:`Realization`.""" @model_validator(mode="before") @classmethod @@ -490,12 +585,15 @@ class CaseMetadata(MetadataBase): alias="class", title="metadata_class", ) + """The class of this metadata object. In this case, always an FMU case.""" fmu: FMUCaseAttributes """The ``fmu`` block contains all attributes specific to FMU. See :class:`FMUCaseAttributes`.""" access: Access + """The ``access`` block contains information related to access control for + this data object. See :class:`Access`.""" class ObjectMetadata(MetadataBase): @@ -515,15 +613,28 @@ class ObjectMetadata(MetadataBase): alias="class", title="metadata_class", ) + """The class of the data object being exported and described by the metadata + contained herein.""" fmu: FMUAttributes """The ``fmu`` block contains all attributes specific to FMU. See :class:`FMUAttributes`.""" access: SsdlAccess + """The ``access`` block contains information related to access control for + this data object. See :class:`SsdlAccess`.""" + data: content.AnyContent + """The ``data`` block contains information about the data contains in this + object. See :class:`content.AnyContent`.""" + file: File + """ The ``file`` block contains references to this data object as a file on a disk. + See :class:`File`.""" + display: Display + """ The ``display`` block contains information related to how this data object + should/could be displayed. See :class:`Display`.""" class Root( diff --git a/tests/test_units/test_metadata_class.py b/tests/test_units/test_metadata_class.py index 10ccb14cf..be9fb3216 100644 --- a/tests/test_units/test_metadata_class.py +++ b/tests/test_units/test_metadata_class.py @@ -13,7 +13,7 @@ ) from fmu.dataio._utils import prettyprint_dict from fmu.dataio.datastructure.meta.meta import ( - SystemInformationOperatingSystem, + OperatingSystem, TracklogEvent, ) from fmu.dataio.providers.objectdata._provider import objectdata_provider_factory @@ -95,7 +95,7 @@ def test_generate_meta_tracklog_operating_system(edataobj1, regsurf): parsed = TracklogEvent.model_validate(tracklog[0]) assert isinstance( parsed.sysinfo.operating_system, - SystemInformationOperatingSystem, + OperatingSystem, )