Skip to content

Commit

Permalink
WIP: Make format and extension provider properties
Browse files Browse the repository at this point in the history
  • Loading branch information
mferrera committed Jun 11, 2024
1 parent 2099e77 commit 7a3e1f2
Show file tree
Hide file tree
Showing 9 changed files with 105 additions and 67 deletions.
3 changes: 1 addition & 2 deletions src/fmu/dataio/aggregation.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,6 @@ def _generate_aggrd_metadata(
etemp = dataio.ExportData(config=config, name=self.name)

objectdata_provider = objectdata_provider_factory(obj=obj, dataio=etemp)
objdata = objectdata_provider.get_objectdata()

template["tracklog"] = [generate_meta_tracklog()[0].model_dump(mode="json")]
template["file"] = {
Expand All @@ -254,7 +253,7 @@ def _generate_aggrd_metadata(
}
if compute_md5:
template["file"]["checksum_md5"] = _utils.compute_md5_using_temp_file(
obj, objdata.extension
obj, objectdata_provider.extension
)

# data section
Expand Down
29 changes: 19 additions & 10 deletions src/fmu/dataio/providers/objectdata/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from fmu.dataio.datastructure.meta.content import BoundingBox2D, BoundingBox3D
from fmu.dataio.datastructure.meta.enums import FMUClassEnum
from fmu.dataio.datastructure.meta.specification import AnySpecification
from fmu.dataio.types import Efolder, Inferrable, Layout, Subtype
from fmu.dataio.types import Efolder, Inferrable, Layout

logger: Final = null_logger(__name__)

Expand All @@ -33,11 +33,8 @@

@dataclass
class DerivedObjectDescriptor:
subtype: Subtype
layout: Layout
efolder: Efolder | str
fmt: str
extension: str
table_index: list[str] | None


Expand Down Expand Up @@ -76,8 +73,6 @@ class ObjectDataProvider(Provider):
_metadata: dict = field(default_factory=dict)
name: str = field(default="")
efolder: str = field(default="")
extension: str = field(default="")
fmt: str = field(default="")
time0: datetime | None = field(default=None)
time1: datetime | None = field(default=None)

Expand All @@ -87,8 +82,6 @@ def __post_init__(self) -> None:
obj_data = self.get_objectdata()

self.name = named_stratigraphy.name
self.extension = obj_data.extension
self.fmt = obj_data.fmt
self.efolder = obj_data.efolder

if self.dataio.forcefolder:
Expand Down Expand Up @@ -254,6 +247,22 @@ def _get_timedata(self) -> Time | None:

return Time(t0=start, t1=stop)

# layout: Layout
# efolder: Efolder | str
# fmt: str
# extension: str
# table_index: list[str] | None

@property
@abstractmethod
def fmt(self) -> str:
raise NotImplementedError

@property
@abstractmethod
def extension(self) -> str:
raise NotImplementedError

@property
@abstractmethod
def classname(self) -> FMUClassEnum:
Expand All @@ -279,12 +288,12 @@ def get_metadata(self) -> AnyContent | UnsetAnyContent:
)

@staticmethod
def _validate_get_ext(fmt: str, subtype: str, validator: dict[str, V]) -> V:
def _validate_get_ext(fmt: str, validator: dict[str, V]) -> V:
"""Validate that fmt (file format) matches data and return legal extension."""
try:
return validator[fmt]
except KeyError:
raise ConfigurationError(
f"The file format {fmt} is not supported. ",
f"Valid {subtype} formats are: {list(validator.keys())}",
f"Valid formats are: {list(validator.keys())}",
)
11 changes: 8 additions & 3 deletions src/fmu/dataio/providers/objectdata/_faultroom.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ class FaultRoomSurfaceProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.surface

@property
def fmt(self) -> str:
return self.dataio.dict_fformat

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().dictionary)

def get_bbox(self) -> BoundingBox3D:
"""Derive data.bbox for FaultRoomSurface."""
logger.info("Get bbox for FaultRoomSurface")
Expand Down Expand Up @@ -53,10 +61,7 @@ def get_spec(self) -> FaultRoomSurfaceSpecification:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for FaultRoomSurface"""
return DerivedObjectDescriptor(
subtype="JSON",
layout="faultroom_triangulated",
efolder="maps",
fmt=(fmt := self.dataio.dict_fformat),
extension=self._validate_get_ext(fmt, "JSON", ValidFormats().dictionary),
table_index=None,
)
11 changes: 8 additions & 3 deletions src/fmu/dataio/providers/objectdata/_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,14 @@ class DictionaryDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.dictionary

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().dictionary)

@property
def fmt(self) -> str:
return self.dataio.dict_fformat

def get_spec(self) -> None:
"""Derive data.spec for dict."""

Expand All @@ -177,10 +185,7 @@ def get_bbox(self) -> None:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for dict."""
return DerivedObjectDescriptor(
subtype="JSON",
layout="dictionary",
efolder="dictionaries",
fmt=(fmt := self.dataio.dict_fformat),
extension=self._validate_get_ext(fmt, "JSON", ValidFormats().dictionary),
table_index=None,
)
22 changes: 16 additions & 6 deletions src/fmu/dataio/providers/objectdata/_tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ class DataFrameDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.table

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().table)

@property
def fmt(self) -> str:
return self.dataio.table_fformat

def get_spec(self) -> TableSpecification:
"""Derive data.spec for pd.DataFrame."""
logger.info("Get spec for pd.DataFrame (tables)")
Expand All @@ -79,11 +87,8 @@ def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for pd.DataFrame."""
table_index = _derive_index(self.dataio.table_index, list(self.obj.columns))
return DerivedObjectDescriptor(
subtype="DataFrame",
layout="table",
efolder="tables",
fmt=(fmt := self.dataio.table_fformat),
extension=self._validate_get_ext(fmt, "DataFrame", ValidFormats().table),
table_index=table_index,
)

Expand All @@ -96,6 +101,14 @@ class ArrowTableDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.table

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().table)

@property
def fmt(self) -> str:
return self.dataio.arrow_fformat

def get_spec(self) -> TableSpecification:
"""Derive data.spec for pyarrow.Table."""
logger.info("Get spec for pyarrow (tables)")
Expand All @@ -111,10 +124,7 @@ def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data from pyarrow.Table."""
table_index = _derive_index(self.dataio.table_index, self.obj.column_names)
return DerivedObjectDescriptor(
subtype="ArrowTable",
layout="table",
efolder="tables",
fmt=(fmt := self.dataio.arrow_fformat),
extension=self._validate_get_ext(fmt, "ArrowTable", ValidFormats().table),
table_index=table_index,
)
70 changes: 48 additions & 22 deletions src/fmu/dataio/providers/objectdata/_xtgeo.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ class RegularSurfaceDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.surface

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().surface)

@property
def fmt(self) -> str:
return self.dataio.surface_fformat

def get_spec(self) -> SurfaceSpecification:
"""Derive data.spec for xtgeo.RegularSurface."""
logger.info("Get spec for RegularSurface")
Expand Down Expand Up @@ -85,13 +93,8 @@ def get_bbox(self) -> BoundingBox2D | BoundingBox3D:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.RegularSurface."""
return DerivedObjectDescriptor(
subtype="RegularSurface",
layout="regular",
efolder="maps",
fmt=(fmt := self.dataio.surface_fformat),
extension=self._validate_get_ext(
fmt, "RegularSurface", ValidFormats().surface
),
table_index=None,
)

Expand All @@ -104,6 +107,14 @@ class PolygonsDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.polygons

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().polygons)

@property
def fmt(self) -> str:
return self.dataio.polygons_fformat

def get_spec(self) -> PolygonsSpecification:
"""Derive data.spec for xtgeo.Polygons."""
logger.info("Get spec for Polygons")
Expand Down Expand Up @@ -131,11 +142,8 @@ def get_bbox(self) -> BoundingBox3D:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.Polygons."""
return DerivedObjectDescriptor(
subtype="Polygons",
layout="unset",
efolder="polygons",
fmt=(fmt := self.dataio.polygons_fformat),
extension=self._validate_get_ext(fmt, "Polygons", ValidFormats().polygons),
table_index=None,
)

Expand All @@ -153,6 +161,14 @@ def obj_dataframe(self) -> pd.DataFrame:
def classname(self) -> FMUClassEnum:
return FMUClassEnum.points

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().points)

@property
def fmt(self) -> str:
return self.dataio.points_fformat

def get_spec(self) -> PointSpecification:
"""Derive data.spec for xtgeo.Points."""
logger.info("Get spec for Points")
Expand Down Expand Up @@ -180,11 +196,8 @@ def get_bbox(self) -> BoundingBox3D:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.Points."""
return DerivedObjectDescriptor(
subtype="Points",
layout="unset",
efolder="points",
fmt=(fmt := self.dataio.points_fformat),
extension=self._validate_get_ext(fmt, "Points", ValidFormats().points),
table_index=None,
)

Expand All @@ -197,6 +210,14 @@ class CubeDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.cube

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().cube)

@property
def fmt(self) -> str:
return self.dataio.cube_fformat

def get_spec(self) -> CubeSpecification:
"""Derive data.spec for xtgeo.Cube."""
logger.info("Get spec for Cube")
Expand Down Expand Up @@ -251,11 +272,8 @@ def get_bbox(self) -> BoundingBox3D:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.Cube."""
return DerivedObjectDescriptor(
subtype="RegularCube",
layout="regular",
efolder="cubes",
fmt=(fmt := self.dataio.cube_fformat),
extension=self._validate_get_ext(fmt, "RegularCube", ValidFormats().cube),
table_index=None,
)

Expand All @@ -268,6 +286,14 @@ class CPGridDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.cpgrid

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().grid)

@property
def fmt(self) -> str:
return self.dataio.grid_fformat

def get_spec(self) -> CPGridSpecification:
"""Derive data.spec for xtgeo.Grid."""
logger.info("Get spec for Grid geometry")
Expand Down Expand Up @@ -306,11 +332,8 @@ def get_bbox(self) -> BoundingBox3D:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.Grid."""
return DerivedObjectDescriptor(
subtype="CPGrid",
layout="cornerpoint",
efolder="grids",
fmt=(fmt := self.dataio.grid_fformat),
extension=self._validate_get_ext(fmt, "CPGrid", ValidFormats().grid),
table_index=None,
)

Expand All @@ -323,6 +346,14 @@ class CPGridPropertyDataProvider(ObjectDataProvider):
def classname(self) -> FMUClassEnum:
return FMUClassEnum.cpgrid_property

@property
def extension(self) -> str:
return self._validate_get_ext(self.fmt, ValidFormats().grid)

@property
def fmt(self) -> str:
return self.dataio.grid_fformat

def get_spec(self) -> CPGridPropertySpecification:
"""Derive data.spec for xtgeo.GridProperty."""
logger.info("Get spec for GridProperty")
Expand All @@ -339,12 +370,7 @@ def get_bbox(self) -> None:
def get_objectdata(self) -> DerivedObjectDescriptor:
"""Derive object data for xtgeo.GridProperty."""
return DerivedObjectDescriptor(
subtype="CPGridProperty",
layout="cornerpoint",
efolder="grids",
fmt=(fmt := self.dataio.grid_fformat),
extension=self._validate_get_ext(
fmt, "CPGridProperty", ValidFormats().grid
),
table_index=None,
)
12 changes: 0 additions & 12 deletions src/fmu/dataio/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,18 +66,6 @@ class PolygonsProxy(Polygons): ...
"dictionaries",
]

Subtype: TypeAlias = Literal[
"RegularSurface",
"Polygons",
"Points",
"RegularCube",
"CPGrid",
"CPGridProperty",
"DataFrame",
"JSON",
"ArrowTable",
]

Layout: TypeAlias = Literal[
"regular",
"unset",
Expand Down
Loading

0 comments on commit 7a3e1f2

Please sign in to comment.