From 64401a2434dd63c235e61d5786875cc025e1f646 Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Wed, 5 Oct 2022 15:13:44 +0200 Subject: [PATCH 01/10] entries.location nullable + db revision --- metacatalog/__version__.py | 2 +- metacatalog/db/revisions/__init__.py | 2 ++ metacatalog/db/revisions/rev10.py | 40 ++++++++++++++++++++++++++++ metacatalog/models/entry.py | 7 ++++- 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 metacatalog/db/revisions/rev10.py diff --git a/metacatalog/__version__.py b/metacatalog/__version__.py index 45869b62..ed7d50e7 100644 --- a/metacatalog/__version__.py +++ b/metacatalog/__version__.py @@ -1 +1 @@ -__version__ = '0.5.2' +__version__ = '0.5.3' diff --git a/metacatalog/db/revisions/__init__.py b/metacatalog/db/revisions/__init__.py index a8451338..79da9c6c 100644 --- a/metacatalog/db/revisions/__init__.py +++ b/metacatalog/db/revisions/__init__.py @@ -9,6 +9,7 @@ rev7, rev8, rev9, + rev10, ) revisions = { @@ -22,4 +23,5 @@ 7: rev7, 8: rev8, 9: rev9, + 10: rev10, } diff --git a/metacatalog/db/revisions/rev10.py b/metacatalog/db/revisions/rev10.py new file mode 100644 index 00000000..2d6f0cec --- /dev/null +++ b/metacatalog/db/revisions/rev10.py @@ -0,0 +1,40 @@ +""" +Metacatalog database revision +----------------------------- +date: 2022-10-05T14:41:32.055433 + +revision #10 + +Make column entries.location nullable (raster data do not have a POINT location). + +""" +from sqlalchemy.orm import Session +from metacatalog import api, models + +UPGRADE_SQL = """ +-- entries.location nullable +ALTER TABLE entries ALTER COLUMN location DROP NOT NULL; +COMMIT; +""" + +DOWNGRADE_SQL = """ +-- entries.location not nullable +ALTER TABLE entries ALTER COLUMN location SET NOT NULL; +COMMIT; +""" + +# define the upgrade function +def upgrade(session: Session): + print('run update') + # make entries.location nullable + with session.bind.connect() as con: + con.execute(UPGRADE_SQL) + + +# define the downgrade function +def downgrade(session: Session): + print('run downgrade') + # make entries.location not nullable + with session.bind.connect() as con: + con.execute(DOWNGRADE_SQL) + diff --git a/metacatalog/models/entry.py b/metacatalog/models/entry.py index 5948c731..e30e6d79 100644 --- a/metacatalog/models/entry.py +++ b/metacatalog/models/entry.py @@ -68,6 +68,11 @@ class Entry(Base): Usually an unque ID field of other data-storage solutions. The exernal_id is only stored for reference reasons. location : str, tuple + .. versionchanged:: 0.5.3 + location is now nullable, as spatial / raster data cannot be represented + as POINT location. Please add the spatial representation of the data to + `Datasource.spatial_scale`. + The location as a POINT Geometry in unprojected WGS84 (EPSG: 4326). The location is primarily used to show all Entry objects on a map, or perform geo-searches. If the data-source needs to store more complex @@ -165,7 +170,7 @@ class Entry(Base): title = Column(String(512), nullable=False) abstract = Column(String) external_id = Column(String) - location = Column(Geometry(geometry_type='POINT', srid=4326), nullable=False) + location = Column(Geometry(geometry_type='POINT', srid=4326), nullable=True) geom = Column(Geometry) version = Column(Integer, default=1, nullable=False) latest_version_id = Column(Integer, ForeignKey('entries.id'), nullable=True) From 5c2b35ac4c033d07dd66dd2c7cf132f4c5ecb19d Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Thu, 6 Oct 2022 10:00:22 +0200 Subject: [PATCH 02/10] revision, downgrade sql --- metacatalog/db/revisions/rev10.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metacatalog/db/revisions/rev10.py b/metacatalog/db/revisions/rev10.py index 2d6f0cec..180cddf5 100644 --- a/metacatalog/db/revisions/rev10.py +++ b/metacatalog/db/revisions/rev10.py @@ -18,6 +18,9 @@ """ DOWNGRADE_SQL = """ +-- replace eventually existing NULL values with (POINT 0 0) +UPDATE entries SET location = 'POINT (0 0)' WHERE location IS NULL; +COMMIT; -- entries.location not nullable ALTER TABLE entries ALTER COLUMN location SET NOT NULL; COMMIT; From f4a6ff6dc68ebbb34163c1f8b85b005e31686e8d Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Thu, 6 Oct 2022 10:08:06 +0200 Subject: [PATCH 03/10] revision downgrade --- metacatalog/db/revisions/rev10.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metacatalog/db/revisions/rev10.py b/metacatalog/db/revisions/rev10.py index 180cddf5..702ca4ab 100644 --- a/metacatalog/db/revisions/rev10.py +++ b/metacatalog/db/revisions/rev10.py @@ -19,7 +19,7 @@ DOWNGRADE_SQL = """ -- replace eventually existing NULL values with (POINT 0 0) -UPDATE entries SET location = 'POINT (0 0)' WHERE location IS NULL; +UPDATE entries SET location = 'SRID=4326; POINT (0 0)' WHERE location IS NULL; COMMIT; -- entries.location not nullable ALTER TABLE entries ALTER COLUMN location SET NOT NULL; From 109faab92a08ebbe405e9d9784e234c560ae356b Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Mon, 10 Oct 2022 15:33:02 +0200 Subject: [PATCH 04/10] location documentation --- metacatalog/api/add.py | 9 ++++++++- metacatalog/models/datasource.py | 9 ++++++--- metacatalog/models/entry.py | 8 +++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/metacatalog/api/add.py b/metacatalog/api/add.py index 4147173b..2a366ad3 100644 --- a/metacatalog/api/add.py +++ b/metacatalog/api/add.py @@ -505,7 +505,14 @@ def add_entry(session, title, author, location, variable, abstract=None, externa database and can be found by exact match on id (int) or last_name (str). location : str, tuple Can be either a WKT of a EPSG:4326 location, or the coordinates as a - tuple. It has to be (X,Y), to (longitude, latitude) + tuple. It has to be (X,Y), to (longitude, latitude). + + .. versionchanged:: 0.5.3 + A POINT location should be specified here if there is a physical measurement + point that is different from the centroid of the spatial extent (e.g., + discharge measurement with the extent of the catchment). + Otherwise, ``Datasource.spatial_scale.extent`` should be used to specify the + location of the measured data. variable : int, str **Full** variable name (str) or ID (int) of the data described by the Entry. abstract : str diff --git a/metacatalog/models/datasource.py b/metacatalog/models/datasource.py index f763e0a7..cf5d3682 100644 --- a/metacatalog/models/datasource.py +++ b/metacatalog/models/datasource.py @@ -337,9 +337,12 @@ class SpatialScale(Base): cell size, which only applies to gridded datasets. Use the :attr:`resolution_str` property for a string representation extent : geoalchemy2.Geometry - The spatial extent of the dataset is given as a ``'POLYGON'``. While - metacatalog is capable of storing any kind of valid POLYGON as extent, - it is best practice to allow only Bounding Boxes on upload. + The spatial extent of the dataset is given as a ``'POLYGON'``. + .. versionchanged:: 0.5.3 + From this ``POLYGON``, a bounding box and the centroid are internally + calculated. + To specify a point location here, use the same value for easting and + westing and the same value for northing and southing. support : float The support gives the spatial validity for a single observation. It specifies the spatial extent at which an observed value is valid. diff --git a/metacatalog/models/entry.py b/metacatalog/models/entry.py index e30e6d79..67011af8 100644 --- a/metacatalog/models/entry.py +++ b/metacatalog/models/entry.py @@ -69,9 +69,11 @@ class Entry(Base): exernal_id is only stored for reference reasons. location : str, tuple .. versionchanged:: 0.5.3 - location is now nullable, as spatial / raster data cannot be represented - as POINT location. Please add the spatial representation of the data to - `Datasource.spatial_scale`. + A POINT location should be specified here if there is a physical measurement + point that is different from the centroid of the spatial extent (e.g., + discharge measurement with the extent of the catchment). + Otherwise, ``Datasource.spatial_scale.extent`` should be used to specify the + location of the measured data. The location as a POINT Geometry in unprojected WGS84 (EPSG: 4326). The location is primarily used to show all Entry objects on a map, or From 60e559b220022a8b229be30df98ceb104b8c8a1b Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Mon, 10 Oct 2022 15:33:35 +0200 Subject: [PATCH 05/10] View of location parameters of an entry --- metacatalog/db/views/locations.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 metacatalog/db/views/locations.sql diff --git a/metacatalog/db/views/locations.sql b/metacatalog/db/views/locations.sql new file mode 100644 index 00000000..a2ed2d69 --- /dev/null +++ b/metacatalog/db/views/locations.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW locations AS +SELECT entries.id, + entries.location, + spatial_scales.extent, + ST_Envelope(spatial_scales.extent) "bbox", + ST_Centroid(spatial_scales.extent) "centroid", + ST_Area(spatial_scales.extent) * 0.3048 ^ 2 "area_sqm" +FROM entries +INNER JOIN datasources ON entries.datasource_id = datasources.id +INNER JOIN spatial_scales ON spatial_scales.id = datasources.spatial_scale_id \ No newline at end of file From 22f68114a0a31bb537f1a0bba9ca12ec8d34acbd Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Tue, 11 Oct 2022 14:44:29 +0200 Subject: [PATCH 06/10] always commit scales, raise error if scale exists --- metacatalog/models/datasource.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/metacatalog/models/datasource.py b/metacatalog/models/datasource.py index cf5d3682..ea3705ad 100644 --- a/metacatalog/models/datasource.py +++ b/metacatalog/models/datasource.py @@ -13,6 +13,7 @@ from metacatalog.db.base import Base from metacatalog.ext import extension +from metacatalog.util.exceptions import MetadataMissingError class DataSourceType(Base): @@ -610,8 +611,12 @@ def create_scale(self, resolution, extent, support, scale_dimension): # get the correct class if scale_dimension.lower() == 'temporal': Cls = TemporalScale + if self.temporal_scale is not None: + raise MetadataMissingError('Temporal scale already exists. You can edit that one.') elif scale_dimension.lower() == 'spatial': Cls = SpatialScale + if self.spatial_scale is not None: + raise MetadataMissingError('Spatial scale already exists. You can edit that one.') else: raise AttributeError("scale_dimension has to be in ['temporal', 'spatial']") @@ -619,5 +624,14 @@ def create_scale(self, resolution, extent, support, scale_dimension): scale = Cls(resolution=resolution, extent=extent, support=support) setattr(self, '%s_scale' % scale_dimension.lower(), scale) + # commit + try: + session = object_session(self) + session.add(self) + session.commit() + except Exception as e: + session.rollback() + raise e + def __str__(self): return "%s data source at %s " % (self.type.name, self.path, self.id) From 6fa2b9ceeefe8a15fcb794be282d2756567695a5 Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Tue, 11 Oct 2022 14:46:11 +0200 Subject: [PATCH 07/10] cast extent to geography to compute area in sqm --- metacatalog/db/views/locations.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metacatalog/db/views/locations.sql b/metacatalog/db/views/locations.sql index a2ed2d69..b898af31 100644 --- a/metacatalog/db/views/locations.sql +++ b/metacatalog/db/views/locations.sql @@ -4,7 +4,7 @@ SELECT entries.id, spatial_scales.extent, ST_Envelope(spatial_scales.extent) "bbox", ST_Centroid(spatial_scales.extent) "centroid", - ST_Area(spatial_scales.extent) * 0.3048 ^ 2 "area_sqm" + ST_Area(spatial_scales.extent :: geography) "area_sqm" FROM entries INNER JOIN datasources ON entries.datasource_id = datasources.id INNER JOIN spatial_scales ON spatial_scales.id = datasources.spatial_scale_id \ No newline at end of file From 07652b94800a77b5b7bb9cd29fb7f3a15e348485 Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Tue, 29 Nov 2022 10:31:26 +0100 Subject: [PATCH 08/10] location view (sql) --- metacatalog/db/views/locations.sql | 59 +++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/metacatalog/db/views/locations.sql b/metacatalog/db/views/locations.sql index b898af31..025a3405 100644 --- a/metacatalog/db/views/locations.sql +++ b/metacatalog/db/views/locations.sql @@ -1,10 +1,51 @@ -CREATE OR REPLACE VIEW locations AS -SELECT entries.id, - entries.location, - spatial_scales.extent, - ST_Envelope(spatial_scales.extent) "bbox", - ST_Centroid(spatial_scales.extent) "centroid", - ST_Area(spatial_scales.extent :: geography) "area_sqm" +--CREATE OR REPLACE VIEW locations AS +select + st_asewkt(t.point_location), + t.*, + st_area(t.geom::geography) as "area_sqm" +from +(SELECT +entries.id, + case when entries.location is not null then + entries.location + else + case when spatial_scales.extent is not null then + st_centroid(spatial_scales.extent) + else + case when entries.geom is not null then + st_centroid(entries.geom) + else + null + end + end + end + as point_location + , + + case when entries.geom is not null then + st_envelope(entries.geom) + else + case when spatial_scales.extent is not null then + st_envelope(spatial_scales.extent) + else + entries.location + end + end + as "bbox", + case when entries.geom is not null then + entries.geom + else + case when spatial_scales.extent is not null then + spatial_scales.extent + else + entries.location + end + end + as "geom" FROM entries -INNER JOIN datasources ON entries.datasource_id = datasources.id -INNER JOIN spatial_scales ON spatial_scales.id = datasources.spatial_scale_id \ No newline at end of file +LEFT JOIN datasources ON entries.datasource_id = datasources.id +LEFT JOIN spatial_scales ON spatial_scales.id = datasources.spatial_scale_id + ) as t + +order by t.id asc +limit 25 \ No newline at end of file From 9655d0691db8f5c8b57bcd3b6b423b69217b1b7a Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Tue, 29 Nov 2022 14:56:52 +0100 Subject: [PATCH 09/10] view locations (sql) drop bbox --- metacatalog/db/views/locations.sql | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/metacatalog/db/views/locations.sql b/metacatalog/db/views/locations.sql index 025a3405..fabb8ec9 100644 --- a/metacatalog/db/views/locations.sql +++ b/metacatalog/db/views/locations.sql @@ -1,4 +1,4 @@ ---CREATE OR REPLACE VIEW locations AS +--CREATE OR REPLACE VIEW locations_realdata AS select st_asewkt(t.point_location), t.*, @@ -15,23 +15,12 @@ entries.id, case when entries.geom is not null then st_centroid(entries.geom) else - null + null::geometry end end end as point_location , - - case when entries.geom is not null then - st_envelope(entries.geom) - else - case when spatial_scales.extent is not null then - st_envelope(spatial_scales.extent) - else - entries.location - end - end - as "bbox", case when entries.geom is not null then entries.geom else @@ -46,6 +35,5 @@ FROM entries LEFT JOIN datasources ON entries.datasource_id = datasources.id LEFT JOIN spatial_scales ON spatial_scales.id = datasources.spatial_scale_id ) as t - -order by t.id asc -limit 25 \ No newline at end of file + +order by t.id asc \ No newline at end of file From 0152ab777a6824cbbc9f8802921e8bb0375dc11f Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Mon, 12 Dec 2022 09:51:02 +0100 Subject: [PATCH 10/10] resolve merge conflicts --- metacatalog/__version__.py | 2 +- metacatalog/api/add.py | 2 +- metacatalog/db/revisions/__init__.py | 2 ++ .../db/revisions/{rev10.py => rev11.py} | 5 ++++- metacatalog/models/datasource.py | 20 +++++++++---------- metacatalog/models/entry.py | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) rename metacatalog/db/revisions/{rev10.py => rev11.py} (82%) diff --git a/metacatalog/__version__.py b/metacatalog/__version__.py index ed7d50e7..8411e551 100644 --- a/metacatalog/__version__.py +++ b/metacatalog/__version__.py @@ -1 +1 @@ -__version__ = '0.5.3' +__version__ = '0.6.1' diff --git a/metacatalog/api/add.py b/metacatalog/api/add.py index 2a366ad3..2b2b3677 100644 --- a/metacatalog/api/add.py +++ b/metacatalog/api/add.py @@ -507,7 +507,7 @@ def add_entry(session, title, author, location, variable, abstract=None, externa Can be either a WKT of a EPSG:4326 location, or the coordinates as a tuple. It has to be (X,Y), to (longitude, latitude). - .. versionchanged:: 0.5.3 + .. versionchanged:: 0.6.1 A POINT location should be specified here if there is a physical measurement point that is different from the centroid of the spatial extent (e.g., discharge measurement with the extent of the catchment). diff --git a/metacatalog/db/revisions/__init__.py b/metacatalog/db/revisions/__init__.py index 79da9c6c..c489f81e 100644 --- a/metacatalog/db/revisions/__init__.py +++ b/metacatalog/db/revisions/__init__.py @@ -10,6 +10,7 @@ rev8, rev9, rev10, + rev11, ) revisions = { @@ -24,4 +25,5 @@ 8: rev8, 9: rev9, 10: rev10, + 11: rev11, } diff --git a/metacatalog/db/revisions/rev10.py b/metacatalog/db/revisions/rev11.py similarity index 82% rename from metacatalog/db/revisions/rev10.py rename to metacatalog/db/revisions/rev11.py index 702ca4ab..4e70a63a 100644 --- a/metacatalog/db/revisions/rev10.py +++ b/metacatalog/db/revisions/rev11.py @@ -3,9 +3,12 @@ ----------------------------- date: 2022-10-05T14:41:32.055433 -revision #10 +revision #11 Make column entries.location nullable (raster data do not have a POINT location). +Attention, the downgrade sets everywhere where entries.location == NULL to +(POINT 0 0), which may not correspond to the state of the database before upgrading, +so the revision upgrade cannot be undone. """ from sqlalchemy.orm import Session diff --git a/metacatalog/models/datasource.py b/metacatalog/models/datasource.py index ea3705ad..85c88e6f 100644 --- a/metacatalog/models/datasource.py +++ b/metacatalog/models/datasource.py @@ -339,7 +339,7 @@ class SpatialScale(Base): :attr:`resolution_str` property for a string representation extent : geoalchemy2.Geometry The spatial extent of the dataset is given as a ``'POLYGON'``. - .. versionchanged:: 0.5.3 + .. versionchanged:: 0.6.1 From this ``POLYGON``, a bounding box and the centroid are internally calculated. To specify a point location here, use the same value for easting and @@ -604,7 +604,7 @@ def save_args_from_dict(self, args_dict, commit=False): session.rollback() raise e - def create_scale(self, resolution, extent, support, scale_dimension): + def create_scale(self, resolution, extent, support, scale_dimension, commit=False): """ Create a new scale for the dataset """ @@ -624,14 +624,14 @@ def create_scale(self, resolution, extent, support, scale_dimension): scale = Cls(resolution=resolution, extent=extent, support=support) setattr(self, '%s_scale' % scale_dimension.lower(), scale) - # commit - try: - session = object_session(self) - session.add(self) - session.commit() - except Exception as e: - session.rollback() - raise e + if commit: + try: + session = object_session(self) + session.add(self) + session.commit() + except Exception as e: + session.rollback() + raise e def __str__(self): return "%s data source at %s " % (self.type.name, self.path, self.id) diff --git a/metacatalog/models/entry.py b/metacatalog/models/entry.py index 67011af8..1a495c70 100644 --- a/metacatalog/models/entry.py +++ b/metacatalog/models/entry.py @@ -68,7 +68,7 @@ class Entry(Base): Usually an unque ID field of other data-storage solutions. The exernal_id is only stored for reference reasons. location : str, tuple - .. versionchanged:: 0.5.3 + .. versionchanged:: 0.6.1 A POINT location should be specified here if there is a physical measurement point that is different from the centroid of the spatial extent (e.g., discharge measurement with the extent of the catchment).