Optimize getFilesAndIndicesForDimensions

.dockerignore

@@ -1,6 +1,7 @@
 *
 !compile.sh
 !runtests.sh
+!runtests_psql.sh
 !CMakeLists.txt
 !Docker/adaguc-server-config-python-postgres.xml
 !Docker/start.sh
@@ -22,3 +23,4 @@
 !tests/inspiretests/
 !tests/functional_test.py
 !tests/starttests.sh
+!tests/starttests_psql.sh

Dockerfile

@@ -85,16 +85,21 @@ COPY data /adaguc/adaguc-server-master/data
 COPY python /adaguc/adaguc-server-master/python
 
 ######### Third stage, test ############
-FROM base AS test
+# To run the tests against a postgres db, see docs/test_postgesql.md
+FROM base as test
+
+ENV TEST_IN_CONTAINER 1
 
 COPY requirements-dev.txt /adaguc/adaguc-server-master/requirements-dev.txt
 RUN pip install --no-cache-dir -r requirements-dev.txt
 
 COPY tests /adaguc/adaguc-server-master/tests
 COPY runtests.sh /adaguc/adaguc-server-master/runtests.sh
+COPY runtests_psql.sh /adaguc/adaguc-server-master/runtests_psql.sh
 
-# Run adaguc-server functional and regression tests
+# Run adaguc-server functional and regression tests. See also `./doc/developing/testing.md`
 RUN bash runtests.sh
+# RUN bash runtests_psql.sh
 
 # Create a file indicating that the test succeeded. This file is used in the final stage
 RUN echo "TESTSDONE" >  /adaguc/adaguc-server-master/testsdone.txt

Readme.md

@@ -23,6 +23,7 @@ Graphical representations of the data can be made using contour lines, shading,
 * [Glossary](./doc/GLOSSARY.md)
 * [Quick Tips](./doc/overview/QuickTips.md)
 * [Profiling](./doc/developing/profiling.md)
+* [Testing](./doc/developing/testing.md)
 
 
 For our detailed paper please read https://doi.org/10.5334/jors.382

adagucserverEC/CDBAdapterPostgreSQL.h

@@ -27,13 +27,19 @@
 #include "CDebugger.h"
 #include "CPGSQLDB.h"
 
+struct DimInfo {
+  CT::string tableName;
+  CT::string dataType;
+};
+typedef struct DimInfo DimInfo;
+
 class CDBAdapterPostgreSQL : public CDBAdapter {
 private:
   DEF_ERRORFUNCTION();
   CPGSQLDB *dataBaseConnection;
   CPGSQLDB *getDataBaseConnection();
   CServerConfig::XMLE_Configuration *configurationObject;
-  std::map<std::string, std::string> lookupTableNameCacheMap;
+  std::map<std::string, DimInfo> lookupTableNameCacheMap;
   std::map<std::string, std::vector<std::string>> fileListPerTable;
   int createDimTableOfType(const char *dimname, const char *tablename, int type);
 
@@ -46,6 +52,12 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   CDBStore::Store *getClosestDataTimeToSystemTime(const char *netcdfDimName, const char *tableName);
 
   CT::string getTableNameForPathFilterAndDimension(const char *path, const char *filter, const char *dimension, CDataSource *dataSource);
+  std::map<CT::string, DimInfo> getTableNamesForPathFilterAndDimensions(const char *path, const char *filter, std::vector<CT::string> dimensions, CDataSource *dataSource);
+
+  CT::string getLookupIdentifier(const char *path, const char *filter, const char *dimension);
+  void assertLookupTableExists();
+  void addToLookupTable(const char *path, const char *filter, CT::string dimensionName, CT::string tableName);
+
   int autoUpdateAndScanDimensionTables(CDataSource *dataSource);
   CDBStore::Store *getMin(const char *name, const char *table);
   CDBStore::Store *getMax(const char *name, const char *table);

adagucserverEC/CDBAdapterSQLLite.cpp

@@ -183,7 +183,16 @@ CDBStore::Store *CDBAdapterSQLLite::CSQLLiteDB::queryToStore(const char *pszQuer
 
   for (size_t rowNumber = 0; rowNumber < numRows; rowNumber++) {
     CDBStore::Record *record = new CDBStore::Record(colModel);
-    for (size_t colNumber = 0; colNumber < numCols; colNumber++) record->push(colNumber, queryValues[colNumber + rowNumber * numCols].c_str());
+    for (size_t colNumber = 0; colNumber < numCols; colNumber++) {
+      // SQlite always returns `10.0` if you insert `10.0` or `10`. PSQL always returns `10`
+      // HACK: to make output consistent with PSQL
+      CT::string s(queryValues[colNumber + rowNumber * numCols].c_str());
+      if (s.endsWith(".0")) {
+        s.substringSelf(0, s.length() - 2);
+      }
+
+      record->push(colNumber, s);
+    }
     store->push(record);
   }
 

adagucserverEC/CRequest.cpp

@@ -1177,6 +1177,20 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
       }
     }
 
+    // Check if requested time dimensions use valid characters
+    for (auto &dim : dataSource->requiredDims) {
+      // FIXME: checkTimeFormat used to get called on every dim value, not just datetime. Check if this is required
+      if (!dim->isATimeDimension) continue;
+
+      CT::string *dimValues = dim->value.splitToArray(",");
+      for (size_t i = 0; i < dimValues->count; i++) {
+        if (!CServerParams::checkTimeFormat(dimValues[i])) {
+          CDBError("Queried dimension %s=%s failed datetime regex", dim->name.c_str(), dim->value.c_str());
+          throw InvalidDimensionValue;
+        }
+      }
+    }
+
     // STOP NOW
   } catch (int i) {
     CDBError("%d", i);

doc/developing/testing.md

@@ -0,0 +1,70 @@
+# Testing Adaguc-server
+
+Adaguc contains two types of tests:
+
+- Tests specifically for adaguc-server
+- Tests specifically for the Python wrapper/EDR functionality
+
+During the tests, adaguc can use either the SQLite or PostgreSQL backend.
+
+## Running tests in Docker via SQLite
+
+The main [Dockerfile](~/Dockerfile) contains a _test stage_ which executes `runtests.sh`. This means that all tests will be executed during a regular docker build. Running the tests in Docker is recommended especially for users that don't run Linux/amd64.
+
+To run all tests, trigger a build:
+```bash
+docker build --progress plain -t adaguc-server .
+```
+
+The build will fail if any of the tests have failed.
+
+
+## Running tests outside Docker via SQLite
+
+Running the tests outside of Docker is possible by executing the `./runtests.sh` script.
+
+> **⚠️ Note**: some tests might fail due to platform architecture differences between amd64 and arm64.
+
+## Using postgres
+
+To run the tests against Postgres, the following needs to happen:
+
+- The adaguc XML configuration files used by tests all require a `<DataBase parameters="{ADAGUC_DB}"/>` section. This gets handled by `starttests_psql.sh`
+- PostgreSQL must be reachable on port `5432`.
+
+### Via Docker
+
+To do this through Docker, follow these steps:
+1. Edit `docker-compose.yml`
+2. Comment out all servers that are not `adaguc-db`
+3. Add the `ports` section to the `adaguc-db` service to expose port 5432:
+```yaml
+  adaguc-db:
+    (...)
+    ports:
+      - 5432:5432
+```
+4. Start postgres while inside the `./Docker` directory:
+```bash
+docker compose up -d
+```
+5. Edit `./Dockerfile`, to enable the `runtests_psql.sh`:
+```Dockerfile
+# RUN bash runtests.sh
+RUN bash runtests_psql.sh
+```
+6. Trigger a new build:
+```bash
+docker build --progress plain -t adaguc-server .
+```
+
+All tests should now run against postgres via the docker build.
+
+### Without Docker
+
+To run the tests outside of Docker, make sure Postgres is reachable on port 5432, and then execute the test script directly:
+```bash
+./runtests_psql.sh
+```
+
+> **⚠️ Note**: some tests might fail due to platform architecture differences between amd64 and arm64.

python/lib/adaguc/AdagucTestTools.py

@@ -7,6 +7,7 @@
 from lxml import etree, objectify
 import urllib.request
 from PIL import Image
+import subprocess
 
 ADAGUC_PATH = os.getenv("ADAGUC_PATH", " ")
 
@@ -142,6 +143,22 @@ def cleanTempDir(self):
         self.mkdir_p(ADAGUC_TMP)
         return
 
+    def cleanPostgres(self):
+        """Clean the postgres database if the ADAGUC_DB variable has been set.
+
+        Some tests fail when using postgres if there is already data present in the database.
+        Running the test separately works."""
+
+        if adaguc_db := os.getenv("ADAGUC_DB", None):
+            subprocess.run(
+                [
+                    "psql",
+                    adaguc_db,
+                    "-c",
+                    "DROP SCHEMA public CASCADE; CREATE SCHEMA public;",
+                ]
+            )
+
     def mkdir_p(self, directory):
         if not os.path.exists(directory):
             os.makedirs(directory)

python/python_fastapi_server/test_ogc_api_edr.py

@@ -20,6 +20,7 @@ def set_environ():
 def setup_test_data():
     print("About to ingest data")
     AdagucTestTools().cleanTempDir()
+    AdagucTestTools().cleanPostgres()
     for service in [
         "netcdf_5d.xml",
         "dataset_a.xml",
@@ -180,6 +181,50 @@ def test_coll_multi_dim_position(client: TestClient):
         330000,
     ]
 
+    # Should handle querying multiple heights separated by comma
+    resp = client.get(
+        "/edr/collections/testcollection/instances/2024060100/position?coords=POINT(5.2 52.0)&datetime=2024-06-01T01:00:00Z/2024-06-01T04:00:00Z&parameter-name=testdata&z=10,20"
+    )
+    assert resp.status_code, 200
+    covjson = resp.json()
+
+    assert covjson["domain"]["axes"]["z"]["values"] == [10, 20]
+    assert covjson["ranges"]["testdata"]["shape"] == [1, 1, 2, 4]
+    assert covjson["ranges"]["testdata"]["values"] == [
+        0.0,
+        10000.0,
+        20000.0,
+        30000.0,
+        100000.0,
+        110000.0,
+        120000.0,
+        130000.0,
+    ]
+
+    # Should handle querying multiple heights separated by comma, combined with querying ranges
+    resp = client.get(
+        "/edr/collections/testcollection/instances/2024060100/position?coords=POINT(5.2 52.0)&datetime=2024-06-01T01:00:00Z/2024-06-01T04:00:00Z&parameter-name=testdata&z=10,30/40"
+    )
+    assert resp.status_code, 200
+    covjson = resp.json()
+
+    assert covjson["domain"]["axes"]["z"]["values"] == [10, 30, 40]
+    assert covjson["ranges"]["testdata"]["shape"] == [1, 1, 3, 4]
+    assert covjson["ranges"]["testdata"]["values"] == [
+        0.0,
+        10000.0,
+        20000.0,
+        30000.0,
+        200000.0,
+        210000.0,
+        220000.0,
+        230000.0,
+        300000.0,
+        310000.0,
+        320000.0,
+        330000.0,
+    ]
+
 
 def test_coll_multi_dim_cube(client: TestClient):
     resp = client.get(

python/python_fastapi_server/test_ogc_api_features.py

@@ -20,6 +20,7 @@ def set_environ():
 def setup_test_data():
     print("About to ingest data")
     AdagucTestTools().cleanTempDir()
+    AdagucTestTools().cleanPostgres()
     for service in ["netcdf_5d.xml", "dataset_a.xml"]:
         _status, _data, _headers = AdagucTestTools().runADAGUCServer(
             args=[

runtests_psql.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd tests
+bash starttests_psql.sh

tests/AdagucTests/TestWMS.py

@@ -347,6 +347,7 @@ def test_WMSCMDUpdateDB(self):
 
     def test_WMSCMDUpdateDBTailPath(self):
         AdagucTestTools().cleanTempDir()
+        AdagucTestTools().cleanPostgres()
         # pylint: disable=unused-variable
         status, data, headers = AdagucTestTools().runADAGUCServer(
             args=[
@@ -404,6 +405,7 @@ def test_WMSCMDUpdateDBTailPath(self):
 
     def test_WMSCMDUpdateDBPath(self):
         AdagucTestTools().cleanTempDir()
+        AdagucTestTools().cleanPostgres()
         # pylint: disable=unused-variable
         status, data, headers = AdagucTestTools().runADAGUCServer(
             args=[
@@ -2021,6 +2023,7 @@ def test_WMSCMDUpdateDBPathFileWithNonMatchingPath(self):
         This tests if the autofinddataset option correctly finds the file if it is in a subfolder
         """
         AdagucTestTools().cleanTempDir()
+        AdagucTestTools().cleanPostgres()
         # pylint: disable=unused-variable
 
         config = ADAGUC_PATH + "data/config/adaguc.tests.dataset.xml"

tests/AdagucTests/TestWMSDocumentCache.py

@@ -41,6 +41,8 @@ def test_WMSCMDUpdateDB(self):
 
   def test_WMSCMDUpdateDBTailPath(self):
     AdagucTestTools().cleanTempDir()
+    AdagucTestTools().cleanPostgres()
+
     ADAGUC_PATH = os.environ['ADAGUC_PATH']
     config = ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml,' + \
         ADAGUC_PATH + '/data/config/datasets/adaguc.testtimeseriescached.xml'

tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetFeatureInfo.json

@@ -1 +1 @@
-[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"16.701019"}}}]
+[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"16.701019"}}}]

tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetTimeSeries.json

@@ -1 +1 @@
-[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"-0.203398","2019-01-01T20:00:00Z":"-0.221830","2019-01-01T21:00:00Z":"-0.240139","2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"17.272943","2019-01-01T20:00:00Z":"17.284334","2019-01-01T21:00:00Z":"16.998619","2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"17.069546","2019-01-01T20:00:00Z":"17.062504","2019-01-01T21:00:00Z":"16.758480","2019-01-01T22:00:00Z":"16.701019"}}}]
+[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"-0.203398","2019-01-01T20:00:00Z":"-0.221830","2019-01-01T21:00:00Z":"-0.240139","2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"17.272943","2019-01-01T20:00:00Z":"17.284334","2019-01-01T21:00:00Z":"16.998619","2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"17.069546","2019-01-01T20:00:00Z":"17.062504","2019-01-01T21:00:00Z":"16.758480","2019-01-01T22:00:00Z":"16.701019"}}}]

tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_DimensionUnits.xml

@@ -123,7 +123,7 @@
 <BoundingBox CRS="EPSG:3067" minx="-1467755.480414" miny="5755116.469362" maxx="-1360791.221574" maxy="5860768.051867" />
 <BoundingBox CRS="PROJ4:%2Bproj%3Dlonglat%20%2Bellps%3DWGS84%20%2Bdatum%3DWGS84%20%2Bno_defs" minx="-0.050000" miny="48.950000" maxx="1.050000" maxy="49.650000" />
 <Dimension name="time" units="ISO8601" default="2023-10-26T05:00:00Z" multipleValues="1" nearestValue="0" current="1">2023-10-26T05:00:00Z</Dimension>
-<Dimension name="height_above_ground_level_in_m" units="m" default="10.0" multipleValues="1" nearestValue="0" >10.0</Dimension>
+<Dimension name="height_above_ground_level_in_m" units="m" default="10" multipleValues="1" nearestValue="0" >10</Dimension>
 <Dimension name="reference_time" units="ISO8601" default="2023-10-26T00:00:00Z" multipleValues="1" nearestValue="0" current="1">2023-10-26T00:00:00Z</Dimension>
    <Style>    <Name>auto/nearest</Name>    <Title>auto/nearest</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.311-getcapabilities-dimension-units&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=wind-speed-of-gust-hagl&amp;format=image/png&amp;STYLE=auto/nearest"/>    </LegendURL>  </Style>   <Style>    <Name>auto/bilinear</Name>    <Title>auto/bilinear</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.311-getcapabilities-dimension-units&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=wind-speed-of-gust-hagl&amp;format=image/png&amp;STYLE=auto/bilinear"/>    </LegendURL>  </Style>   <Style>    <Name>auto/point</Name>    <Title>auto/point</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.311-getcapabilities-dimension-units&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=wind-speed-of-gust-hagl&amp;format=image/png&amp;STYLE=auto/point"/>    </LegendURL>  </Style>   <Style>    <Name>auto/barb</Name>    <Title>auto/barb</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.311-getcapabilities-dimension-units&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=wind-speed-of-gust-hagl&amp;format=image/png&amp;STYLE=auto/barb"/>    </LegendURL>  </Style></Layer>
     </Layer>