diff --git a/.dockerignore b/.dockerignore index 9d6aec78..23bfebb5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ * !compile.sh !runtests.sh +!runtests_psql.sh !CMakeLists.txt !Docker/adaguc-server-config-python-postgres.xml !Docker/start.sh @@ -22,3 +23,4 @@ !tests/inspiretests/ !tests/functional_test.py !tests/starttests.sh +!tests/starttests_psql.sh diff --git a/Dockerfile b/Dockerfile index 1c0755b9..0e4aa4cf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.26.0" +LABEL version="2.27.0" # Try to update image packages RUN apt-get -q -y update \ @@ -85,16 +85,21 @@ COPY data /adaguc/adaguc-server-master/data COPY python /adaguc/adaguc-server-master/python ######### Third stage, test ############ +# To run the tests against a postgres db, see docs/test_postgesql.md FROM base AS test +ENV TEST_IN_CONTAINER 1 + COPY requirements-dev.txt /adaguc/adaguc-server-master/requirements-dev.txt RUN pip install --no-cache-dir -r requirements-dev.txt COPY tests /adaguc/adaguc-server-master/tests COPY runtests.sh /adaguc/adaguc-server-master/runtests.sh +COPY runtests_psql.sh /adaguc/adaguc-server-master/runtests_psql.sh -# Run adaguc-server functional and regression tests +# Run adaguc-server functional and regression tests. See also `./doc/developing/testing.md` RUN bash runtests.sh +# RUN bash runtests_psql.sh # Create a file indicating that the test succeeded. This file is used in the final stage RUN echo "TESTSDONE" > /adaguc/adaguc-server-master/testsdone.txt diff --git a/NEWS.md b/NEWS.md index 607fd293..6f027f05 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +**Version 2.27.0 2024-09-02** +- PostgreSQL query from `getFilesAndIndicesForDimensions` has been rewritten, which fixes https://github.com/KNMI/adaguc-server/issues/341. +- Optimized existing PostgreSQL queries and reduced number of PostgreSQL queries in general. This results in better performance, the benchmark tool runs 9% faster. + **Version 2.26.0 2024-07-12** - Added EDR cube call for gridded datsets to ADAGUC. diff --git a/Readme.md b/Readme.md index d07fd4d7..75205a65 100644 --- a/Readme.md +++ b/Readme.md @@ -23,6 +23,7 @@ Graphical representations of the data can be made using contour lines, shading, * [Glossary](./doc/GLOSSARY.md) * [Quick Tips](./doc/overview/QuickTips.md) * [Profiling](./doc/developing/profiling.md) +* [Testing](./doc/developing/testing.md) For our detailed paper please read https://doi.org/10.5334/jors.382 diff --git a/adagucserverEC/CDBAdapterPostgreSQL.cpp b/adagucserverEC/CDBAdapterPostgreSQL.cpp index f2913868..d8b9aac8 100644 --- a/adagucserverEC/CDBAdapterPostgreSQL.cpp +++ b/adagucserverEC/CDBAdapterPostgreSQL.cpp @@ -25,6 +25,7 @@ #include "CDBAdapterPostgreSQL.h" #include +#include #include "CDebugger.h" const char *CDBAdapterPostgreSQL::className = "CDBAdapterPostgreSQL"; @@ -61,6 +62,9 @@ CPGSQLDB *CDBAdapterPostgreSQL::getDataBaseConnection() { CDBError("Unable to connect to DB"); return NULL; } + + assertLookupTableExists(); + #ifdef MEASURETIME StopWatch_Stop(" dims; + for (const auto &dim : dataSource->requiredDims) { + CT::string dimString(dim->netCDFDimName); + dimString.toLowerCaseSelf(); + dims.push_back(dimString); + } + std::map mapping = + getTableNamesForPathFilterAndDimensions(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), dims, dataSource); + // Compose the query for (size_t i = 0; i < dataSource->requiredDims.size(); i++) { CT::string netCDFDimName(&dataSource->requiredDims[i]->netCDFDimName); - - CT::string tableName; - try { - tableName = getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), netCDFDimName.c_str(), dataSource); - } catch (int e) { - CDBError("Unable to create tableName from '%s' '%s' '%s'", dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), netCDFDimName.c_str()); - return NULL; - } + CT::string tableName = mapping[netCDFDimName].tableName; CT::string subQuery; subQuery.print("(select path,dim%s,%s from %s ", netCDFDimName.c_str(), netCDFDimName.c_str(), tableName.c_str()); @@ -343,6 +350,17 @@ CDBStore::Store *CDBAdapterPostgreSQL::getFilesForIndices(CDataSource *dataSourc } CDBStore::Store *CDBAdapterPostgreSQL::getFilesAndIndicesForDimensions(CDataSource *dataSource, int limit) { + /* + Build query to find combination of file paths and dimensions, by filtering on requested dimensions. Query has following form: + + SELECT DISTINCT t1.path, , dim, ... FROM t1 + INNER JOIN
tn ON t1.path = tn.path ... + WHERE AND ... + ORDER BY LIMIT + + Returns the DBStore of this query. + */ + #ifdef MEASURETIME StopWatch_Stop(">CDBAdapterPostgreSQL::getFilesAndIndicesForDimensions"); #endif @@ -356,181 +374,97 @@ CDBStore::Store *CDBAdapterPostgreSQL::getFilesAndIndicesForDimensions(CDataSour return NULL; } - // CDBDebug("dataSource->requiredDims.size() = %d",dataSource->requiredDims.size()); - - CT::string queryOrderedDESC; - CT::string query; - queryOrderedDESC.print("select a0.path"); - for (size_t i = 0; i < dataSource->requiredDims.size(); i++) { - queryOrderedDESC.printconcat(",%s,dim%s", dataSource->requiredDims[i]->netCDFDimName.c_str(), dataSource->requiredDims[i]->netCDFDimName.c_str()); + // Find all tables belonging to given path, filter and dimensions + std::vector dims; + for (const auto &dim : dataSource->requiredDims) { + CT::string dimString(dim->netCDFDimName); + dimString.toLowerCaseSelf(); + dims.push_back(dimString); } + std::map mapping = + getTableNamesForPathFilterAndDimensions(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), dims, dataSource); - queryOrderedDESC.concat(" from "); - bool timeValidationError = false; - -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", queryOrderedDESC.c_str()); -#endif + // Create a mapping for filtering where key=dimension name, value=requested dimension value(s) + std::map> requestedDimMap; + for (const auto &dim : dataSource->requiredDims) { + requestedDimMap[dim->netCDFDimName.c_str()] = (&dim->value)->splitToStack(","); + } - // Compose the query - for (size_t i = 0; i < dataSource->requiredDims.size(); i++) { - CT::string netCDFDimName(&dataSource->requiredDims[i]->netCDFDimName); + CT::string query; + query.print("SELECT DISTINCT t1.path"); + for (const auto &dim : dims) { + query.printconcat(", %s, dim%s", dim.c_str(), dim.c_str()); + } - CT::string tableName; - try { - tableName = getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), netCDFDimName.c_str(), dataSource); - } catch (int e) { - CDBError("Unable to create tableName from '%s' '%s' '%s'", dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), netCDFDimName.c_str()); - return NULL; + int i = 0; + for (const auto &m : mapping) { + if (i == 0) { + query.printconcat(" FROM %s t1 ", m.second.tableName.c_str()); + } else { + query.printconcat("INNER JOIN %s t%d ON t1.path = t%d.path ", m.second.tableName.c_str(), i + 1, i + 1); } + i++; + } - CT::string subQuery; - subQuery.print("(select path,dim%s,%s from %s ", netCDFDimName.c_str(), netCDFDimName.c_str(), tableName.c_str()); - CT::string queryParams(&dataSource->requiredDims[i]->value); - int numQueriesAdded = 0; - if (queryParams.equals("*") == false) { - CT::string *cDims = queryParams.splitToArray(","); // Split up by commas (and put into cDims) - for (size_t k = 0; k < cDims->count; k++) { - CT::string *sDims = cDims[k].splitToArray("/"); // Split up by slashes (and put into sDims) - - if (k == 0) { - subQuery.concat("where "); - } - if (sDims->count > 0 && k > 0) subQuery.concat("or "); - for (size_t l = 0; l < sDims->count && l < 2; l++) { - if (sDims[l].length() > 0) { - numQueriesAdded++; - // Determine column type (timestamp, integer, real) - bool isRealType = false; - CT::string dataTypeQuery; - dataTypeQuery.print("select data_type from information_schema.columns where table_name = '%s' and column_name = '%s'", tableName.c_str(), netCDFDimName.c_str()); -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", dataTypeQuery.c_str()); -#endif - try { + // Filter on tiling bounding box (or not) + query.concat("WHERE "); + if (dataSource->queryBBOX) { + query.printconcat("t1.adaguctilinglevel = %d and minx >= %f and maxx <= %f and miny >= %f and maxy <= %f ", dataSource->queryLevel, dataSource->nativeViewPortBBOX[0], + dataSource->nativeViewPortBBOX[2], dataSource->nativeViewPortBBOX[1], dataSource->nativeViewPortBBOX[3]); + } else { + query.concat("t1.adaguctilinglevel != -1 "); + } - CDBStore::Store *dataType = DB->queryToStore(dataTypeQuery.c_str(), true); + // Filter on the requested dimensions + for (const auto &m : mapping) { + const char *dimName = m.first.c_str(); + const char *tableName = m.second.tableName.c_str(); + CT::string dimDataType = m.second.dataType; + CT::string whereStatement; + std::vector requestedDimVals = requestedDimMap[m.first]; - if (dataType != NULL) { - if (dataType->getSize() == 1) { -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", dataType->getRecord(0)->get(0)->c_str()); -#endif - if (dataType->getRecord(0)->get(0)->equals("real")) { - isRealType = true; - } - } - } - - delete dataType; - dataType = NULL; - - } catch (int e) { - if ((CServerParams::checkDataRestriction() & SHOW_QUERYINFO) == false) query.copy("hidden"); - CDBError("Unable to determine column type: '%s'", dataTypeQuery.c_str()); - return NULL; - } - - if (l > 0) subQuery.concat("and "); - if (sDims->count == 1) { - - if (!CServerParams::checkTimeFormat(sDims[l])) timeValidationError = true; - - if (isRealType == false) { - subQuery.printconcat("%s = '%s' ", netCDFDimName.c_str(), sDims[l].c_str()); - } - - // This query gets the closest value from the table. - if (isRealType) { - subQuery.printconcat("abs(%s - %s) = (select min(abs(%s - %s)) from %s)", sDims[l].c_str(), netCDFDimName.c_str(), sDims[l].c_str(), netCDFDimName.c_str(), tableName.c_str()); - } - } - - // TODO Currently only start/stop is supported, start/stop/resolution is not supported yet. - if (sDims->count >= 2) { - if (l == 0) { - if (!CServerParams::checkTimeFormat(sDims[l])) timeValidationError = true; - // Get closest lowest value to this requested one, or if request value is way below get earliest value: - subQuery.printconcat("%s >= (select max(%s) from %s where %s <= '%s' or %s = (select min(%s) from %s)) ", netCDFDimName.c_str(), netCDFDimName.c_str(), tableName.c_str(), - netCDFDimName.c_str(), sDims[l].c_str(), netCDFDimName.c_str(), netCDFDimName.c_str(), tableName.c_str()); - // subQuery.printconcat("%s >= '%s' ",netCDFDimName.c_str(),sDims[l].c_str()); - } - if (l == 1) { - if (!CServerParams::checkTimeFormat(sDims[l])) timeValidationError = true; - subQuery.printconcat("%s <= '%s' ", netCDFDimName.c_str(), sDims[l].c_str()); - } - } - } - } - delete[] sDims; - } - delete[] cDims; - } - if (i == 0) { - if (numQueriesAdded == 0) { - subQuery.printconcat("where "); - } else { - subQuery.printconcat("and "); + for (int i = 0; i < requestedDimVals.size(); ++i) { + if (requestedDimVals[i].equals("*")) continue; + + if (i != 0) { + whereStatement.printconcat(" OR "); } - if (dataSource->queryBBOX) { - subQuery.printconcat("adaguctilinglevel = %d and minx >= %f and maxx <= %f and miny >= %f and maxy <= %f ", dataSource->queryLevel, dataSource->nativeViewPortBBOX[0], - dataSource->nativeViewPortBBOX[2], dataSource->nativeViewPortBBOX[1], dataSource->nativeViewPortBBOX[3]); + std::vector dimVals = requestedDimVals[i].splitToStack("/"); + if (dimVals.size() == 1) { + const char *dimVal = dimVals[0].c_str(); + // If dimension value is a number, find closest value. + if (dimDataType.equals("real")) { + whereStatement.printconcat("ABS(%s - %s) = (SELECT MIN(ABS(%s - %s)) FROM %s)", dimVal, dimName, dimVal, dimName, tableName); + } else { + whereStatement.printconcat("%s = '%s'", dimName, dimVal); + } } else { - subQuery.printconcat("adaguctilinglevel != %d ", -1); + // Find value within range of dimVals[0] and dimVals[1] + // Get closest lowest value to this requested one, or if request value is way below get earliest value: + whereStatement.printconcat("%s >= (SELECT MAX(%s) FROM %s WHERE %s <= '%s' OR %s = (SELECT MIN(%s) FROM %s)) AND %s <= '%s'", dimName, dimName, tableName, dimName, dimVals[0].c_str(), dimName, + dimName, tableName, dimName, dimVals[1].c_str()); } - subQuery.printconcat("ORDER BY %s DESC limit %d)a%d ", netCDFDimName.c_str(), limit, i); - // subQuery.printconcat("ORDER BY %s DESC )a%d ",netCDFDimName.c_str(),i); - } else { - subQuery.printconcat("ORDER BY %s DESC)a%d ", netCDFDimName.c_str(), i); } - // subQuery.printconcat("ORDER BY %s DESC)a%d ",netCDFDimName.c_str(),i); - if (i < dataSource->requiredDims.size() - 1) subQuery.concat(","); - queryOrderedDESC.concat(&subQuery); - } - // CDBDebug("%s",queryOrderedDESC.c_str()); -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", queryOrderedDESC.c_str()); -#endif - // Join by path - if (dataSource->requiredDims.size() > 1) { - queryOrderedDESC.concat(" where a0.path=a1.path"); - for (size_t i = 2; i < dataSource->requiredDims.size(); i++) { - queryOrderedDESC.printconcat(" and a0.path=a%d.path", i); + if (!whereStatement.empty()) { + query.printconcat("AND (%s) ", whereStatement.c_str()); } } -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", queryOrderedDESC.c_str()); -#endif - - if (timeValidationError == true) { - if ((CServerParams::checkDataRestriction() & SHOW_QUERYINFO) == false) queryOrderedDESC.copy("hidden"); - CDBError("queryOrderedDESC fails regular expression: '%s'", queryOrderedDESC.c_str()); - return NULL; - } - query.print("select distinct * from (%s)T order by ", queryOrderedDESC.c_str()); - query.concat(&dataSource->requiredDims[0]->netCDFDimName); + // Order and limit query + query.printconcat("ORDER BY %s", dataSource->requiredDims[0]->netCDFDimName.c_str()); for (size_t i = 1; i < dataSource->requiredDims.size(); i++) { - query.printconcat(",%s", dataSource->requiredDims[i]->netCDFDimName.c_str()); + query.printconcat(", %s", dataSource->requiredDims[i]->netCDFDimName.c_str()); } + query.printconcat(" LIMIT %d", limit); // Execute the query - - // writeLogFile3(query.c_str()); - // writeLogFile3("\n"); -// values_path = DB.query_select(query.c_str(),0); -#ifdef CDBAdapterPostgreSQL_DEBUG - CDBDebug("%s", query.c_str()); -#endif - CDBStore::Store *store = NULL; try { store = DB->queryToStore(query.c_str(), true); } catch (int e) { if ((CServerParams::checkDataRestriction() & SHOW_QUERYINFO) == false) query.copy("hidden"); - setExceptionType(InvalidDimensionValue); CDBDebug("Query failed with code %d (%s)", e, query.c_str()); return NULL; } @@ -545,7 +479,6 @@ int CDBAdapterPostgreSQL::autoUpdateAndScanDimensionTables(CDataSource *dataSour StopWatch_Stop(">CDBAdapterPostgreSQL::autoUpdateAndScanDimensionTables"); #endif CServerParams *srvParams = dataSource->srvParams; - ; CServerConfig::XMLE_Layer *cfgLayer = dataSource->cfgLayer; CPGSQLDB *dataBaseConnection = getDataBaseConnection(); if (dataBaseConnection == NULL) { @@ -558,19 +491,21 @@ int CDBAdapterPostgreSQL::autoUpdateAndScanDimensionTables(CDataSource *dataSour bool tableNotFound = false; bool fileNeedsUpdate = false; CT::string dimName; - for (size_t i = 0; i < cfgLayer->Dimension.size(); i++) { - dimName = cfgLayer->Dimension[i]->attr.name.c_str(); - - CT::string tableName; - try { - tableName = getTableNameForPathFilterAndDimension(cfgLayer->FilePath[0]->value.c_str(), cfgLayer->FilePath[0]->attr.filter.c_str(), dimName.c_str(), dataSource); - } catch (int e) { - CDBError("Unable to create tableName from '%s' '%s' '%s'", cfgLayer->FilePath[0]->value.c_str(), cfgLayer->FilePath[0]->attr.filter.c_str(), dimName.c_str()); - return 1; - } + + // Find all tables belonging to given path, filter, and dimensions + std::vector dims; + for (const auto &dim : cfgLayer->Dimension) { + CT::string dimString(dim->attr.name); + dimString.toLowerCaseSelf(); + dims.push_back(dimString); + } + std::map mapping = getTableNamesForPathFilterAndDimensions(cfgLayer->FilePath[0]->value.c_str(), cfgLayer->FilePath[0]->attr.filter.c_str(), dims, dataSource); + + for (const auto &m : mapping) { + dimName = m.first; CT::string query; - query.print("select path,filedate,%s from %s limit 1", dimName.c_str(), tableName.c_str()); + query.print("select path,filedate,%s from %s limit 1", dimName.c_str(), m.second.tableName.c_str()); CDBStore::Store *store = dataBaseConnection->queryToStore(query.c_str()); if (store == NULL) { tableNotFound = true; @@ -607,20 +542,13 @@ int CDBAdapterPostgreSQL::autoUpdateAndScanDimensionTables(CDataSource *dataSour if (fileNeedsUpdate == true) { // Recreate table if (srvParams->isAutoLocalFileResourceEnabled() == true) { - for (size_t i = 0; i < cfgLayer->Dimension.size(); i++) { - dimName = cfgLayer->Dimension[i]->attr.name.c_str(); + for (auto &m : mapping) { + dimName = m.first; - CT::string tableName; - try { - tableName = getTableNameForPathFilterAndDimension(cfgLayer->FilePath[0]->value.c_str(), cfgLayer->FilePath[0]->attr.filter.c_str(), dimName.c_str(), dataSource); - } catch (int e) { - CDBError("Unable to create tableName from '%s' '%s' '%s'", cfgLayer->FilePath[0]->value.c_str(), cfgLayer->FilePath[0]->attr.filter.c_str(), dimName.c_str()); - return 1; - } - CDBFileScanner::markTableDirty(&tableName); + CDBFileScanner::markTableDirty(&m.second.tableName); // CDBDebug("Dropping old table (if exists)",tableName.c_str()); CT::string query; - query.print("drop table %s", tableName.c_str()); + query.print("drop table %s", m.second.tableName.c_str()); CDBDebug("Try to %s for %s", query.c_str(), dimName.c_str()); dataBaseConnection->query(query.c_str()); } @@ -651,148 +579,191 @@ int CDBAdapterPostgreSQL::autoUpdateAndScanDimensionTables(CDataSource *dataSour return 0; } -CT::string CDBAdapterPostgreSQL::getTableNameForPathFilterAndDimension(const char *path, const char *filter, const char *dimension, CDataSource *dataSource) { -#ifdef MEASURETIME - StopWatch_Stop(">CDBAdapterPostgreSQL::getTableNameForPathFilterAndDimension"); -#endif - if (dataSource->cfgLayer->DataBaseTable.size() == 1) { - CT::string tableName = ""; +CT::string CDBAdapterPostgreSQL::getLookupIdentifier(const char *path, const char *filter, const char *dimension) { + CT::string lookupIdentifier = "lookuptable/"; + lookupIdentifier.printconcat("%s/%s/%s", path, filter, dimension); + return lookupIdentifier; +} - tableName.concat(dataSource->cfgLayer->DataBaseTable[0]->value.c_str()); - CT::string dimName = ""; - if (dimension != NULL) { - dimName = dimension; - } +void CDBAdapterPostgreSQL::assertLookupTableExists() { + CPGSQLDB *DB = getDataBaseConnection(); + if (DB == NULL) { + CDBError("Unable to connect to DB"); + throw(1); + } - dataSource->srvParams->makeCorrectTableName(&tableName, &dimName); -#ifdef MEASURETIME - StopWatch_Stop("checkTable(CDBAdapterPostgreSQL_PATHFILTERTABLELOOKUP, tableColumns.c_str()); + if (status == 1) { + CDBError("FAIL: Table %s could not be created: %s", CDBAdapterPostgreSQL_PATHFILTERTABLELOOKUP, tableColumns.c_str()); + CDBError("Error: %s", DB->getError()); + throw(1); } +} - CT::string identifier = "lookuptable/"; - identifier.concat(path); - identifier.concat("/"); - identifier.concat(filter); - if (dimension != NULL) { - identifier.concat("/"); - identifier.concat(dimension); +void CDBAdapterPostgreSQL::addToLookupTable(const char *path, const char *filter, CT::string dimensionName, CT::string tableName) { + CPGSQLDB *DB = getDataBaseConnection(); + if (DB == NULL) { + CDBError("Unable to connect to DB"); + throw(1); } + + CT::string query; + query.print("INSERT INTO %s values (E'P_%s', E'F_%s', E'%s', E'%s')", CDBAdapterPostgreSQL_PATHFILTERTABLELOOKUP, path, filter, dimensionName.c_str(), tableName.c_str()); + int status = DB->query(query.c_str()); + if (status != 0) { + CDBError("Unable to insert records in lookup table: \"%s\"", query.c_str()); + throw(1); + } +} + +CT::string CDBAdapterPostgreSQL::generateRandomTableName() { CT::string tableName; + tableName.print("t%s_%s", CTime::currentDateTime().c_str(), CServerParams::randomString(20).c_str()); + tableName.replaceSelf(":", ""); + tableName.replaceSelf("-", ""); + tableName.replaceSelf("Z", ""); + tableName.toLowerCaseSelf(); - std::map::iterator it = lookupTableNameCacheMap.find(identifier.c_str()); - if (it != lookupTableNameCacheMap.end()) { - tableName = (*it).second.c_str(); -// CDBDebug("Returning tablename %s from map",tableName.c_str()); + return tableName; +} + +std::map CDBAdapterPostgreSQL::getTableNamesForPathFilterAndDimensions(const char *path, const char *filter, std::vector dimensions, CDataSource *dataSource) { #ifdef MEASURETIME - StopWatch_Stop("CDBAdapterPostgreSQL::getTableNamesForPathFilterAndDimensions"); #endif - return tableName; - } + /* + Given a path to a netcdf file, a filter, and a vector of requested dimensions, return a mapping of dimension name -> dimension table - // This makes use of a lookup table to find the tablename belonging to the filter and path combinations. - // Database collumns: path filter tablename + This information can be fetched from multiple areas, which means this function does the following: + - Use hardcoded database table name if it appears in the configuration + - Else, check if the requested dimension(s) appear(s) in the `lookupTableNameCache` + - If no mapping found for all requested dimensions, check if the requested dimensions are in the sql lookup table + - If the mapping is not complete, fill the lookuptable + */ - CT::string filterString = "F_"; - filterString.concat(filter); - CT::string pathString = "P_"; - pathString.concat(path); - CT::string dimString = ""; - if (dimension != NULL) { - dimString.concat(dimension); - dimString.toLowerCaseSelf(); + std::map mapping; + + // If config has a hardcoded db table name for layer, use it + if (dataSource->cfgLayer->DataBaseTable.size() == 1) { + CT::string tableName = dataSource->cfgLayer->DataBaseTable[0]->value.c_str(); + + for (auto &dim : dimensions) { + dataSource->srvParams->makeCorrectTableName(&tableName, &dim); + mapping[dim] = {tableName, ""}; + } +#ifdef MEASURETIME + StopWatch_Stop("::iterator it = lookupTableNameCacheMap.find(lookupIdentifier.c_str()); - CT::string lookupTableName = CDBAdapterPostgreSQL_PATHFILTERTABLELOOKUP; + if (it != lookupTableNameCacheMap.end()) { + mapping[dim].tableName = (*it).second.tableName.c_str(); + mapping[dim].dataType = (*it).second.dataType; + } else { + mapping[dim].tableName = ""; + done = false; + } + } + + // We've found all requested dimensions in the lookupTableNameCache. No need to update the lookup table, we're done! + if (done) { +#ifdef MEASURETIME + StopWatch_Stop("checkTable(lookupTableName.c_str(), tableColumns.c_str()); - // if(status == 0){CDBDebug("OK: Table %s is available",lookupTableName.c_str());} - if (status == 1) { - CDBError("FAIL: Table %s could not be created: %s", lookupTableName.c_str(), tableColumns.c_str()); - CDBError("Error: %s", DB->getError()); - throw(1); + // Select the tablename, dimension from the lookup table, and select the dimension's data type via postgres (similar to information_schema.columns except faster) + CT::string query; + query.print("SELECT p.tablename, p.dimension, (SELECT format_type(a.atttypid, a.atttypmod) AS data_type FROM pg_attribute a JOIN pg_class b ON (a.attrelid=b.relfilenode) WHERE " + "b.relname=p.tablename and a.attname=p.dimension and a.attstattarget=-1) FROM %s p WHERE path=E'P_%s' AND filter=E'F_%s' AND dimension IN (%s)", + CDBAdapterPostgreSQL_PATHFILTERTABLELOOKUP, path, filter, dimList.c_str()); + CDBStore::Store *tableDimStore = DB->queryToStore(query.c_str()); + + for (size_t i = 0; i < tableDimStore->size(); i++) { + CT::string dim = tableDimStore->getRecord(i)->get("dimension"); + + if (mapping[dim].tableName.length() == 0) { + mapping[dim].tableName = tableDimStore->getRecord(i)->get("tablename"); + mapping[dim].dataType = tableDimStore->getRecord(i)->get("data_type"); + + // Found tablename in SQL lookup table, also add to the lookupTableNameCacheMap + CT::string lookupIdentifier = getLookupIdentifier(path, filter, dim.c_str()); + DimInfo d = {mapping[dim].tableName, mapping[dim].dataType}; + lookupTableNameCacheMap[lookupIdentifier.c_str()] = d; } - // if(status == 2){CDBDebug("OK: Table %s is created",lookupTableName.c_str()); } - - // Check wether a records exists with this path and filter combination. + } - bool lookupTableIsAvailable = false; + // Found all tablenames for requested dimensions in lookup table + int found = tableDimStore->size(); + delete tableDimStore; + if (found == mapping.size()) { +#ifdef MEASURETIME + StopWatch_Stop(" 1) { - mvRecordQuery.print("SELECT * FROM %s where path=E'%s' and filter=E'%s' and dimension=E'%s'", lookupTableName.c_str(), pathString.c_str(), filterString.c_str(), dimString.c_str()); - } else { - mvRecordQuery.print("SELECT * FROM %s where path=E'%s' and filter=E'%s'", lookupTableName.c_str(), pathString.c_str(), filterString.c_str()); - } + // We're missing tables for the requested dimensions. Create the missing tables, and add to cache and lookup table + for (auto &m : mapping) { + if (!m.second.tableName.empty()) continue; - CDBStore::Store *rec = DB->queryToStore(mvRecordQuery.c_str()); - if (rec == NULL) { - CDBError("Unable to select records: \"%s\"", mvRecordQuery.c_str()); - throw(1); - } - if (rec->getSize() > 0) { - tableName.copy(rec->getRecord(0)->get(3)); - if (tableName.length() > 0) { - lookupTableIsAvailable = true; - } - } + CT::string tableName = generateRandomTableName(); + addToLookupTable(path, filter, m.first.c_str(), tableName.c_str()); + CT::string lookupIdentifier = getLookupIdentifier(path, filter, m.first.c_str()); + DimInfo d = {tableName, m.second.dataType}; + lookupTableNameCacheMap[lookupIdentifier.c_str()] = d; + mapping[m.first.c_str()] = d; + } - delete rec; +#ifdef MEASURETIME + StopWatch_Stop("CDBAdapterPostgreSQL::getTableNameForPathFilterAndDimension"); +#endif - tableName.copy(randomTableString.c_str()); - tableName.toLowerCaseSelf(); - mvRecordQuery.print("INSERT INTO %s values (E'%s',E'%s',E'%s',E'%s')", lookupTableName.c_str(), pathString.c_str(), filterString.c_str(), dimString.c_str(), tableName.c_str()); - // CDBDebug("%s",mvRecordQuery.c_str()); - status = DB->query(mvRecordQuery.c_str()); - if (status != 0) { - CDBError("Unable to insert records: \"%s\"", mvRecordQuery.c_str()); - throw(1); - } - } - // Close the database - } catch (int e) { - throw(e); - } + CT::string dimString(dimension); + dimString.toLowerCaseSelf(); + std::vector dims{dimString}; - if (tableName.length() > 0) { - // CDBDebug("Pushing %s with id %s",tableName.c_str(),identifier.c_str()); - lookupTableNameCacheMap.insert(std::pair(identifier.c_str(), tableName.c_str())); - } + std::map mapping = getTableNamesForPathFilterAndDimensions(path, filter, dims, dataSource); +#ifdef CDBAdapterPostgreSQL_DEBUG + CDBDebug("Using getTableNamesForPathFilterAndDimensions, found mapping %s -> %s", dimension, mapping[dimString].tableName.c_str()); +#endif - if (tableName.length() <= 0) { - CDBError("Unable to generate lookup table name for %s", identifier.c_str()); - throw(1); - } #ifdef MEASURETIME StopWatch_Stop(" lookupTableNameCacheMap; + std::map lookupTableNameCacheMap; std::map> fileListPerTable; int createDimTableOfType(const char *dimname, const char *tablename, int type); @@ -46,6 +52,13 @@ class CDBAdapterPostgreSQL : public CDBAdapter { CDBStore::Store *getClosestDataTimeToSystemTime(const char *netcdfDimName, const char *tableName); CT::string getTableNameForPathFilterAndDimension(const char *path, const char *filter, const char *dimension, CDataSource *dataSource); + std::map getTableNamesForPathFilterAndDimensions(const char *path, const char *filter, std::vector dimensions, CDataSource *dataSource); + + CT::string getLookupIdentifier(const char *path, const char *filter, const char *dimension); + void assertLookupTableExists(); + void addToLookupTable(const char *path, const char *filter, CT::string dimensionName, CT::string tableName); + CT::string generateRandomTableName(); + int autoUpdateAndScanDimensionTables(CDataSource *dataSource); CDBStore::Store *getMin(const char *name, const char *table); CDBStore::Store *getMax(const char *name, const char *table); diff --git a/adagucserverEC/CDBAdapterSQLLite.cpp b/adagucserverEC/CDBAdapterSQLLite.cpp index 78372afe..641a1a9d 100644 --- a/adagucserverEC/CDBAdapterSQLLite.cpp +++ b/adagucserverEC/CDBAdapterSQLLite.cpp @@ -183,7 +183,16 @@ CDBStore::Store *CDBAdapterSQLLite::CSQLLiteDB::queryToStore(const char *pszQuer for (size_t rowNumber = 0; rowNumber < numRows; rowNumber++) { CDBStore::Record *record = new CDBStore::Record(colModel); - for (size_t colNumber = 0; colNumber < numCols; colNumber++) record->push(colNumber, queryValues[colNumber + rowNumber * numCols].c_str()); + for (size_t colNumber = 0; colNumber < numCols; colNumber++) { + // SQlite always returns `10.0` if you insert `10.0` or `10`. PSQL always returns `10` + // HACK: to make output consistent with PSQL + CT::string s(queryValues[colNumber + rowNumber * numCols].c_str()); + if (s.endsWith(".0")) { + s.substringSelf(0, s.length() - 2); + } + + record->push(colNumber, s); + } store->push(record); } diff --git a/adagucserverEC/CDBFactory.cpp b/adagucserverEC/CDBFactory.cpp index 9ea015c4..e5f6ffb1 100644 --- a/adagucserverEC/CDBFactory.cpp +++ b/adagucserverEC/CDBFactory.cpp @@ -36,7 +36,7 @@ CDBAdapter *CDBFactory::getDBAdapter(CServerConfig::XMLE_Configuration *cfg) { CDBError("DataBase not properly configured"); exit(1); } - if (cfg->DataBase[0]->attr.dbtype.equals("sqlite")) { + if (cfg->DataBase[0]->attr.parameters.endsWith(".db")) { CDBDebug("Using sqlite"); #ifdef ADAGUC_USE_SQLITE staticCDBAdapter = new CDBAdapterSQLLite(); diff --git a/adagucserverEC/CRequest.cpp b/adagucserverEC/CRequest.cpp index 4abcfd35..4e615e4f 100644 --- a/adagucserverEC/CRequest.cpp +++ b/adagucserverEC/CRequest.cpp @@ -1177,6 +1177,20 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams } } + // Check if requested time dimensions use valid characters + for (auto &dim : dataSource->requiredDims) { + // FIXME: checkTimeFormat used to get called on every dim value, not just datetime. Check if this is required + if (!dim->isATimeDimension) continue; + + CT::string *dimValues = dim->value.splitToArray(","); + for (size_t i = 0; i < dimValues->count; i++) { + if (!CServerParams::checkTimeFormat(dimValues[i])) { + CDBError("Queried dimension %s=%s failed datetime regex", dim->name.c_str(), dim->value.c_str()); + throw InvalidDimensionValue; + } + } + } + // STOP NOW } catch (int i) { CDBError("%d", i); diff --git a/adagucserverEC/CServerConfig_CPPXSD.h b/adagucserverEC/CServerConfig_CPPXSD.h index 96b3224c..e0f51837 100644 --- a/adagucserverEC/CServerConfig_CPPXSD.h +++ b/adagucserverEC/CServerConfig_CPPXSD.h @@ -1335,15 +1335,12 @@ class CServerConfig : public CXMLSerializerInterface { public: class Cattr { public: - CT::string parameters, dbtype, maxquerylimit; + CT::string parameters, maxquerylimit; } attr; void addAttribute(const char *attrname, const char *attrvalue) { if (equals("parameters", attrname)) { attr.parameters.copy(attrvalue); return; - } else if (equals("dbtype", attrname)) { - attr.dbtype.copy(attrvalue); - return; } else if (equals("maxquerylimit", attrname)) { attr.maxquerylimit.copy(attrvalue); return; diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index 0958c919..74e33a56 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.26.0" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.27.0" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 diff --git a/data/config/adaguc.autoresource.xml b/data/config/adaguc.autoresource.xml index 2cd1a495..88a68eea 100644 --- a/data/config/adaguc.autoresource.xml +++ b/data/config/adaguc.autoresource.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.dataset.xml b/data/config/adaguc.dataset.xml index 4cc20009..4e300c8f 100644 --- a/data/config/adaguc.dataset.xml +++ b/data/config/adaguc.dataset.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.docker.xml b/data/config/adaguc.docker.xml index f54e4040..a0a97697 100644 --- a/data/config/adaguc.docker.xml +++ b/data/config/adaguc.docker.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.geojson.xml b/data/config/adaguc.geojson.xml index 316dbf8f..fc004a65 100644 --- a/data/config/adaguc.geojson.xml +++ b/data/config/adaguc.geojson.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.minimal.xml b/data/config/adaguc.minimal.xml index 8bf74833..c55ddce3 100644 --- a/data/config/adaguc.minimal.xml +++ b/data/config/adaguc.minimal.xml @@ -5,7 +5,7 @@ - + diff --git a/data/config/adaguc.ogcfeatures.xml b/data/config/adaguc.ogcfeatures.xml index 3b915526..85c24029 100644 --- a/data/config/adaguc.ogcfeatures.xml +++ b/data/config/adaguc.ogcfeatures.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.opendapserver.xml b/data/config/adaguc.opendapserver.xml index ef994ca5..610d9b15 100644 --- a/data/config/adaguc.opendapserver.xml +++ b/data/config/adaguc.opendapserver.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.poi.xml b/data/config/adaguc.poi.xml index a3d1bfac..153229a3 100644 --- a/data/config/adaguc.poi.xml +++ b/data/config/adaguc.poi.xml @@ -8,7 +8,7 @@ - + diff --git a/data/config/adaguc.sld.xml b/data/config/adaguc.sld.xml index 03307f33..e68a24e1 100644 --- a/data/config/adaguc.sld.xml +++ b/data/config/adaguc.sld.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.testdata.xml b/data/config/adaguc.testdata.xml index bc07e0d5..64429d4d 100644 --- a/data/config/adaguc.testdata.xml +++ b/data/config/adaguc.testdata.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.tests.autostyle.xml b/data/config/adaguc.tests.autostyle.xml index c7de797d..33b552b3 100644 --- a/data/config/adaguc.tests.autostyle.xml +++ b/data/config/adaguc.tests.autostyle.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.tests.dataset.xml b/data/config/adaguc.tests.dataset.xml index 56a92190..3bb62207 100644 --- a/data/config/adaguc.tests.dataset.xml +++ b/data/config/adaguc.tests.dataset.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.tests.datasetsmall.xml b/data/config/adaguc.tests.datasetsmall.xml index e08ee4ac..b1d42e0a 100644 --- a/data/config/adaguc.tests.datasetsmall.xml +++ b/data/config/adaguc.tests.datasetsmall.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.tests.manycontours.xml b/data/config/adaguc.tests.manycontours.xml index 05db150f..48b2d2e8 100644 --- a/data/config/adaguc.tests.manycontours.xml +++ b/data/config/adaguc.tests.manycontours.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.tiled.xml b/data/config/adaguc.tiled.xml index 24ff08aa..dbc731a0 100644 --- a/data/config/adaguc.tiled.xml +++ b/data/config/adaguc.tiled.xml @@ -3,8 +3,8 @@ - - + + diff --git a/data/config/adaguc.tiledworld.xml b/data/config/adaguc.tiledworld.xml index 3a287b0d..68e3ca86 100644 --- a/data/config/adaguc.tiledworld.xml +++ b/data/config/adaguc.tiledworld.xml @@ -3,8 +3,8 @@ - - + + diff --git a/data/config/adaguc.tiling.xml b/data/config/adaguc.tiling.xml index 529b1e6a..0dcba18b 100644 --- a/data/config/adaguc.tiling.xml +++ b/data/config/adaguc.tiling.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.timeseries.xml b/data/config/adaguc.timeseries.xml index 984afa8d..4ce02d5f 100644 --- a/data/config/adaguc.timeseries.xml +++ b/data/config/adaguc.timeseries.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.vm-datasets.xml b/data/config/adaguc.vm-datasets.xml index 26fde4bf..ddef9523 100644 --- a/data/config/adaguc.vm-datasets.xml +++ b/data/config/adaguc.vm-datasets.xml @@ -3,7 +3,7 @@ - + diff --git a/data/config/adaguc.vm.xml b/data/config/adaguc.vm.xml index 7fea3978..c970041e 100644 --- a/data/config/adaguc.vm.xml +++ b/data/config/adaguc.vm.xml @@ -3,7 +3,7 @@ - + diff --git a/doc/configuration/Configuration.md b/doc/configuration/Configuration.md index 1f18781b..bfc72e1a 100644 --- a/doc/configuration/Configuration.md +++ b/doc/configuration/Configuration.md @@ -16,7 +16,7 @@ Configuration - TempDir (value) - Mandatory, directory where cache-files are stored. - [OnlineResource](OnlineResource.md) (value) - Mandatory, the external address of the server, usually ends with a ? token. -- [DataBase](DataBase.md) (dbtype,parameters) - Mandatory, parameters is the +- [DataBase](DataBase.md) (parameters) - Mandatory, parameters is the attribute where database settings can be configured. - CacheDocs (enabled) - Defaults to false. Set enabled to "true" to enable caching of GetCapabilities documents. diff --git a/doc/configuration/DataBase.md b/doc/configuration/DataBase.md index 1862b530..372c87f1 100644 --- a/doc/configuration/DataBase.md +++ b/doc/configuration/DataBase.md @@ -1,4 +1,4 @@ -DataBase (dbtype,parameters, maxquerylimit) +DataBase (parameters, maxquerylimit) =========================================== Back to [Configuration](./Configuration.md) @@ -10,12 +10,11 @@ Configuration to access a PostgreSQL or SQLite database, for example: ``` ```xml - + ``` -- dbtype can be either sqlite or postgresql. Default is postgresl -- parameters are the database parameters required for making the - connection -- maxquerylimit is the maximum number of results for a getfeatureinfo +- `parameters` are the database parameters required for making the + connection. If the `parameters` string ends with `.db`, we assume adaguc should use `sqlite`. +- `maxquerylimit` is the maximum number of results for a getfeatureinfo call. diff --git a/doc/developing/testing.md b/doc/developing/testing.md new file mode 100644 index 00000000..37b574ed --- /dev/null +++ b/doc/developing/testing.md @@ -0,0 +1,70 @@ +# Testing Adaguc-server + +Adaguc contains two types of tests: + +- Tests specifically for adaguc-server +- Tests specifically for the Python wrapper/EDR functionality + +During the tests, adaguc can use either the SQLite or PostgreSQL backend. + +## Running tests in Docker via SQLite + +The main [Dockerfile](~/Dockerfile) contains a _test stage_ which executes `runtests.sh`. This means that all tests will be executed during a regular docker build. Running the tests in Docker is recommended especially for users that don't run Linux/amd64. + +To run all tests, trigger a build: +```bash +docker build --progress plain -t adaguc-server . +``` + +The build will fail if any of the tests have failed. + + +## Running tests outside Docker via SQLite + +Running the tests outside of Docker is possible by executing the `./runtests.sh` script. + +> **⚠️ Note**: some tests might fail due to platform architecture differences between amd64 and arm64. + +## Using postgres + +To run the tests against Postgres, the following needs to happen: + +- The adaguc XML configuration files used by tests all require a `` section. This gets handled by `starttests_psql.sh` +- PostgreSQL must be reachable on port `5432`. + +### Via Docker + +To do this through Docker, follow these steps: +1. Edit `docker-compose.yml` +2. Comment out all servers that are not `adaguc-db` +3. Add the `ports` section to the `adaguc-db` service to expose port 5432: +```yaml + adaguc-db: + (...) + ports: + - 5432:5432 +``` +4. Start postgres while inside the `./Docker` directory: +```bash +docker compose up -d +``` +5. Edit `./Dockerfile`, to enable the `runtests_psql.sh`: +```Dockerfile +# RUN bash runtests.sh +RUN bash runtests_psql.sh +``` +6. Trigger a new build: +```bash +docker build --progress plain -t adaguc-server . +``` + +All tests should now run against postgres via the docker build. + +### Without Docker + +To run the tests outside of Docker, make sure Postgres is reachable on port 5432, and then execute the test script directly: +```bash +./runtests_psql.sh +``` + +> **⚠️ Note**: some tests might fail due to platform architecture differences between amd64 and arm64. \ No newline at end of file diff --git a/python/lib/adaguc/AdagucTestTools.py b/python/lib/adaguc/AdagucTestTools.py index 8225acd1..6cb6ca2a 100644 --- a/python/lib/adaguc/AdagucTestTools.py +++ b/python/lib/adaguc/AdagucTestTools.py @@ -7,6 +7,7 @@ from lxml import etree, objectify import urllib.request from PIL import Image +import subprocess ADAGUC_PATH = os.getenv("ADAGUC_PATH", " ") @@ -142,6 +143,22 @@ def cleanTempDir(self): self.mkdir_p(ADAGUC_TMP) return + def cleanPostgres(self): + """Clean the postgres database if the ADAGUC_DB variable has been set. + + Some tests fail when using postgres if there is already data present in the database. + Running the test separately works.""" + + if adaguc_db := os.getenv("ADAGUC_DB", None): + subprocess.run( + [ + "psql", + adaguc_db, + "-c", + "DROP SCHEMA public CASCADE; CREATE SCHEMA public;", + ] + ) + def mkdir_p(self, directory): if not os.path.exists(directory): os.makedirs(directory) diff --git a/python/lib/adaguc/adaguc-server-config-python.xml b/python/lib/adaguc/adaguc-server-config-python.xml index f183424b..d319f099 100644 --- a/python/lib/adaguc/adaguc-server-config-python.xml +++ b/python/lib/adaguc/adaguc-server-config-python.xml @@ -3,7 +3,7 @@ - + diff --git a/python/python_fastapi_server/test_ogc_api_edr.py b/python/python_fastapi_server/test_ogc_api_edr.py index f8e78c74..5ce7a3f0 100644 --- a/python/python_fastapi_server/test_ogc_api_edr.py +++ b/python/python_fastapi_server/test_ogc_api_edr.py @@ -20,6 +20,7 @@ def set_environ(): def setup_test_data(): print("About to ingest data") AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() for service in [ "netcdf_5d.xml", "dataset_a.xml", @@ -180,6 +181,50 @@ def test_coll_multi_dim_position(client: TestClient): 330000, ] + # Should handle querying multiple heights separated by comma + resp = client.get( + "/edr/collections/testcollection/instances/2024060100/position?coords=POINT(5.2 52.0)&datetime=2024-06-01T01:00:00Z/2024-06-01T04:00:00Z¶meter-name=testdata&z=10,20" + ) + assert resp.status_code, 200 + covjson = resp.json() + + assert covjson["domain"]["axes"]["z"]["values"] == [10, 20] + assert covjson["ranges"]["testdata"]["shape"] == [1, 1, 2, 4] + assert covjson["ranges"]["testdata"]["values"] == [ + 0.0, + 10000.0, + 20000.0, + 30000.0, + 100000.0, + 110000.0, + 120000.0, + 130000.0, + ] + + # Should handle querying multiple heights separated by comma, combined with querying ranges + resp = client.get( + "/edr/collections/testcollection/instances/2024060100/position?coords=POINT(5.2 52.0)&datetime=2024-06-01T01:00:00Z/2024-06-01T04:00:00Z¶meter-name=testdata&z=10,30/40" + ) + assert resp.status_code, 200 + covjson = resp.json() + + assert covjson["domain"]["axes"]["z"]["values"] == [10, 30, 40] + assert covjson["ranges"]["testdata"]["shape"] == [1, 1, 3, 4] + assert covjson["ranges"]["testdata"]["values"] == [ + 0.0, + 10000.0, + 20000.0, + 30000.0, + 200000.0, + 210000.0, + 220000.0, + 230000.0, + 300000.0, + 310000.0, + 320000.0, + 330000.0, + ] + def test_coll_multi_dim_cube(client: TestClient): resp = client.get( diff --git a/python/python_fastapi_server/test_ogc_api_features.py b/python/python_fastapi_server/test_ogc_api_features.py index 669d76d3..d1067923 100644 --- a/python/python_fastapi_server/test_ogc_api_features.py +++ b/python/python_fastapi_server/test_ogc_api_features.py @@ -20,6 +20,7 @@ def set_environ(): def setup_test_data(): print("About to ingest data") AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() for service in ["netcdf_5d.xml", "dataset_a.xml"]: _status, _data, _headers = AdagucTestTools().runADAGUCServer( args=[ diff --git a/runtests_psql.sh b/runtests_psql.sh new file mode 100755 index 00000000..4a61ba6a --- /dev/null +++ b/runtests_psql.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd tests +bash starttests_psql.sh diff --git a/tests/AdagucTests/TestWMS.py b/tests/AdagucTests/TestWMS.py index 106de94c..8a5052b8 100644 --- a/tests/AdagucTests/TestWMS.py +++ b/tests/AdagucTests/TestWMS.py @@ -347,6 +347,7 @@ def test_WMSCMDUpdateDB(self): def test_WMSCMDUpdateDBTailPath(self): AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() # pylint: disable=unused-variable status, data, headers = AdagucTestTools().runADAGUCServer( args=[ @@ -404,6 +405,7 @@ def test_WMSCMDUpdateDBTailPath(self): def test_WMSCMDUpdateDBPath(self): AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() # pylint: disable=unused-variable status, data, headers = AdagucTestTools().runADAGUCServer( args=[ @@ -2021,6 +2023,7 @@ def test_WMSCMDUpdateDBPathFileWithNonMatchingPath(self): This tests if the autofinddataset option correctly finds the file if it is in a subfolder """ AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() # pylint: disable=unused-variable config = ADAGUC_PATH + "data/config/adaguc.tests.dataset.xml" diff --git a/tests/AdagucTests/TestWMSDocumentCache.py b/tests/AdagucTests/TestWMSDocumentCache.py index 728d9f0b..82d73d69 100644 --- a/tests/AdagucTests/TestWMSDocumentCache.py +++ b/tests/AdagucTests/TestWMSDocumentCache.py @@ -41,6 +41,8 @@ def test_WMSCMDUpdateDB(self): def test_WMSCMDUpdateDBTailPath(self): AdagucTestTools().cleanTempDir() + AdagucTestTools().cleanPostgres() + ADAGUC_PATH = os.environ['ADAGUC_PATH'] config = ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml,' + \ ADAGUC_PATH + '/data/config/datasets/adaguc.testtimeseriescached.xml' diff --git a/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetFeatureInfo.json b/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetFeatureInfo.json index 16ddef34..dda0f820 100644 --- a/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetFeatureInfo.json +++ b/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetFeatureInfo.json @@ -1 +1 @@ -[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T22:00:00Z":"16.701019"}}}] \ No newline at end of file +[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T22:00:00Z":"16.701019"}}}] \ No newline at end of file diff --git a/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetTimeSeries.json b/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetTimeSeries.json index 27d4c154..446b410a 100644 --- a/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetTimeSeries.json +++ b/tests/expectedoutputs/TestDataPostProcessor/test_DataPostProcessor_WindShear_GetTimeSeries.json @@ -1 +1 @@ -[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"-0.203398","2019-01-01T20:00:00Z":"-0.221830","2019-01-01T21:00:00Z":"-0.240139","2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"17.272943","2019-01-01T20:00:00Z":"17.284334","2019-01-01T21:00:00Z":"16.998619","2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600.0":{"2019-01-01T19:00:00Z":"17.069546","2019-01-01T20:00:00Z":"17.062504","2019-01-01T21:00:00Z":"16.758480","2019-01-01T22:00:00Z":"16.701019"}}}] \ No newline at end of file +[{"name":"output","standard_name":"output","feature_name":"output_0","units":"m/s","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"-0.203398","2019-01-01T20:00:00Z":"-0.221830","2019-01-01T21:00:00Z":"-0.240139","2019-01-01T22:00:00Z":"-0.195704"}}},{"name":"var_wind_speed_at_600","standard_name":"wind_speed","feature_name":"wind_speed_1","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"17.272943","2019-01-01T20:00:00Z":"17.284334","2019-01-01T21:00:00Z":"16.998619","2019-01-01T22:00:00Z":"16.896723"}}},{"name":"var_wind_speed_at_200","standard_name":"wind_speed","feature_name":"wind_speed_2","units":"m s-1","point":{"SRS":"EPSG:4326","coords":"3.256556,52.592725"},"dims":["elevation","time"],"data":{"600":{"2019-01-01T19:00:00Z":"17.069546","2019-01-01T20:00:00Z":"17.062504","2019-01-01T21:00:00Z":"16.758480","2019-01-01T22:00:00Z":"16.701019"}}}] \ No newline at end of file diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_DimensionUnits.xml b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_DimensionUnits.xml index 49aae689..548f9701 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_DimensionUnits.xml +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_DimensionUnits.xml @@ -123,7 +123,7 @@ 2023-10-26T05:00:00Z -10.0 +10 2023-10-26T00:00:00Z diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimnc_autostyle.xml b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimnc_autostyle.xml index 2c5f7f0b..a9bb574e 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimnc_autostyle.xml +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimnc_autostyle.xml @@ -127,7 +127,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z,2017-01-01T00:05:00Z,2017-01-01T00:10:00Z diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimncdataset_autostyle.xml b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimncdataset_autostyle.xml index d8fe81e6..826d6343 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimncdataset_autostyle.xml +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_multidimncdataset_autostyle.xml @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq1 b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq1 index 818b983d..6b328657 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq1 +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq1 @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z,2017-01-01T00:05:00Z,2017-01-01T00:10:00Z diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq2 b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq2 index a73c2a3e..74beb35f 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq2 +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_path_netcdf_5dims_seq2 @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 index 818b983d..6b328657 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z,2017-01-01T00:05:00Z,2017-01-01T00:10:00Z diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 index a73c2a3e..c923c0c3 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 @@ -19,7 +19,7 @@ - + no conditions apply None @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_twofiles b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_twofiles index a73c2a3e..74beb35f 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_twofiles +++ b/tests/expectedoutputs/TestWMS/test_WMSGetCapabilities_timeseries_twofiles @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_1Dimensional_latlon_nextdimensionstep_getcapabilities.xml b/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_1Dimensional_latlon_nextdimensionstep_getcapabilities.xml index cf08197b..cf7638e4 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_1Dimensional_latlon_nextdimensionstep_getcapabilities.xml +++ b/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_1Dimensional_latlon_nextdimensionstep_getcapabilities.xml @@ -119,7 +119,7 @@ -0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 +0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 2017-01-01T00:10:00Z,2017-01-01T00:11:00Z diff --git a/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_2Dimensional_latlon_nextdimensionstep_getcapabilities.xml b/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_2Dimensional_latlon_nextdimensionstep_getcapabilities.xml index d2abdc9e..3aa6b8c1 100644 --- a/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_2Dimensional_latlon_nextdimensionstep_getcapabilities.xml +++ b/tests/expectedoutputs/TestWMS/test_WMSGetMap_IrregularGrid_2Dimensional_latlon_nextdimensionstep_getcapabilities.xml @@ -119,7 +119,7 @@ -0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 +0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9 2017-01-01T00:10:00Z,2017-01-01T00:11:00Z diff --git a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 index e2b4ffdc..9a22324d 100644 --- a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 +++ b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1 @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z,2017-01-01T00:05:00Z,2017-01-01T00:10:00Z diff --git a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 index a6f03911..51d105d3 100644 --- a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 +++ b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_tailpath_netcdf_5dims_seq1_and_seq2 @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_twofiles b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_twofiles index a6f03911..51d105d3 100644 --- a/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_twofiles +++ b/tests/expectedoutputs/TestWMSDocumentCache/test_WMSGetCapabilities_timeseries_twofiles @@ -123,7 +123,7 @@ member6,member5,member4,member3,member2,member1 -1000.0,2000.0,3000.0,4000.0,5000.0,6000.0,7000.0,8000.0,9000.0 +1000,2000,3000,4000,5000,6000,7000,8000,9000 2017-01-01T00:00:00Z/2017-01-01T00:25:00Z/PT5M diff --git a/tests/expectedoutputs/TestWMSTimeSeries/test_WMSGetFeatureInfo_timeseries_5dims_json.json b/tests/expectedoutputs/TestWMSTimeSeries/test_WMSGetFeatureInfo_timeseries_5dims_json.json index bc89c2b3..346a726e 100644 --- a/tests/expectedoutputs/TestWMSTimeSeries/test_WMSGetFeatureInfo_timeseries_5dims_json.json +++ b/tests/expectedoutputs/TestWMSTimeSeries/test_WMSGetFeatureInfo_timeseries_5dims_json.json @@ -1 +1 @@ -[{"name":"data","standard_name":"data","units":"km","point":{"SRS":"EPSG:4326","coords":"177.454547,82.135346"},"dims":["member","elevation","time"],"data":{"member1":{"1000.0":{"2017-01-01T00:00:00Z":"0.000000","2017-01-01T00:05:00Z":"1.000000","2017-01-01T00:10:00Z":"2.000000","2017-01-01T00:15:00Z":"3.000000","2017-01-01T00:20:00Z":"4.000000","2017-01-01T00:25:00Z":"5.000000"},"2000.0":{"2017-01-01T00:00:00Z":"6.000000","2017-01-01T00:05:00Z":"7.000000","2017-01-01T00:10:00Z":"8.000000","2017-01-01T00:15:00Z":"9.000000","2017-01-01T00:20:00Z":"10.000000","2017-01-01T00:25:00Z":"11.000000"},"3000.0":{"2017-01-01T00:00:00Z":"12.000000","2017-01-01T00:05:00Z":"13.000000","2017-01-01T00:10:00Z":"14.000000","2017-01-01T00:15:00Z":"15.000000","2017-01-01T00:20:00Z":"16.000000","2017-01-01T00:25:00Z":"17.000000"},"4000.0":{"2017-01-01T00:00:00Z":"18.000000","2017-01-01T00:05:00Z":"19.000000","2017-01-01T00:10:00Z":"20.000000","2017-01-01T00:15:00Z":"21.000000","2017-01-01T00:20:00Z":"22.000000","2017-01-01T00:25:00Z":"23.000000"},"5000.0":{"2017-01-01T00:00:00Z":"24.000000","2017-01-01T00:05:00Z":"25.000000","2017-01-01T00:10:00Z":"26.000000","2017-01-01T00:15:00Z":"27.000000","2017-01-01T00:20:00Z":"28.000000","2017-01-01T00:25:00Z":"29.000000"},"6000.0":{"2017-01-01T00:00:00Z":"30.000000","2017-01-01T00:05:00Z":"31.000000","2017-01-01T00:10:00Z":"32.000000","2017-01-01T00:15:00Z":"33.000000","2017-01-01T00:20:00Z":"34.000000","2017-01-01T00:25:00Z":"35.000000"},"7000.0":{"2017-01-01T00:00:00Z":"36.000000","2017-01-01T00:05:00Z":"37.000000","2017-01-01T00:10:00Z":"38.000000","2017-01-01T00:15:00Z":"39.000000","2017-01-01T00:20:00Z":"40.000000","2017-01-01T00:25:00Z":"41.000000"},"8000.0":{"2017-01-01T00:00:00Z":"42.000000","2017-01-01T00:05:00Z":"43.000000","2017-01-01T00:10:00Z":"44.000000","2017-01-01T00:15:00Z":"45.000000","2017-01-01T00:20:00Z":"46.000000","2017-01-01T00:25:00Z":"47.000000"},"9000.0":{"2017-01-01T00:00:00Z":"48.000000","2017-01-01T00:05:00Z":"49.000000","2017-01-01T00:10:00Z":"50.000000","2017-01-01T00:15:00Z":"51.000000","2017-01-01T00:20:00Z":"52.000000","2017-01-01T00:25:00Z":"53.000000"}},"member2":{"1000.0":{"2017-01-01T00:00:00Z":"54.000000","2017-01-01T00:05:00Z":"55.000000","2017-01-01T00:10:00Z":"56.000000","2017-01-01T00:15:00Z":"57.000000","2017-01-01T00:20:00Z":"58.000000","2017-01-01T00:25:00Z":"59.000000"},"2000.0":{"2017-01-01T00:00:00Z":"60.000000","2017-01-01T00:05:00Z":"61.000000","2017-01-01T00:10:00Z":"62.000000","2017-01-01T00:15:00Z":"63.000000","2017-01-01T00:20:00Z":"64.000000","2017-01-01T00:25:00Z":"65.000000"},"3000.0":{"2017-01-01T00:00:00Z":"66.000000","2017-01-01T00:05:00Z":"67.000000","2017-01-01T00:10:00Z":"68.000000","2017-01-01T00:15:00Z":"69.000000","2017-01-01T00:20:00Z":"70.000000","2017-01-01T00:25:00Z":"71.000000"},"4000.0":{"2017-01-01T00:00:00Z":"72.000000","2017-01-01T00:05:00Z":"73.000000","2017-01-01T00:10:00Z":"74.000000","2017-01-01T00:15:00Z":"75.000000","2017-01-01T00:20:00Z":"76.000000","2017-01-01T00:25:00Z":"77.000000"},"5000.0":{"2017-01-01T00:00:00Z":"78.000000","2017-01-01T00:05:00Z":"79.000000","2017-01-01T00:10:00Z":"80.000000","2017-01-01T00:15:00Z":"81.000000","2017-01-01T00:20:00Z":"82.000000","2017-01-01T00:25:00Z":"83.000000"},"6000.0":{"2017-01-01T00:00:00Z":"84.000000","2017-01-01T00:05:00Z":"85.000000","2017-01-01T00:10:00Z":"86.000000","2017-01-01T00:15:00Z":"87.000000","2017-01-01T00:20:00Z":"88.000000","2017-01-01T00:25:00Z":"89.000000"},"7000.0":{"2017-01-01T00:00:00Z":"90.000000","2017-01-01T00:05:00Z":"91.000000","2017-01-01T00:10:00Z":"92.000000","2017-01-01T00:15:00Z":"93.000000","2017-01-01T00:20:00Z":"94.000000","2017-01-01T00:25:00Z":"95.000000"},"8000.0":{"2017-01-01T00:00:00Z":"96.000000","2017-01-01T00:05:00Z":"97.000000","2017-01-01T00:10:00Z":"98.000000","2017-01-01T00:15:00Z":"99.000000","2017-01-01T00:20:00Z":"100.000000","2017-01-01T00:25:00Z":"101.000000"},"9000.0":{"2017-01-01T00:00:00Z":"102.000000","2017-01-01T00:05:00Z":"103.000000","2017-01-01T00:10:00Z":"104.000000","2017-01-01T00:15:00Z":"105.000000","2017-01-01T00:20:00Z":"106.000000","2017-01-01T00:25:00Z":"107.000000"}},"member3":{"1000.0":{"2017-01-01T00:00:00Z":"108.000000","2017-01-01T00:05:00Z":"109.000000","2017-01-01T00:10:00Z":"110.000000","2017-01-01T00:15:00Z":"111.000000","2017-01-01T00:20:00Z":"112.000000","2017-01-01T00:25:00Z":"113.000000"},"2000.0":{"2017-01-01T00:00:00Z":"114.000000","2017-01-01T00:05:00Z":"115.000000","2017-01-01T00:10:00Z":"116.000000","2017-01-01T00:15:00Z":"117.000000","2017-01-01T00:20:00Z":"118.000000","2017-01-01T00:25:00Z":"119.000000"},"3000.0":{"2017-01-01T00:00:00Z":"120.000000","2017-01-01T00:05:00Z":"121.000000","2017-01-01T00:10:00Z":"122.000000","2017-01-01T00:15:00Z":"123.000000","2017-01-01T00:20:00Z":"124.000000","2017-01-01T00:25:00Z":"125.000000"},"4000.0":{"2017-01-01T00:00:00Z":"126.000000","2017-01-01T00:05:00Z":"127.000000","2017-01-01T00:10:00Z":"128.000000","2017-01-01T00:15:00Z":"129.000000","2017-01-01T00:20:00Z":"130.000000","2017-01-01T00:25:00Z":"131.000000"},"5000.0":{"2017-01-01T00:00:00Z":"132.000000","2017-01-01T00:05:00Z":"133.000000","2017-01-01T00:10:00Z":"134.000000","2017-01-01T00:15:00Z":"135.000000","2017-01-01T00:20:00Z":"136.000000","2017-01-01T00:25:00Z":"137.000000"},"6000.0":{"2017-01-01T00:00:00Z":"138.000000","2017-01-01T00:05:00Z":"139.000000","2017-01-01T00:10:00Z":"140.000000","2017-01-01T00:15:00Z":"141.000000","2017-01-01T00:20:00Z":"142.000000","2017-01-01T00:25:00Z":"143.000000"},"7000.0":{"2017-01-01T00:00:00Z":"144.000000","2017-01-01T00:05:00Z":"145.000000","2017-01-01T00:10:00Z":"146.000000","2017-01-01T00:15:00Z":"147.000000","2017-01-01T00:20:00Z":"148.000000","2017-01-01T00:25:00Z":"149.000000"},"8000.0":{"2017-01-01T00:00:00Z":"150.000000","2017-01-01T00:05:00Z":"151.000000","2017-01-01T00:10:00Z":"152.000000","2017-01-01T00:15:00Z":"153.000000","2017-01-01T00:20:00Z":"154.000000","2017-01-01T00:25:00Z":"155.000000"},"9000.0":{"2017-01-01T00:00:00Z":"156.000000","2017-01-01T00:05:00Z":"157.000000","2017-01-01T00:10:00Z":"158.000000","2017-01-01T00:15:00Z":"159.000000","2017-01-01T00:20:00Z":"160.000000","2017-01-01T00:25:00Z":"161.000000"}},"member4":{"1000.0":{"2017-01-01T00:00:00Z":"162.000000","2017-01-01T00:05:00Z":"163.000000","2017-01-01T00:10:00Z":"164.000000","2017-01-01T00:15:00Z":"165.000000","2017-01-01T00:20:00Z":"166.000000","2017-01-01T00:25:00Z":"167.000000"},"2000.0":{"2017-01-01T00:00:00Z":"168.000000","2017-01-01T00:05:00Z":"169.000000","2017-01-01T00:10:00Z":"170.000000","2017-01-01T00:15:00Z":"171.000000","2017-01-01T00:20:00Z":"172.000000","2017-01-01T00:25:00Z":"173.000000"},"3000.0":{"2017-01-01T00:00:00Z":"174.000000","2017-01-01T00:05:00Z":"175.000000","2017-01-01T00:10:00Z":"176.000000","2017-01-01T00:15:00Z":"177.000000","2017-01-01T00:20:00Z":"178.000000","2017-01-01T00:25:00Z":"179.000000"},"4000.0":{"2017-01-01T00:00:00Z":"180.000000","2017-01-01T00:05:00Z":"181.000000","2017-01-01T00:10:00Z":"182.000000","2017-01-01T00:15:00Z":"183.000000","2017-01-01T00:20:00Z":"184.000000","2017-01-01T00:25:00Z":"185.000000"},"5000.0":{"2017-01-01T00:00:00Z":"186.000000","2017-01-01T00:05:00Z":"187.000000","2017-01-01T00:10:00Z":"188.000000","2017-01-01T00:15:00Z":"189.000000","2017-01-01T00:20:00Z":"190.000000","2017-01-01T00:25:00Z":"191.000000"},"6000.0":{"2017-01-01T00:00:00Z":"192.000000","2017-01-01T00:05:00Z":"193.000000","2017-01-01T00:10:00Z":"194.000000","2017-01-01T00:15:00Z":"195.000000","2017-01-01T00:20:00Z":"196.000000","2017-01-01T00:25:00Z":"197.000000"},"7000.0":{"2017-01-01T00:00:00Z":"198.000000","2017-01-01T00:05:00Z":"199.000000","2017-01-01T00:10:00Z":"200.000000","2017-01-01T00:15:00Z":"201.000000","2017-01-01T00:20:00Z":"202.000000","2017-01-01T00:25:00Z":"203.000000"},"8000.0":{"2017-01-01T00:00:00Z":"204.000000","2017-01-01T00:05:00Z":"205.000000","2017-01-01T00:10:00Z":"206.000000","2017-01-01T00:15:00Z":"207.000000","2017-01-01T00:20:00Z":"208.000000","2017-01-01T00:25:00Z":"209.000000"},"9000.0":{"2017-01-01T00:00:00Z":"210.000000","2017-01-01T00:05:00Z":"211.000000","2017-01-01T00:10:00Z":"212.000000","2017-01-01T00:15:00Z":"213.000000","2017-01-01T00:20:00Z":"214.000000","2017-01-01T00:25:00Z":"215.000000"}},"member5":{"1000.0":{"2017-01-01T00:00:00Z":"216.000000","2017-01-01T00:05:00Z":"217.000000","2017-01-01T00:10:00Z":"218.000000","2017-01-01T00:15:00Z":"219.000000","2017-01-01T00:20:00Z":"220.000000","2017-01-01T00:25:00Z":"221.000000"},"2000.0":{"2017-01-01T00:00:00Z":"222.000000","2017-01-01T00:05:00Z":"223.000000","2017-01-01T00:10:00Z":"224.000000","2017-01-01T00:15:00Z":"225.000000","2017-01-01T00:20:00Z":"226.000000","2017-01-01T00:25:00Z":"227.000000"},"3000.0":{"2017-01-01T00:00:00Z":"228.000000","2017-01-01T00:05:00Z":"229.000000","2017-01-01T00:10:00Z":"230.000000","2017-01-01T00:15:00Z":"231.000000","2017-01-01T00:20:00Z":"232.000000","2017-01-01T00:25:00Z":"233.000000"},"4000.0":{"2017-01-01T00:00:00Z":"234.000000","2017-01-01T00:05:00Z":"235.000000","2017-01-01T00:10:00Z":"236.000000","2017-01-01T00:15:00Z":"237.000000","2017-01-01T00:20:00Z":"238.000000","2017-01-01T00:25:00Z":"239.000000"},"5000.0":{"2017-01-01T00:00:00Z":"240.000000","2017-01-01T00:05:00Z":"241.000000","2017-01-01T00:10:00Z":"242.000000","2017-01-01T00:15:00Z":"243.000000","2017-01-01T00:20:00Z":"244.000000","2017-01-01T00:25:00Z":"245.000000"},"6000.0":{"2017-01-01T00:00:00Z":"246.000000","2017-01-01T00:05:00Z":"247.000000","2017-01-01T00:10:00Z":"248.000000","2017-01-01T00:15:00Z":"249.000000","2017-01-01T00:20:00Z":"250.000000","2017-01-01T00:25:00Z":"251.000000"},"7000.0":{"2017-01-01T00:00:00Z":"252.000000","2017-01-01T00:05:00Z":"253.000000","2017-01-01T00:10:00Z":"254.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"8000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"9000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"}},"member6":{"1000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"2000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"3000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"4000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"5000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"6000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"7000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"8000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"9000.0":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"}}}}] +[{"name":"data","standard_name":"data","units":"km","point":{"SRS":"EPSG:4326","coords":"177.454547,82.135346"},"dims":["member","elevation","time"],"data":{"member1":{"1000":{"2017-01-01T00:00:00Z":"0.000000","2017-01-01T00:05:00Z":"1.000000","2017-01-01T00:10:00Z":"2.000000","2017-01-01T00:15:00Z":"3.000000","2017-01-01T00:20:00Z":"4.000000","2017-01-01T00:25:00Z":"5.000000"},"2000":{"2017-01-01T00:00:00Z":"6.000000","2017-01-01T00:05:00Z":"7.000000","2017-01-01T00:10:00Z":"8.000000","2017-01-01T00:15:00Z":"9.000000","2017-01-01T00:20:00Z":"10.000000","2017-01-01T00:25:00Z":"11.000000"},"3000":{"2017-01-01T00:00:00Z":"12.000000","2017-01-01T00:05:00Z":"13.000000","2017-01-01T00:10:00Z":"14.000000","2017-01-01T00:15:00Z":"15.000000","2017-01-01T00:20:00Z":"16.000000","2017-01-01T00:25:00Z":"17.000000"},"4000":{"2017-01-01T00:00:00Z":"18.000000","2017-01-01T00:05:00Z":"19.000000","2017-01-01T00:10:00Z":"20.000000","2017-01-01T00:15:00Z":"21.000000","2017-01-01T00:20:00Z":"22.000000","2017-01-01T00:25:00Z":"23.000000"},"5000":{"2017-01-01T00:00:00Z":"24.000000","2017-01-01T00:05:00Z":"25.000000","2017-01-01T00:10:00Z":"26.000000","2017-01-01T00:15:00Z":"27.000000","2017-01-01T00:20:00Z":"28.000000","2017-01-01T00:25:00Z":"29.000000"},"6000":{"2017-01-01T00:00:00Z":"30.000000","2017-01-01T00:05:00Z":"31.000000","2017-01-01T00:10:00Z":"32.000000","2017-01-01T00:15:00Z":"33.000000","2017-01-01T00:20:00Z":"34.000000","2017-01-01T00:25:00Z":"35.000000"},"7000":{"2017-01-01T00:00:00Z":"36.000000","2017-01-01T00:05:00Z":"37.000000","2017-01-01T00:10:00Z":"38.000000","2017-01-01T00:15:00Z":"39.000000","2017-01-01T00:20:00Z":"40.000000","2017-01-01T00:25:00Z":"41.000000"},"8000":{"2017-01-01T00:00:00Z":"42.000000","2017-01-01T00:05:00Z":"43.000000","2017-01-01T00:10:00Z":"44.000000","2017-01-01T00:15:00Z":"45.000000","2017-01-01T00:20:00Z":"46.000000","2017-01-01T00:25:00Z":"47.000000"},"9000":{"2017-01-01T00:00:00Z":"48.000000","2017-01-01T00:05:00Z":"49.000000","2017-01-01T00:10:00Z":"50.000000","2017-01-01T00:15:00Z":"51.000000","2017-01-01T00:20:00Z":"52.000000","2017-01-01T00:25:00Z":"53.000000"}},"member2":{"1000":{"2017-01-01T00:00:00Z":"54.000000","2017-01-01T00:05:00Z":"55.000000","2017-01-01T00:10:00Z":"56.000000","2017-01-01T00:15:00Z":"57.000000","2017-01-01T00:20:00Z":"58.000000","2017-01-01T00:25:00Z":"59.000000"},"2000":{"2017-01-01T00:00:00Z":"60.000000","2017-01-01T00:05:00Z":"61.000000","2017-01-01T00:10:00Z":"62.000000","2017-01-01T00:15:00Z":"63.000000","2017-01-01T00:20:00Z":"64.000000","2017-01-01T00:25:00Z":"65.000000"},"3000":{"2017-01-01T00:00:00Z":"66.000000","2017-01-01T00:05:00Z":"67.000000","2017-01-01T00:10:00Z":"68.000000","2017-01-01T00:15:00Z":"69.000000","2017-01-01T00:20:00Z":"70.000000","2017-01-01T00:25:00Z":"71.000000"},"4000":{"2017-01-01T00:00:00Z":"72.000000","2017-01-01T00:05:00Z":"73.000000","2017-01-01T00:10:00Z":"74.000000","2017-01-01T00:15:00Z":"75.000000","2017-01-01T00:20:00Z":"76.000000","2017-01-01T00:25:00Z":"77.000000"},"5000":{"2017-01-01T00:00:00Z":"78.000000","2017-01-01T00:05:00Z":"79.000000","2017-01-01T00:10:00Z":"80.000000","2017-01-01T00:15:00Z":"81.000000","2017-01-01T00:20:00Z":"82.000000","2017-01-01T00:25:00Z":"83.000000"},"6000":{"2017-01-01T00:00:00Z":"84.000000","2017-01-01T00:05:00Z":"85.000000","2017-01-01T00:10:00Z":"86.000000","2017-01-01T00:15:00Z":"87.000000","2017-01-01T00:20:00Z":"88.000000","2017-01-01T00:25:00Z":"89.000000"},"7000":{"2017-01-01T00:00:00Z":"90.000000","2017-01-01T00:05:00Z":"91.000000","2017-01-01T00:10:00Z":"92.000000","2017-01-01T00:15:00Z":"93.000000","2017-01-01T00:20:00Z":"94.000000","2017-01-01T00:25:00Z":"95.000000"},"8000":{"2017-01-01T00:00:00Z":"96.000000","2017-01-01T00:05:00Z":"97.000000","2017-01-01T00:10:00Z":"98.000000","2017-01-01T00:15:00Z":"99.000000","2017-01-01T00:20:00Z":"100.000000","2017-01-01T00:25:00Z":"101.000000"},"9000":{"2017-01-01T00:00:00Z":"102.000000","2017-01-01T00:05:00Z":"103.000000","2017-01-01T00:10:00Z":"104.000000","2017-01-01T00:15:00Z":"105.000000","2017-01-01T00:20:00Z":"106.000000","2017-01-01T00:25:00Z":"107.000000"}},"member3":{"1000":{"2017-01-01T00:00:00Z":"108.000000","2017-01-01T00:05:00Z":"109.000000","2017-01-01T00:10:00Z":"110.000000","2017-01-01T00:15:00Z":"111.000000","2017-01-01T00:20:00Z":"112.000000","2017-01-01T00:25:00Z":"113.000000"},"2000":{"2017-01-01T00:00:00Z":"114.000000","2017-01-01T00:05:00Z":"115.000000","2017-01-01T00:10:00Z":"116.000000","2017-01-01T00:15:00Z":"117.000000","2017-01-01T00:20:00Z":"118.000000","2017-01-01T00:25:00Z":"119.000000"},"3000":{"2017-01-01T00:00:00Z":"120.000000","2017-01-01T00:05:00Z":"121.000000","2017-01-01T00:10:00Z":"122.000000","2017-01-01T00:15:00Z":"123.000000","2017-01-01T00:20:00Z":"124.000000","2017-01-01T00:25:00Z":"125.000000"},"4000":{"2017-01-01T00:00:00Z":"126.000000","2017-01-01T00:05:00Z":"127.000000","2017-01-01T00:10:00Z":"128.000000","2017-01-01T00:15:00Z":"129.000000","2017-01-01T00:20:00Z":"130.000000","2017-01-01T00:25:00Z":"131.000000"},"5000":{"2017-01-01T00:00:00Z":"132.000000","2017-01-01T00:05:00Z":"133.000000","2017-01-01T00:10:00Z":"134.000000","2017-01-01T00:15:00Z":"135.000000","2017-01-01T00:20:00Z":"136.000000","2017-01-01T00:25:00Z":"137.000000"},"6000":{"2017-01-01T00:00:00Z":"138.000000","2017-01-01T00:05:00Z":"139.000000","2017-01-01T00:10:00Z":"140.000000","2017-01-01T00:15:00Z":"141.000000","2017-01-01T00:20:00Z":"142.000000","2017-01-01T00:25:00Z":"143.000000"},"7000":{"2017-01-01T00:00:00Z":"144.000000","2017-01-01T00:05:00Z":"145.000000","2017-01-01T00:10:00Z":"146.000000","2017-01-01T00:15:00Z":"147.000000","2017-01-01T00:20:00Z":"148.000000","2017-01-01T00:25:00Z":"149.000000"},"8000":{"2017-01-01T00:00:00Z":"150.000000","2017-01-01T00:05:00Z":"151.000000","2017-01-01T00:10:00Z":"152.000000","2017-01-01T00:15:00Z":"153.000000","2017-01-01T00:20:00Z":"154.000000","2017-01-01T00:25:00Z":"155.000000"},"9000":{"2017-01-01T00:00:00Z":"156.000000","2017-01-01T00:05:00Z":"157.000000","2017-01-01T00:10:00Z":"158.000000","2017-01-01T00:15:00Z":"159.000000","2017-01-01T00:20:00Z":"160.000000","2017-01-01T00:25:00Z":"161.000000"}},"member4":{"1000":{"2017-01-01T00:00:00Z":"162.000000","2017-01-01T00:05:00Z":"163.000000","2017-01-01T00:10:00Z":"164.000000","2017-01-01T00:15:00Z":"165.000000","2017-01-01T00:20:00Z":"166.000000","2017-01-01T00:25:00Z":"167.000000"},"2000":{"2017-01-01T00:00:00Z":"168.000000","2017-01-01T00:05:00Z":"169.000000","2017-01-01T00:10:00Z":"170.000000","2017-01-01T00:15:00Z":"171.000000","2017-01-01T00:20:00Z":"172.000000","2017-01-01T00:25:00Z":"173.000000"},"3000":{"2017-01-01T00:00:00Z":"174.000000","2017-01-01T00:05:00Z":"175.000000","2017-01-01T00:10:00Z":"176.000000","2017-01-01T00:15:00Z":"177.000000","2017-01-01T00:20:00Z":"178.000000","2017-01-01T00:25:00Z":"179.000000"},"4000":{"2017-01-01T00:00:00Z":"180.000000","2017-01-01T00:05:00Z":"181.000000","2017-01-01T00:10:00Z":"182.000000","2017-01-01T00:15:00Z":"183.000000","2017-01-01T00:20:00Z":"184.000000","2017-01-01T00:25:00Z":"185.000000"},"5000":{"2017-01-01T00:00:00Z":"186.000000","2017-01-01T00:05:00Z":"187.000000","2017-01-01T00:10:00Z":"188.000000","2017-01-01T00:15:00Z":"189.000000","2017-01-01T00:20:00Z":"190.000000","2017-01-01T00:25:00Z":"191.000000"},"6000":{"2017-01-01T00:00:00Z":"192.000000","2017-01-01T00:05:00Z":"193.000000","2017-01-01T00:10:00Z":"194.000000","2017-01-01T00:15:00Z":"195.000000","2017-01-01T00:20:00Z":"196.000000","2017-01-01T00:25:00Z":"197.000000"},"7000":{"2017-01-01T00:00:00Z":"198.000000","2017-01-01T00:05:00Z":"199.000000","2017-01-01T00:10:00Z":"200.000000","2017-01-01T00:15:00Z":"201.000000","2017-01-01T00:20:00Z":"202.000000","2017-01-01T00:25:00Z":"203.000000"},"8000":{"2017-01-01T00:00:00Z":"204.000000","2017-01-01T00:05:00Z":"205.000000","2017-01-01T00:10:00Z":"206.000000","2017-01-01T00:15:00Z":"207.000000","2017-01-01T00:20:00Z":"208.000000","2017-01-01T00:25:00Z":"209.000000"},"9000":{"2017-01-01T00:00:00Z":"210.000000","2017-01-01T00:05:00Z":"211.000000","2017-01-01T00:10:00Z":"212.000000","2017-01-01T00:15:00Z":"213.000000","2017-01-01T00:20:00Z":"214.000000","2017-01-01T00:25:00Z":"215.000000"}},"member5":{"1000":{"2017-01-01T00:00:00Z":"216.000000","2017-01-01T00:05:00Z":"217.000000","2017-01-01T00:10:00Z":"218.000000","2017-01-01T00:15:00Z":"219.000000","2017-01-01T00:20:00Z":"220.000000","2017-01-01T00:25:00Z":"221.000000"},"2000":{"2017-01-01T00:00:00Z":"222.000000","2017-01-01T00:05:00Z":"223.000000","2017-01-01T00:10:00Z":"224.000000","2017-01-01T00:15:00Z":"225.000000","2017-01-01T00:20:00Z":"226.000000","2017-01-01T00:25:00Z":"227.000000"},"3000":{"2017-01-01T00:00:00Z":"228.000000","2017-01-01T00:05:00Z":"229.000000","2017-01-01T00:10:00Z":"230.000000","2017-01-01T00:15:00Z":"231.000000","2017-01-01T00:20:00Z":"232.000000","2017-01-01T00:25:00Z":"233.000000"},"4000":{"2017-01-01T00:00:00Z":"234.000000","2017-01-01T00:05:00Z":"235.000000","2017-01-01T00:10:00Z":"236.000000","2017-01-01T00:15:00Z":"237.000000","2017-01-01T00:20:00Z":"238.000000","2017-01-01T00:25:00Z":"239.000000"},"5000":{"2017-01-01T00:00:00Z":"240.000000","2017-01-01T00:05:00Z":"241.000000","2017-01-01T00:10:00Z":"242.000000","2017-01-01T00:15:00Z":"243.000000","2017-01-01T00:20:00Z":"244.000000","2017-01-01T00:25:00Z":"245.000000"},"6000":{"2017-01-01T00:00:00Z":"246.000000","2017-01-01T00:05:00Z":"247.000000","2017-01-01T00:10:00Z":"248.000000","2017-01-01T00:15:00Z":"249.000000","2017-01-01T00:20:00Z":"250.000000","2017-01-01T00:25:00Z":"251.000000"},"7000":{"2017-01-01T00:00:00Z":"252.000000","2017-01-01T00:05:00Z":"253.000000","2017-01-01T00:10:00Z":"254.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"8000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"9000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"}},"member6":{"1000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"2000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"3000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"4000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"5000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"6000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"7000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"8000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"},"9000":{"2017-01-01T00:00:00Z":"255.000000","2017-01-01T00:05:00Z":"255.000000","2017-01-01T00:10:00Z":"255.000000","2017-01-01T00:15:00Z":"255.000000","2017-01-01T00:20:00Z":"255.000000","2017-01-01T00:25:00Z":"255.000000"}}}}] diff --git a/tests/starttests.sh b/tests/starttests.sh index 8e57fa52..b63b6da8 100644 --- a/tests/starttests.sh +++ b/tests/starttests.sh @@ -19,6 +19,7 @@ export ADAGUC_ENABLELOGBUFFER=FALSE export ADAGUC_DATASET_DIR=${ADAGUC_PATH}/data/config/datasets/ export ADAGUC_DATA_DIR=${ADAGUC_PATH}/data/datasets/ export ADAGUC_AUTOWMS_DIR=${ADAGUC_PATH}/data/datasets/ +export ADAGUC_DB="${ADAGUC_TMP}/adaguc.autoresource.db" ulimit -c unlimited diff --git a/tests/starttests_psql.sh b/tests/starttests_psql.sh new file mode 100644 index 00000000..4ab22cfb --- /dev/null +++ b/tests/starttests_psql.sh @@ -0,0 +1,36 @@ +#!/bin/bash +echo "Starting adaguc-server functional tests" +pushd `dirname $0` > /dev/null +SCRIPTPATH=`pwd` +popd > /dev/null +unset ADAGUC_CONFIG +unset QUERY_STRING +export ADAGUC_PATH=${SCRIPTPATH}/../ +export ADAGUC_PATH=`readlink -f ${ADAGUC_PATH}`/ +export PYTHONPATH=${ADAGUC_PATH}/python/lib/ +echo "ADAGUC-Server path is [$ADAGUC_PATH]" +export ADAGUC_TMP=${ADAGUC_PATH}/tests/tmp/ +export ADAGUC_LOGFILE=${ADAGUC_PATH}/tests/log/ +rm -rf $ADAGUC_LOGFILE && mkdir -p $ADAGUC_LOGFILE +export ADAGUC_LOGFILE=${ADAGUC_PATH}/tests/log/adaguc-server.log +export ADAGUC_FONT="${ADAGUC_PATH}/data/fonts/FreeSans.ttf" +export ADAGUC_ONLINERESOURCE="" +export ADAGUC_ENABLELOGBUFFER=FALSE +export ADAGUC_DATASET_DIR=${ADAGUC_PATH}/data/config/datasets/ +export ADAGUC_DATA_DIR=${ADAGUC_PATH}/data/datasets/ +export ADAGUC_AUTOWMS_DIR=${ADAGUC_PATH}/data/datasets/ + +# This assumes you can reach psql on port 5432 +# Tests will use a separate `adaguc_test` database +# You cannot drop the database you are currently logged in as +db_host=$([[ "${TEST_IN_CONTAINER}" == 1 ]] && echo "host.docker.internal" || echo "localhost") +export ADAGUC_DB="user=adaguc password=adaguc host=${db_host} dbname=postgres" +psql "$ADAGUC_DB" -c "DROP DATABASE IF EXISTS adaguc_test;" +psql "$ADAGUC_DB" -c "CREATE DATABASE adaguc_test;" +export ADAGUC_DB="user=adaguc password=adaguc host=${db_host} dbname=adaguc_test" + +ulimit -c unlimited + +python3 ${ADAGUC_PATH}/tests/functional_test.py $1 && \ +cd ../python/python_fastapi_server && \ +bash ./test_server.sh \ No newline at end of file