PostgreSQL Query is optimized

Docker/adaguc-server-updatedatasets.sh

@@ -16,6 +16,11 @@ THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )
 # Unbufferd logging for realtime output
 export ADAGUC_ENABLELOGBUFFER=FALSE
 
+
+# Do export ADAGUC_VERBOSE="--verboseon" to enable verbose logging
+
+ADAGUC_VERBOSE="${ADAGUC_VERBOSE:=--verboseoff}" 
+
 if [[ $1 ]]; then
   # Update a specific dataset
   for configfile in /data/adaguc-datasets/$1 ;do
@@ -28,7 +33,7 @@ if [[ $1 ]]; then
       OUT=$?
     else
       echo "*** Starting update for ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
+      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
       OUT=$?
     fi
 
@@ -49,7 +54,7 @@ else
     fi
     echo ""
     echo "Starting update for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
+    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
     OUT=$?
      done
 

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="[email protected]"
 
 # Version should be same as in Definitions.h
-LABEL version="2.14.3"
+LABEL version="2.15.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \

NEWS.md

@@ -1,3 +1,6 @@
+**Version 2.15.0 2024-01-29**
+- PostgreSQL queries have been optimized
+
 **Version 2.14.3 2024-01-19**
 - Opendap services are accessible again in the Layer configuration: https://github.com/KNMI/adaguc-server/issues/315
 

adagucserverEC/CAutoConfigure.cpp

@@ -51,6 +51,11 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
     return 1;
   }
 
+  if (dataSource->cfgLayer->FilePath.size() != 1 && dataSource->cfgLayer->FilePath[0] != nullptr) {
+    CDBDebug("(dataSource->cfgLayer->FilePath.size() != 1");
+    return 1;
+  }
+
   /**
    * Load dimension information about the layer from the autoconfigure_dimensions table
    * This table stores only layerid, netcdf dimname, adaguc dimname and units
@@ -61,9 +66,12 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
 
   CT::string layerTableId;
   try {
-
-    layerTableId = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)
-                       ->getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), NULL, dataSource);
+    auto dbAdapter = CDBFactory::getDBAdapter(dataSource->srvParams->cfg);
+    if (dbAdapter == nullptr) {
+      CDBError("Unable to get a getDBAdapter");
+      return 1;
+    }
+    layerTableId = dbAdapter->getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), NULL, dataSource);
 
   } catch (int e) {
     CDBError("Unable to get layerTableId for autoconfigure_dimensions");
@@ -84,7 +92,6 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
   CT::string layerIdentifier = dataSource->getLayerName();
   CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getDimensionInfoForLayerTableAndLayerName(layerTableId.c_str(), layerIdentifier.c_str());
   if (store != NULL) {
-
     try {
 
       for (size_t j = 0; j < store->size(); j++) {

adagucserverEC/CDBFactory.cpp

@@ -34,7 +34,7 @@ CDBAdapter *CDBFactory::getDBAdapter(CServerConfig::XMLE_Configuration *cfg) {
   if (staticCDBAdapter == NULL) {
     if (cfg->DataBase.size() != 1) {
       CDBError("DataBase not properly configured");
-      return NULL;
+      exit(1);
     }
     if (cfg->DataBase[0]->attr.dbtype.equals("sqlite")) {
       CDBDebug("Using sqlite");

adagucserverEC/CDBFileScanner.cpp

@@ -441,6 +441,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
           break;
         }
       }
+      size_t numberOfFilesAddedToDbStore = 0;
       for (size_t d = 0; d < dataSource->cfgLayer->Dimension.size(); d++) {
         if (skipDim[d] == true) {
 #ifdef CDBFILESCANNER_DEBUG
@@ -449,7 +450,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
           continue;
         }
         {
-
+          numberOfFilesAddedToDbStore += 1;
           numberOfFilesAddedFromDB = 0;
           int fileExistsInDB = 0;
 
@@ -831,7 +832,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
         }
       }
       // End of dimloop, start inserting our collected records in one statement
-      if (j % 50 == 0) dbAdapter->addFilesToDataBase();
+      if (numberOfFilesAddedToDbStore % 50 == 0) dbAdapter->addFilesToDataBase();
     }
 
     // End of dimloop, start inserting our collected records in one statement

adagucserverEC/CPGSQLDB.cpp

@@ -24,6 +24,7 @@
  ******************************************************************************/
 
 #include "CPGSQLDB.h"
+// #define CPGSQLDB_DEBUG_H
 const char *CPGSQLDB::className = "CPGSQLDB";
 void CPGSQLDB::clearResult() {
   if (result != NULL) PQclear(result);
@@ -72,55 +73,49 @@ int CPGSQLDB::checkTable(const char *pszTableName, const char *pszColumns) {
   // 2 = table created
 
   LastErrorMsg[0] = '\0';
-  int i;
+
   if (dConnected == 0) {
     CDBError("checkTable: Not connected to DB");
     return 1;
   }
-  char query_string[1024];
-  snprintf(query_string, 1023, "select * from pg_tables where schemaname='public';");
-  result = PQexec(connection, query_string);     /* send the query */
-  if (PQresultStatus(result) != PGRES_TUPLES_OK) /* did the query fail? */
+
+  CT::string queryString;
+  queryString.print("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '%s') AS table_existence;", pszTableName);
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("checkTable PQexec SELECT EXISTS  %s", pszTableName);
+#endif
+  result = PQexec(connection, queryString.c_str()); /* send the query */
+  if (PQresultStatus(result) != PGRES_TUPLES_OK)    /* did the query fail? */
   {
     CDBError("checkTable: select from pg_tables failed");
     clearResult();
     return 1;
   }
 
-  for (i = 0; i < PQntuples(result); i++) {
-    char *pqval = PQgetvalue(result, i, 1);
-    if (strncmp(pszTableName, pqval, strlen(pszTableName)) == 0 && strlen(pqval) == strlen(pszTableName)) {
-      clearResult();
-      // CDBDebug("Found: %s == %s",pqval,pszTableName);
-      return 0;
-    } else {
-      // CDBDebug("Found: [%s] != [%s]",pqval,pszTableName);
-    }
-  }
-  // No table exists yet
-  if (i == PQntuples(result)) {
+  if (PQntuples(result) == 1 && PQgetvalue(result, 0, 0)[0] == 't') {
     clearResult();
+    return 0;
+  }
 
-    snprintf(query_string, 1023, "CREATE TABLE %s (%s)", pszTableName, pszColumns);
-
-    result = PQexec(connection, query_string);      /* send the query */
-    if (PQresultStatus(result) != PGRES_COMMAND_OK) /* did the query fail? */
-    {
+  // No table exists yet
+  queryString.print("CREATE TABLE %s (%s)", pszTableName, pszColumns);
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("checkTable PQexec CREATE TABLE %s", pszTableName);
+#endif
+  result = PQexec(connection, queryString.c_str()); /* send the query */
+  if (PQresultStatus(result) != PGRES_COMMAND_OK)   /* did the query fail? */
+  {
 
-      snprintf(LastErrorMsg, CPGSQLDB_MAX_STR_LEN, "%s: %s (%s)", PQresStatus(PQresultStatus(result)), PQresultErrorMessage(result), query_string);
+    snprintf(LastErrorMsg, CPGSQLDB_MAX_STR_LEN, "%s: %s (%s)", PQresStatus(PQresultStatus(result)), PQresultErrorMessage(result), queryString.c_str());
 
-      // snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"checkTable: CREATE TABLE %s failed",pszTableName);
-      // CDBError(LastErrorMsg);
-      clearResult();
-      return 1;
-    }
-    // Table created set status to 2
+    // snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"checkTable: CREATE TABLE %s failed",pszTableName);
+    // CDBError(LastErrorMsg);
     clearResult();
-    return 2;
+    return 1;
   }
-
+  // Table created set status to 2
   clearResult();
-  return 0;
+  return 2;
 }
 
 int CPGSQLDB::query(const char *pszQuery) {
@@ -129,6 +124,9 @@ int CPGSQLDB::query(const char *pszQuery) {
     CDBError("query: Not connected to DB");
     return 1;
   }
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("query PQexec %s", pszQuery);
+#endif
   result = PQexec(connection, pszQuery);
   if (PQresultStatus(result) != PGRES_COMMAND_OK) /* did the query fail? */
   {
@@ -141,40 +139,9 @@ int CPGSQLDB::query(const char *pszQuery) {
   clearResult();
   return 0;
 }
-// CT::string* CPGSQLDB::query_select_deprecated(const char *pszQuery,int dColumn){
-// //  CDBDebug("query_select %d %s",dColumn,pszQuery);
-//   LastErrorMsg[0]='\0';
-//   int i;
-//   if(dConnected == 0){
-//     CDBError("query_select: Not connected to DB");
-//     return NULL;
-//   }
-//
-//   result = PQexec(connection, pszQuery);
-//
-//   if (PQresultStatus(result) != PGRES_TUPLES_OK) // did the query fail?
-//   {
-//     //snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"query_select: %s failed",pszQuery);
-//     //CDBError(szTemp);
-//     clearResult();
-//     return NULL;
-//   }
-//   int n=PQntuples(result);
-//   CT::string *strings=new CT::string[n+1];
-//   for(i=0;i<n;i++){
-//     strings[i].copy(PQgetvalue(result, i, dColumn));
-//     strings[i].count=n;
-//   }
-//   CT::CTlink<CT::string>(strings,n);
-//   clearResult();
-//   return strings;
-// }
-// CT::string* CPGSQLDB::query_select_deprecated(const char *pszQuery){
-//   return query_select_deprecated(pszQuery,0);
-// }
 
 CDBStore::Store *CPGSQLDB::queryToStore(const char *pszQuery, bool throwException) {
-  // CDBDebug("query_select %s",pszQuery);
+
   LastErrorMsg[0] = '\0';
 
   if (dConnected == 0) {
@@ -184,7 +151,9 @@ CDBStore::Store *CPGSQLDB::queryToStore(const char *pszQuery, bool throwExceptio
     CDBError("queryToStore: Not connected to DB");
     return NULL;
   }
-
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("queryToStore PQexec %s", pszQuery);
+#endif
   result = PQexec(connection, pszQuery);
 
   if (PQresultStatus(result) != PGRES_TUPLES_OK) // did the query fail?

adagucserverEC/CPGSQLDB.h

@@ -51,6 +51,13 @@ class CPGSQLDB {
   ~CPGSQLDB();
   int close2();
   int connect(const char *pszOptions);
+  /**
+   * Checks if a table exists, if not it will be created with the specified columns.
+   * @param pszTableName The tablename to check for existence
+   * @param pszColumns The columns needed to create the table
+   * @returns  0: no change, table exists. 1: error,  2: table was created
+
+  */
   int checkTable(const char *pszTableName, const char *pszColumns);
   int query(const char *pszQuery);
   //     CT::string* query_select_deprecated(const char *pszQuery);

adagucserverEC/CRequest.cpp

@@ -1156,7 +1156,7 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
         CT::string dimName(dataSource->cfgLayer->Dimension[i]->value.c_str());
         CT::string fixedValue = dataSource->cfgLayer->Dimension[i]->attr.fixvalue;
         dimName.toLowerCaseSelf();
-        for (auto & requiredDim : dataSource->requiredDims) {
+        for (auto &requiredDim : dataSource->requiredDims) {
           if (requiredDim->name.equals(&dimName)) {
             CDBDebug("Forcing dimension %s from %s to %s", dimName.c_str(), requiredDim->value.c_str(), fixedValue.c_str());
             requiredDim->value = fixedValue;
@@ -1191,13 +1191,13 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
 #endif
   bool allNonFixedDimensionsAreAsRequestedInQueryString = true;
   for (auto requiredDim : dataSource->requiredDims) {
-    CDBDebug("%s: [%s] === [%s], fixed:%d", requiredDim->name.c_str(), requiredDim->value.c_str(), requiredDim->queryValue.c_str(), requiredDim->hasFixedValue);
+    // CDBDebug("%s: [%s] === [%s], fixed:%d", requiredDim->name.c_str(), requiredDim->value.c_str(), requiredDim->queryValue.c_str(), requiredDim->hasFixedValue);
     if (!requiredDim->hasFixedValue && !requiredDim->value.equals(requiredDim->queryValue)) {
       allNonFixedDimensionsAreAsRequestedInQueryString = false;
     }
   }
 
-  CDBDebug("allNonFixedDimensionsAreAsRequestedInQueryString %d", allNonFixedDimensionsAreAsRequestedInQueryString);
+  // CDBDebug("allNonFixedDimensionsAreAsRequestedInQueryString %d", allNonFixedDimensionsAreAsRequestedInQueryString);
   if (allNonFixedDimensionsAreAsRequestedInQueryString) {
     srvParam->setCacheControlOption(CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE);
   } else {
@@ -1611,12 +1611,12 @@ int CRequest::process_all_layers() {
             for (additionalLayerNo = 0; additionalLayerNo < srvParam->cfg->Layer.size(); additionalLayerNo++) {
               CT::string additional;
               srvParam->makeUniqueLayerName(&additional, srvParam->cfg->Layer[additionalLayerNo]);
-              CDBDebug("comparing for additionallayer %s==%s", additionalLayerName.c_str(), additional.c_str());
+              // CDBDebug("comparing for additionallayer %s==%s", additionalLayerName.c_str(), additional.c_str());
               if (additionalLayerName.equals(additional)) {
-                CDBDebug("Found additionalLayer [%s]", additional.c_str());
+                // CDBDebug("Found additionalLayer [%s]", additional.c_str());
                 CDataSource *additionalDataSource = new CDataSource();
 
-                CDBDebug("setCFGLayer for additionallayer %s", additionalLayerName.c_str());
+                // CDBDebug("setCFGLayer for additionallayer %s", additionalLayerName.c_str());
                 if (additionalDataSource->setCFGLayer(srvParam, srvParam->configObj->Configuration[0], srvParam->cfg->Layer[additionalLayerNo], additionalLayerName.c_str(), j) != 0) {
                   delete additionalDataSource;
                   return 1;

adagucserverEC/CServerParams.h

@@ -86,7 +86,7 @@ class CServerParams {
   double dfResX, dfResY;
   int dFound_BBOX;
   double dX, dY;
-  bool verbose = true;
+  bool verbose = false;
   CT::string *WMSLayers;
   CT::string Format;
   CT::string InfoFormat;

adagucserverEC/Definitions.h

@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.14.3" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.15.0" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0