Merge pull request #263 from KNMI/caching

Made caching configurable

CCDFDataModel/CCDFHDF5IO.cpp

@@ -422,7 +422,7 @@ hid_t CDFHDF5Reader::openH5GroupByName(char *varNameOut, size_t maxVarNameLen, c
   return HDF5_group;
 }
 void CDFHDF5Reader::closeH5GroupByName(const char *variableGroupName) {
-  CDBDebug("Warning %s variableGroupName not used", variableGroupName);
+  // CDBDebug("Warning %s variableGroupName not used", variableGroupName);
   while (opengroups.size() > 0) {
 #ifdef CCDFHDF5IO_DEBUG
     CDBDebug("closing with id %d", opengroups.back());

CCDFDataModel/CProj4ToCF.cpp

@@ -1002,7 +1002,7 @@ int CProj4ToCF::convertCFToProj(CDF::Variable *projectionVariable, CT::string *p
       CREPORT_INFO_NODOC(CT::string("Unsupported projection: ") + grid_mapping_name, CReportMessage::Categories::GENERAL);
       return CPROJ4TOCF_UNSUPPORTED_PROJECTION;
     }
-    CREPORT_INFO_NODOC(CT::string("Determined the projection string using the CF conventions: ") + proj4String, CReportMessage::Categories::GENERAL);
+    // CREPORT_INFO_NODOC(CT::string("Determined the projection string using the CF conventions: ") + proj4String, CReportMessage::Categories::GENERAL);
   } catch (int e) {
     // CDBError("%s\n",CDF::lastErrorMessage.c_str());
     try {

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="[email protected]"
 
 # Version should be same as in Definitions.h
-LABEL version="2.13.8"
+LABEL version="2.13.9"
 
 # Try to update image packages
 RUN apt-get -q -y update \
@@ -93,15 +93,15 @@ COPY tests /adaguc/adaguc-server-master/tests
 COPY runtests.sh /adaguc/adaguc-server-master/runtests.sh
 
 # Run adaguc-server functional and regression tests
-RUN bash runtests.sh && echo "TESTSDONE" >  /adaguc/adaguc-server-master/testsdone.txt
+RUN bash runtests.sh
+
+# Create a file indicating that the test succeeded. This file is used in the final stage
+RUN echo "TESTSDONE" >  /adaguc/adaguc-server-master/testsdone.txt
 
 
 ######### Fourth stage, prod ############
 FROM base as prod
 
-# This ensures that the test stage is ran without issues.
-COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt 
-
 # Set same uid as vivid
 RUN useradd -m adaguc -u 1000 && \
     # Setup directories
@@ -133,6 +133,9 @@ RUN bash -c "python3 /adaguc/adaguc-server-master/python/examples/runautowms/run
 
 WORKDIR /adaguc/adaguc-server-master
 
+# This checks if the test stage has ran without issues.
+COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt 
+
 USER adaguc
 
 # For HTTP

NEWS.md

@@ -1,3 +1,6 @@
+**Version 2.13.9 2023-11-01**
+- Cache-Control header can be configured via the Settings item 
+
 **Version 2.13.8 2023-11-01**
 - Fixed issue #311, dimension units are now propagated from the Layer configuration to the GetCapabilities
 

adagucserverEC/CAutoConfigure.cpp

@@ -510,6 +510,7 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
         dataSource->requiredDims.push_back(ogcDim);
         ogcDim->name.copy("none");
         ogcDim->value.copy("0");
+        ogcDim->queryValue.copy("0");
         ogcDim->netCDFDimName.copy("none");
       }
     }

adagucserverEC/CDataSource.cpp

@@ -1291,6 +1291,7 @@ CDataSource *CDataSource::clone() {
     d->requiredDims.push_back(ogcDim);
     ogcDim->name = requiredDims[j]->name;
     ogcDim->value = requiredDims[j]->value;
+    ogcDim->queryValue = requiredDims[j]->queryValue;
     ogcDim->netCDFDimName = requiredDims[j]->netCDFDimName;
     for (size_t i = 0; i < requiredDims[j]->uniqueValues.size(); i++) {
       ogcDim->uniqueValues.push_back(requiredDims[j]->uniqueValues[i].c_str());

adagucserverEC/CImageDataWriter.cpp

@@ -3431,25 +3431,28 @@ int CImageDataWriter::end() {
   // Static image
   // CDBDebug("srvParam->imageFormat = %d",srvParam->imageFormat);
   int status = 1;
+
+  CT::string cacheControl = srvParam->getCacheControlHeader(srvParam->getCacheControlOption());
+
   if (srvParam->imageFormat == IMAGEFORMAT_IMAGEPNG8) {
     CDBDebug("Creating 8 bit png with alpha");
-    printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng8(true);
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEPNG8_NOALPHA) {
     CDBDebug("Creating 8 bit png without alpha");
-    printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng8(false);
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEPNG24) {
     CDBDebug("Creating 24 bit png");
-    printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng24();
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEPNG32) {
     CDBDebug("Creating 32 bit png");
-    printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng32();
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEWEBP) {
     CDBDebug("Creating 32 bit webp");
-    printf("%s%c%c\n", "Content-Type:image/webp", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/webp", cacheControl.c_str(), 13, 10);
     int webPQuality = srvParam->imageQuality;
     if (!srvParam->Format.empty()) {
       /* Support setting quality via wms format parameter, e.g. format=image/webp;90& */
@@ -3466,12 +3469,12 @@ int CImageDataWriter::end() {
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEGIF) {
     // CDBDebug("LegendGraphic GIF");
     if (animation == 0) {
-      printf("%s%c%c\n", "Content-Type:image/gif", 13, 10);
+      printf("%s%s%c%c\n", "Content-Type:image/gif", cacheControl.c_str(), 13, 10);
     }
     status = drawImage.printImageGif();
   } else {
     // CDBDebug("LegendGraphic PNG");
-    printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng8(true);
   }
 

adagucserverEC/COGCDims.h

@@ -36,6 +36,12 @@ class COGCDims {
    */
 
   CT::string name;
+
+  /**
+   * Value, as given in the query_string
+   */
+  CT::string queryValue;
+
   /**
    * Value, are all values as given in the KVP request string, can contain / and , tokens
    */

adagucserverEC/CRequest.cpp

@@ -789,7 +789,7 @@ int CRequest::process_wms_getcap_request() {
   if (pszADAGUCWriteToFile != NULL) {
     CReadFile::write(pszADAGUCWriteToFile, XMLdocument.c_str(), XMLdocument.length());
   } else {
-    printf("%s%c%c\n", "Content-Type:text/xml", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type:text/xml", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
     printf("%s", XMLdocument.c_str());
   }
 
@@ -848,6 +848,7 @@ int CRequest::setDimValuesForDataSource(CDataSource *dataSource, CServerParams *
 };
 
 int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams *srvParam) {
+
 #ifdef CREQUEST_DEBUG
   StopWatch_Stop("### [fillDimValuesForDataSource]");
 #endif
@@ -920,6 +921,7 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
             dataSource->requiredDims.push_back(ogcDim);
             ogcDim->name.copy(&dimName);
             ogcDim->value.copy(&srvParam->requestDims[k]->value);
+            ogcDim->queryValue.copy(&srvParam->requestDims[k]->value);
             ogcDim->netCDFDimName.copy(dataSource->cfgLayer->Dimension[i]->attr.name.c_str());
 
             if (ogcDim->name.equals("time") || ogcDim->name.equals("reference_time")) {
@@ -1048,7 +1050,6 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
         if (netCDFDimName.equals("none")) {
           continue;
         }
-
         /* A dimension where the default value is set to filetimedate should not be queried from the db */
         if (dataSource->cfgLayer->Dimension[i]->attr.defaultV.equals("filetimedate")) {
           continue;
@@ -1180,11 +1181,28 @@ int CRequest::fillDimValuesForDataSource(CDataSource *dataSource, CServerParams
 
 #ifdef CREQUEST_DEBUG
   for (size_t j = 0; j < dataSource->requiredDims.size(); j++) {
-    CDBDebug("dataSource->requiredDims[%d][%s] = [%s] (%s)", j, dataSource->requiredDims[j]->name.c_str(), dataSource->requiredDims[j]->value.c_str(),
-             dataSource->requiredDims[j]->netCDFDimName.c_str());
+    auto *requiredDim = dataSource->requiredDims[j];
+    CDBDebug("dataSource->requiredDims[%d][%s] = [%s] (%s)", j, requiredDim->name.c_str(), requiredDim->value.c_str(), requiredDim->netCDFDimName.c_str());
+    CDBDebug("%s: %s === %s", requiredDim->name.c_str(), requiredDim->value.c_str(), requiredDim->queryValue.c_str());
   }
   CDBDebug("### [</fillDimValuesForDataSource>]");
 #endif
+  bool allDimensionsAreAsRequestedInQueryString = true;
+  for (size_t j = 0; j < dataSource->requiredDims.size(); j++) {
+    auto *requiredDim = dataSource->requiredDims[j];
+    CDBDebug("%s: [%s] === [%s]", requiredDim->name.c_str(), requiredDim->value.c_str(), requiredDim->queryValue.c_str());
+    if (!requiredDim->value.equals(requiredDim->queryValue)) {
+      allDimensionsAreAsRequestedInQueryString = false;
+    }
+  }
+
+  CDBDebug("allDimensionsAreAsRequestedInQueryString %d", allDimensionsAreAsRequestedInQueryString);
+  if (allDimensionsAreAsRequestedInQueryString) {
+    srvParam->setCacheControlOption(CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE);
+  } else {
+    srvParam->setCacheControlOption(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE);
+  }
+
   return 0;
 }
 
@@ -3241,7 +3259,8 @@ int CRequest::process_querystring() {
         return 1;
       }
       drawImage.crop(1);
-      printf("%s%c%c\n", "Content-Type:image/png", 13, 10);
+      const char *cacheControl = srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE).c_str();
+      printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl, 13, 10);
       drawImage.printImagePng8(true);
       return 0;
     }

adagucserverEC/CServerConfig_CPPXSD.h

@@ -1283,7 +1283,7 @@ class CServerConfig : public CXMLSerializerInterface {
   public:
     class Cattr {
     public:
-      CT::string enablecleanupsystem, cleanupsystemlimit;
+      CT::string enablecleanupsystem, cleanupsystemlimit, cache_age_cacheableresources, cache_age_volatileresources;
     } attr;
     void addAttribute(const char *attrname, const char *attrvalue) {
       if (equals("enablecleanupsystem", 19, attrname)) {
@@ -1292,6 +1292,12 @@ class CServerConfig : public CXMLSerializerInterface {
       } else if (equals("cleanupsystemlimit", 18, attrname)) {
         attr.cleanupsystemlimit.copy(attrvalue);
         return;
+      } else if (equals("cache_age_cacheableresources", 28, attrname)) {
+        attr.cache_age_cacheableresources.copy(attrvalue);
+        return;
+      } else if (equals("cache_age_volatileresources", 27, attrname)) {
+        attr.cache_age_volatileresources.copy(attrvalue);
+        return;
       }
     }
   };

adagucserverEC/CServerParams.cpp

@@ -649,3 +649,26 @@ int CServerParams::parseConfigFile(CT::string &pszConfigFile, std::vector<CServe
     return 1;
   }
 }
+
+CT::string CServerParams::getCacheControlHeader(int mode) {
+  if (cfg != nullptr && cfg->Settings.size() == 1) {
+    int cacheAge = 0;
+    CT::string cacheString = "\r\nCache-Control:max-age=";
+    if (mode == CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE) {
+      if (!cfg->Settings[0]->attr.cache_age_volatileresources.empty()) {
+        if (cfg->Settings[0]->attr.cache_age_volatileresources.toInt() != 0) {
+          cacheString.printconcat("%d", cfg->Settings[0]->attr.cache_age_volatileresources.toInt());
+          return cacheString;
+        }
+      }
+    } else if (mode == CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE) {
+      if (!cfg->Settings[0]->attr.cache_age_cacheableresources.empty()) {
+        if (cfg->Settings[0]->attr.cache_age_cacheableresources.toInt() != 0) {
+          cacheString.printconcat("%d", cfg->Settings[0]->attr.cache_age_cacheableresources.toInt());
+          return cacheString;
+        }
+      }
+    }
+  }
+  return "";
+}

adagucserverEC/CServerParams.h

@@ -64,6 +64,10 @@ class CWMSExtensions {
   bool logScale;
 };
 
+#define CSERVERPARAMS_CACHE_CONTROL_OPTION_NOCACHE 0
+#define CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE 1
+#define CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE 2
+
 /**
  * Global server settings, initialized at the start, accesible from almost everywhere
  */
@@ -76,6 +80,7 @@ class CServerParams {
   CT::string _onlineResource;
   static int dataRestriction;
   static char debugLoggingIsEnabled;
+  int cacheControlOption = CSERVERPARAMS_CACHE_CONTROL_OPTION_NOCACHE;
 
 public:
   double dfResX, dfResY;
@@ -311,6 +316,17 @@ class CServerParams {
    * returns zero on success       *
    */
   int parseConfigFile(CT::string &pszConfigFile, std::vector<CServerConfig::XMLE_Environment *> *extraEnvironment);
+
+  /**
+   * Returns cache control header
+   * mode CSERVERPARAMS_CACHE_CONTROL_OPTION_NOCACHE returns empty string ("")
+   * mode CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE is for urls which response might change often (shorter max-age)
+   * mode CSERVERPARAMS_CACHE_CONTROL_OPTION_FULLYCACHEABLE is fully specified urls
+   */
+  CT::string getCacheControlHeader(int mode);
+
+  void setCacheControlOption(int mode) { cacheControlOption = mode; }
+  int getCacheControlOption() { return cacheControlOption; }
 };
 
 #endif

adagucserverEC/Definitions.h

@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.13.8" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.13.9" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0

data/config/datasets/adaguc.tests.cacheheader.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Configuration>
+
+  <Settings cache_age_volatileresources="60" cache_age_cacheableresources="7200" />
+
+  <!-- Custom styles -->
+  <Style name="testdata">
+    <Legend fixedclasses="true" tickinterval="0.1" tickround=".01">no2</Legend>
+    <Min>0.01</Min>
+    <Max>30</Max>
+    <Log>10</Log>
+    <ContourLine width="1.8" linecolor="#888888" textcolor="#000000" textformatting="%2.2f" classes="0.05,0.1,0.15,0.2,0.3"/>
+    <ContourLine width="3" linecolor="#0000FF" textcolor="#0000FF" textformatting="%2.2f" classes="0.35"/>
+
+    <ShadeInterval min="0.05" max="0.15"    label="0.05-0.15"    fillcolor="#E6E6FFA0"/>
+    <ShadeInterval min="0.20" max="0.30"    label="0.20-0.30"    fillcolor="#B3B3FF"/>
+    <ShadeInterval min="0.30" max="0.35"    label="0.30-0.35"    fillcolor="#8080FF"/>
+    <ShadeInterval min="0.35" max="1.00"    label="0.35-1.00"    fillcolor="#FF0040"/>
+
+
+    <NameMapping name="nearest"   title="Rainbow colors" abstract="Drawing with rainbow colors"/>
+    <NameMapping name="bilinear"   title="Rainbow colors, bilinear" abstract="Drawing with rainbow colors, bilinear interpolation"/>
+    <NameMapping name="nearestcontour"   title="Rainbow colors, contours" abstract="Drawing with rainbow colors, contours"/>
+    <NameMapping name="shadedcontour"   title="Rainbow colors, shading and contours" abstract="Drawing with rainbow colors, shading and contours"/>
+    <RenderMethod>nearest,bilinear,nearestcontour,shadedcontour</RenderMethod>
+
+  </Style>
+
+  <Layer type="database">
+    <FilePath filter="^nc_5D_.*\.nc$">{ADAGUC_PATH}/data/datasets/netcdf_5dims/</FilePath>
+    <Variable>data</Variable>
+    <Styles>testdata</Styles>
+
+  </Layer>
+
+
+  <!-- End of configuration /-->
+</Configuration>

doc/configuration/Configuration.md

@@ -27,7 +27,7 @@ Configuration
 -   [Include](Include.md) - (location) - Include additional configuration
     files to the service
 -   [Logging](Logging.md) - (debug) - Configure the type of logging
--   [Settings](Settings.md) - (enablecleanupsystem) - Configure global settings of the server / Dataset
+-   [Settings](Settings.md) - (enablecleanupsystem, cleanupsystemlimit, cache_age_cacheableresources, cache_age_volatileresources) - Configure global settings of the server / Dataset
 -   [Environment](Environment.md) - (name, default) - For within dataset configuration, specify which values should be substituted
 
 <!-- -->

doc/configuration/Settings.md

@@ -7,6 +7,8 @@ Available since: 2.8.6
 
 - enablecleanupsystem (false | true | dryrun): Enables the auto cleanup system. See [FilePath](./FilePath.md) and retentionperiod for details
 - cleanupsystemlimit (number): Configures how many files will be deleted at once, defaults to 10 (CDBFILESCANNER_CLEANUP_DEFAULT_LIMIT).
+- cache_age_cacheableresources (number) defaults to 0 (disabled), Sets the cache header for fully specified getmap requests
+- cache_age_volatileresources (number), defaults to 0 (disabled), Sets the cache header for things which often change, like a getcapabilities document.
 
 Allowed values for enablecleanupsystem are:
 - false (or nothing)
@@ -21,3 +23,11 @@ Allowed values for enablecleanupsystem are:
 
 This can be configured in the main Configuration section or a dataset
 configuration. The last one found is used.
+
+## Caching example
+
+
+<Settings cache_age_volatileresources="60" cache_age_cacheableresources="7200"/>
+
+- Volatile resources like the GetCapabilities document, or GetMap requests without fully specified dimensions are only cached for 60 seconds.
+- Fully cacheable resources, in this case a GetMap request with fully specified dimensions are cached for 7200 seconds.

python/python_fastapi_server/routers/wmswcs.py

@@ -91,5 +91,5 @@ def testadaguc():
     status, _data, headers = adaguc_instance.runADAGUCServer(
         url, env=adagucenv, showLogOnError=False)
     assert status == 0
-    assert headers == ["Content-Type:text/xml"]
+    assert "Content-Type:text/xml" in headers
     logger.info("adaguc-server seems [OK]")