fixed MVT tile handling; properly aligning data returned from gdalwarper

mapproxy/src/mapproxy/CMakeLists.txt

@@ -1,3 +1,4 @@
+# ------------------------------------------------------------------------
 # core mapproxy library
 set(mapproxy-core_SOURCES
   error.hpp
@@ -9,6 +10,7 @@ set(mapproxy-core_SOURCES
   support/tileindex.hpp support/tileindex.cpp
   support/fileclass.hpp support/fileclass.cpp
   support/serialization.hpp support/serialization.cpp
+  support/aborter.hpp
   support/glob.hpp support/glob.cpp
   support/tilejson.hpp support/tilejson.cpp
   support/cesiumconf.hpp support/cesiumconf.cpp
@@ -53,12 +55,11 @@ set(mapproxy-core_SOURCES
   mapproxy.hpp mapproxy.cpp
   )
 
-# test program
 define_module(LIBRARY mapproxy-core
   DEPENDS
   qmf
   pysupport pydbglog
-  vts-libs>=2.4 http>=1.0 geo>=1.28 gdal-drivers>=1.14 geometry>=1.7
+  vts-libs>=2.4 http>=1.0 geo>=1.28 geometry>=1.7
   jsoncpp>=2.1
 
   Boost_SERIALIZATION
@@ -74,10 +75,32 @@ buildsys_library(mapproxy-core)
 target_link_libraries(mapproxy-core ${MODULE_LIBRARIES})
 buildsys_target_compile_definitions(mapproxy-core ${MODULE_DEFINITIONS})
 
-# test program
+# ------------------------------------------------------------------------
+# GDAL machinery
+set(mapproxy-gdal_SOURCES
+  gdalsupport.hpp gdalsupport/gdalsupport.cpp
+  gdalsupport/types.hpp
+  gdalsupport/requests.hpp gdalsupport/requests.cpp
+  gdalsupport/process.hpp gdalsupport/process.cpp
+  gdalsupport/datasetcache.hpp gdalsupport/datasetcache.cpp
+  gdalsupport/operations.hpp gdalsupport/operations.cpp
+  )
+
+define_module(LIBRARY mapproxy-gdal
+  DEPENDS mapproxy-core
+  gdal-drivers>=1.14
+  )
+
+add_library(mapproxy-gdal STATIC ${mapproxy-gdal_SOURCES})
+buildsys_library(mapproxy-gdal)
+target_link_libraries(mapproxy-gdal ${MODULE_LIBRARIES})
+buildsys_target_compile_definitions(mapproxy-gdal ${MODULE_DEFINITIONS})
+
+# ------------------------------------------------------------------------
+# Main mapproxy binary
 define_module(BINARY mapproxy
   DEPENDS
-  mapproxy-core service>=1.6
+  mapproxy-gdal mapproxy-core service>=1.6
 
   Boost_SERIALIZATION
   Boost_FILESYSTEM
@@ -123,13 +146,6 @@ set(mapproxy_SOURCES
   fileinfo.hpp fileinfo.cpp
   core.hpp core.cpp
 
-  gdalsupport.hpp gdalsupport/gdalsupport.cpp
-  gdalsupport/types.hpp
-  gdalsupport/requests.hpp gdalsupport/requests.cpp
-  gdalsupport/process.hpp gdalsupport/process.cpp
-  gdalsupport/datasetcache.hpp gdalsupport/datasetcache.cpp
-  gdalsupport/operations.hpp gdalsupport/operations.cpp
-
   main.cpp
   )
 

mapproxy/src/mapproxy/gdalsupport.hpp

@@ -42,18 +42,12 @@
 #include "geo/vectorformat.hpp"
 #include "geo/heightcoding.hpp"
 
-#include "vts-libs/vts/nodeinfo.hpp"
-
 #include "support/geo.hpp"
 #include "support/layerenancer.hpp"
-#include "sink.hpp"
-
-namespace vts = vtslibs::vts;
+#include "support/aborter.hpp"
 
 class GdalWarper {
 public:
-    typedef std::shared_ptr<GdalWarper> pointer;
-
     struct Options {
         unsigned int processCount;
         boost::filesystem::path tmpRoot;

mapproxy/src/mapproxy/gdalsupport/gdalsupport.cpp

@@ -716,6 +716,8 @@ void GdalWarper::Detail::cleanup(bool join)
         cond().notify_all();
     }
 
+    // TODO: kill unresponsive processes
+
     // cleanup
     while (!workers_.empty()) {
         auto head(workers_.begin());

mapproxy/src/mapproxy/gdalsupport/operations.cpp

@@ -27,6 +27,7 @@
 #include <new>
 #include <algorithm>
 #include <cstring>
+#include <type_traits>
 
 #include <boost/iostreams/stream_buffer.hpp>
 #include <boost/iostreams/device/array.hpp>
@@ -37,6 +38,8 @@
 
 #include <ogrsf_frmts.h>
 
+#include "utility/gccversion.hpp"
+
 #include "imgproc/rastermask/cvmat.hpp"
 
 #include "geo/verticaladjuster.hpp"
@@ -60,7 +63,8 @@ cv::Mat* allocateMat(ManagedBuffer &mb
     const auto matSize(sizeof(cv::Mat) + dataSize);
 
     // create raw memory to hold matrix and data
-    char *raw(static_cast<char*>(mb.allocate(matSize)));
+    char *raw(static_cast<char*>
+              (mb.allocate_aligned(matSize, alignof(cv::Mat))));
 
     // allocate matrix in raw data block
     return new (raw) cv::Mat(size.height, size.width, type
@@ -401,7 +405,9 @@ allocateHc(ManagedBuffer &mb
 {
     // create raw memory to hold block and data
     char *raw(static_cast<char*>
-              (mb.allocate(sizeof(GdalWarper::Heightcoded) + data.size())));
+              (mb.allocate_aligned
+               (sizeof(GdalWarper::Heightcoded) + data.size()
+                , alignof(GdalWarper::Heightcoded))));
 
     // poiter to output data
     auto *dataPtr(raw + sizeof(GdalWarper::Heightcoded));

mapproxy/src/mapproxy/generator/generator.cpp

@@ -227,12 +227,18 @@ bool isRemote(const std::string &path)
              || ba::istarts_with(path, "ftp:")));
 }
 
+bool isMvt(const std::string &path)
+{
+    return ba::starts_with(path, "mvt:");
+}
+
 } // namespace
 
 std::string Generator::absoluteDataset(const std::string &path)
     const
 {
     // handle non-path resources (i.e. URL's)
+    if (isMvt(path)) { return "mvt:" + absoluteDataset(path.substr(4)); }
     if (isRemote(path)) { return path; }
     return absolute(path, config_.resourceRoot).string();
 }
@@ -241,7 +247,9 @@ boost::filesystem::path
 Generator::absoluteDataset(const boost::filesystem::path &path)
     const
 {
+    const auto &spath(path.string());
     // handle non-path resources (i.e. URL's)
+    if (isMvt(spath)) { return "mvt:" + absoluteDataset(spath.substr(4)); }
     if (isRemote(path.string())) { return path; }
     return absolute(path, config_.resourceRoot);
 }

mapproxy/src/mapproxy/sink.hpp

@@ -41,21 +41,10 @@
 #include "vts-libs/storage/streams.hpp"
 
 #include "support/fileclass.hpp"
+#include "support/aborter.hpp"
 
 namespace vs = vtslibs::storage;
 
-/** Aborter helper.
- */
-struct Aborter {
-    typedef http::ServerSink::AbortedCallback AbortedCallback;
-
-    virtual ~Aborter() {}
-
-    /** Defaults to dummy aborter
-     */
-    virtual void setAborter(const AbortedCallback&) {};
-};
-
 /** Wraps libhttp's sink.
  */
 class Sink : public Aborter {

mapproxy/src/mapproxy/support/aborter.hpp

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2017 Melown Technologies SE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * *  Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * *  Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef mapproxy_support_aborter_hpp_included_
+#define mapproxy_support_aborter_hpp_included_
+
+#include <functional>
+
+/** Aborter helper.
+ */
+struct Aborter {
+    typedef std::function<void()> AbortedCallback;
+
+    virtual ~Aborter() {}
+
+    /** Defaults to dummy aborter
+     */
+    virtual void setAborter(const AbortedCallback&) {};
+};
+
+#endif // mapproxy_support_aborter_hpp_included_
+

mapproxy/src/mapproxy/support/wmts.cpp

@@ -266,8 +266,8 @@ std::string makeTemplate(const Layer &layer)
     }
 
     os << "{TileMatrix}-{TileCol}-{TileRow}." << layer.format;
-    if (layer.resource.revision) {
-        os << "?r=" << layer.resource.revision;
+    if (layer.resource->revision) {
+        os << "?r=" << layer.resource->revision;
     }
 
     return os.str();
@@ -480,7 +480,7 @@ void content(XML &x, const Layer::list &layers)
 
     Element c(x, "Contents");
     for (const auto &layer : layers) {
-        const auto &r(layer.resource);
+        const auto &r(*layer.resource);
 
         const TmsEntry &tms(cache.get(r));
 

mapproxy/src/mapproxy/support/wmts.hpp

@@ -32,14 +32,14 @@
 namespace wmts {
 
 struct Layer {
-    const Resource &resource;
+    const Resource *resource;
     std::string rootPath;
     RasterFormat format;
 
     typedef std::vector<Layer> list;
 
     Layer(const Resource &resource)
-        : resource(resource), format(RasterFormat::jpg)
+        : resource(&resource), format(RasterFormat::jpg)
     {}
 };