From 68106e478212ef4f85775d4191dafa2153673d83 Mon Sep 17 00:00:00 2001 From: Vaclav Blazek Date: Wed, 10 Apr 2019 14:16:28 +0200 Subject: [PATCH] fixed MVT tile handling; properly aligning data returned from gdalwarper --- externals/gdal-drivers | 2 +- externals/libgeo | 2 +- externals/vts-libs | 2 +- mapproxy/src/mapproxy/CMakeLists.txt | 38 +++++++++++----- mapproxy/src/mapproxy/gdalsupport.hpp | 8 +--- .../src/mapproxy/gdalsupport/gdalsupport.cpp | 2 + .../src/mapproxy/gdalsupport/operations.cpp | 10 ++++- mapproxy/src/mapproxy/generator/generator.cpp | 8 ++++ mapproxy/src/mapproxy/sink.hpp | 13 +----- mapproxy/src/mapproxy/support/aborter.hpp | 45 +++++++++++++++++++ mapproxy/src/mapproxy/support/wmts.cpp | 6 +-- mapproxy/src/mapproxy/support/wmts.hpp | 4 +- 12 files changed, 100 insertions(+), 40 deletions(-) create mode 100644 mapproxy/src/mapproxy/support/aborter.hpp diff --git a/externals/gdal-drivers b/externals/gdal-drivers index 99e5446..9c072af 160000 --- a/externals/gdal-drivers +++ b/externals/gdal-drivers @@ -1 +1 @@ -Subproject commit 99e54468cce3ec60d93d013b40cd0e4a83728f2b +Subproject commit 9c072aff148895fcb1f40ddb4e804527be872d84 diff --git a/externals/libgeo b/externals/libgeo index 19b26f7..0a39ae0 160000 --- a/externals/libgeo +++ b/externals/libgeo @@ -1 +1 @@ -Subproject commit 19b26f725bdb5e2dc1f226385d4df01b8e5eb130 +Subproject commit 0a39ae07a53d30969050ddbb537799c95fbc1852 diff --git a/externals/vts-libs b/externals/vts-libs index b26f161..79a9050 160000 --- a/externals/vts-libs +++ b/externals/vts-libs @@ -1 +1 @@ -Subproject commit b26f161898d03a6fd30c6473966c23bd1c5ae43a +Subproject commit 79a905002b24ed5e8ed55fe7d5f1c558f4cfb168 diff --git a/mapproxy/src/mapproxy/CMakeLists.txt b/mapproxy/src/mapproxy/CMakeLists.txt index 302f5b6..68038f3 100644 --- a/mapproxy/src/mapproxy/CMakeLists.txt +++ b/mapproxy/src/mapproxy/CMakeLists.txt @@ -1,3 +1,4 @@ +# ------------------------------------------------------------------------ # core mapproxy library set(mapproxy-core_SOURCES error.hpp @@ -9,6 +10,7 @@ set(mapproxy-core_SOURCES support/tileindex.hpp support/tileindex.cpp support/fileclass.hpp support/fileclass.cpp support/serialization.hpp support/serialization.cpp + support/aborter.hpp support/glob.hpp support/glob.cpp support/tilejson.hpp support/tilejson.cpp support/cesiumconf.hpp support/cesiumconf.cpp @@ -53,12 +55,11 @@ set(mapproxy-core_SOURCES mapproxy.hpp mapproxy.cpp ) -# test program define_module(LIBRARY mapproxy-core DEPENDS qmf pysupport pydbglog - vts-libs>=2.4 http>=1.0 geo>=1.28 gdal-drivers>=1.14 geometry>=1.7 + vts-libs>=2.4 http>=1.0 geo>=1.28 geometry>=1.7 jsoncpp>=2.1 Boost_SERIALIZATION @@ -74,10 +75,32 @@ buildsys_library(mapproxy-core) target_link_libraries(mapproxy-core ${MODULE_LIBRARIES}) buildsys_target_compile_definitions(mapproxy-core ${MODULE_DEFINITIONS}) -# test program +# ------------------------------------------------------------------------ +# GDAL machinery +set(mapproxy-gdal_SOURCES + gdalsupport.hpp gdalsupport/gdalsupport.cpp + gdalsupport/types.hpp + gdalsupport/requests.hpp gdalsupport/requests.cpp + gdalsupport/process.hpp gdalsupport/process.cpp + gdalsupport/datasetcache.hpp gdalsupport/datasetcache.cpp + gdalsupport/operations.hpp gdalsupport/operations.cpp + ) + +define_module(LIBRARY mapproxy-gdal + DEPENDS mapproxy-core + gdal-drivers>=1.14 + ) + +add_library(mapproxy-gdal STATIC ${mapproxy-gdal_SOURCES}) +buildsys_library(mapproxy-gdal) +target_link_libraries(mapproxy-gdal ${MODULE_LIBRARIES}) +buildsys_target_compile_definitions(mapproxy-gdal ${MODULE_DEFINITIONS}) + +# ------------------------------------------------------------------------ +# Main mapproxy binary define_module(BINARY mapproxy DEPENDS - mapproxy-core service>=1.6 + mapproxy-gdal mapproxy-core service>=1.6 Boost_SERIALIZATION Boost_FILESYSTEM @@ -123,13 +146,6 @@ set(mapproxy_SOURCES fileinfo.hpp fileinfo.cpp core.hpp core.cpp - gdalsupport.hpp gdalsupport/gdalsupport.cpp - gdalsupport/types.hpp - gdalsupport/requests.hpp gdalsupport/requests.cpp - gdalsupport/process.hpp gdalsupport/process.cpp - gdalsupport/datasetcache.hpp gdalsupport/datasetcache.cpp - gdalsupport/operations.hpp gdalsupport/operations.cpp - main.cpp ) diff --git a/mapproxy/src/mapproxy/gdalsupport.hpp b/mapproxy/src/mapproxy/gdalsupport.hpp index 1378a5b..af3cb07 100644 --- a/mapproxy/src/mapproxy/gdalsupport.hpp +++ b/mapproxy/src/mapproxy/gdalsupport.hpp @@ -42,18 +42,12 @@ #include "geo/vectorformat.hpp" #include "geo/heightcoding.hpp" -#include "vts-libs/vts/nodeinfo.hpp" - #include "support/geo.hpp" #include "support/layerenancer.hpp" -#include "sink.hpp" - -namespace vts = vtslibs::vts; +#include "support/aborter.hpp" class GdalWarper { public: - typedef std::shared_ptr pointer; - struct Options { unsigned int processCount; boost::filesystem::path tmpRoot; diff --git a/mapproxy/src/mapproxy/gdalsupport/gdalsupport.cpp b/mapproxy/src/mapproxy/gdalsupport/gdalsupport.cpp index b0947fc..dad7174 100644 --- a/mapproxy/src/mapproxy/gdalsupport/gdalsupport.cpp +++ b/mapproxy/src/mapproxy/gdalsupport/gdalsupport.cpp @@ -716,6 +716,8 @@ void GdalWarper::Detail::cleanup(bool join) cond().notify_all(); } + // TODO: kill unresponsive processes + // cleanup while (!workers_.empty()) { auto head(workers_.begin()); diff --git a/mapproxy/src/mapproxy/gdalsupport/operations.cpp b/mapproxy/src/mapproxy/gdalsupport/operations.cpp index becd0ab..090f728 100644 --- a/mapproxy/src/mapproxy/gdalsupport/operations.cpp +++ b/mapproxy/src/mapproxy/gdalsupport/operations.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,8 @@ #include +#include "utility/gccversion.hpp" + #include "imgproc/rastermask/cvmat.hpp" #include "geo/verticaladjuster.hpp" @@ -60,7 +63,8 @@ cv::Mat* allocateMat(ManagedBuffer &mb const auto matSize(sizeof(cv::Mat) + dataSize); // create raw memory to hold matrix and data - char *raw(static_cast(mb.allocate(matSize))); + char *raw(static_cast + (mb.allocate_aligned(matSize, alignof(cv::Mat)))); // allocate matrix in raw data block return new (raw) cv::Mat(size.height, size.width, type @@ -401,7 +405,9 @@ allocateHc(ManagedBuffer &mb { // create raw memory to hold block and data char *raw(static_cast - (mb.allocate(sizeof(GdalWarper::Heightcoded) + data.size()))); + (mb.allocate_aligned + (sizeof(GdalWarper::Heightcoded) + data.size() + , alignof(GdalWarper::Heightcoded)))); // poiter to output data auto *dataPtr(raw + sizeof(GdalWarper::Heightcoded)); diff --git a/mapproxy/src/mapproxy/generator/generator.cpp b/mapproxy/src/mapproxy/generator/generator.cpp index 08dc6b5..7182754 100644 --- a/mapproxy/src/mapproxy/generator/generator.cpp +++ b/mapproxy/src/mapproxy/generator/generator.cpp @@ -227,12 +227,18 @@ bool isRemote(const std::string &path) || ba::istarts_with(path, "ftp:"))); } +bool isMvt(const std::string &path) +{ + return ba::starts_with(path, "mvt:"); +} + } // namespace std::string Generator::absoluteDataset(const std::string &path) const { // handle non-path resources (i.e. URL's) + if (isMvt(path)) { return "mvt:" + absoluteDataset(path.substr(4)); } if (isRemote(path)) { return path; } return absolute(path, config_.resourceRoot).string(); } @@ -241,7 +247,9 @@ boost::filesystem::path Generator::absoluteDataset(const boost::filesystem::path &path) const { + const auto &spath(path.string()); // handle non-path resources (i.e. URL's) + if (isMvt(spath)) { return "mvt:" + absoluteDataset(spath.substr(4)); } if (isRemote(path.string())) { return path; } return absolute(path, config_.resourceRoot); } diff --git a/mapproxy/src/mapproxy/sink.hpp b/mapproxy/src/mapproxy/sink.hpp index 5013c5f..c455060 100644 --- a/mapproxy/src/mapproxy/sink.hpp +++ b/mapproxy/src/mapproxy/sink.hpp @@ -41,21 +41,10 @@ #include "vts-libs/storage/streams.hpp" #include "support/fileclass.hpp" +#include "support/aborter.hpp" namespace vs = vtslibs::storage; -/** Aborter helper. - */ -struct Aborter { - typedef http::ServerSink::AbortedCallback AbortedCallback; - - virtual ~Aborter() {} - - /** Defaults to dummy aborter - */ - virtual void setAborter(const AbortedCallback&) {}; -}; - /** Wraps libhttp's sink. */ class Sink : public Aborter { diff --git a/mapproxy/src/mapproxy/support/aborter.hpp b/mapproxy/src/mapproxy/support/aborter.hpp new file mode 100644 index 0000000..4c503fc --- /dev/null +++ b/mapproxy/src/mapproxy/support/aborter.hpp @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2017 Melown Technologies SE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef mapproxy_support_aborter_hpp_included_ +#define mapproxy_support_aborter_hpp_included_ + +#include + +/** Aborter helper. + */ +struct Aborter { + typedef std::function AbortedCallback; + + virtual ~Aborter() {} + + /** Defaults to dummy aborter + */ + virtual void setAborter(const AbortedCallback&) {}; +}; + +#endif // mapproxy_support_aborter_hpp_included_ + diff --git a/mapproxy/src/mapproxy/support/wmts.cpp b/mapproxy/src/mapproxy/support/wmts.cpp index f3cb0c5..a328a07 100644 --- a/mapproxy/src/mapproxy/support/wmts.cpp +++ b/mapproxy/src/mapproxy/support/wmts.cpp @@ -266,8 +266,8 @@ std::string makeTemplate(const Layer &layer) } os << "{TileMatrix}-{TileCol}-{TileRow}." << layer.format; - if (layer.resource.revision) { - os << "?r=" << layer.resource.revision; + if (layer.resource->revision) { + os << "?r=" << layer.resource->revision; } return os.str(); @@ -480,7 +480,7 @@ void content(XML &x, const Layer::list &layers) Element c(x, "Contents"); for (const auto &layer : layers) { - const auto &r(layer.resource); + const auto &r(*layer.resource); const TmsEntry &tms(cache.get(r)); diff --git a/mapproxy/src/mapproxy/support/wmts.hpp b/mapproxy/src/mapproxy/support/wmts.hpp index a7abdc5..45573f6 100644 --- a/mapproxy/src/mapproxy/support/wmts.hpp +++ b/mapproxy/src/mapproxy/support/wmts.hpp @@ -32,14 +32,14 @@ namespace wmts { struct Layer { - const Resource &resource; + const Resource *resource; std::string rootPath; RasterFormat format; typedef std::vector list; Layer(const Resource &resource) - : resource(resource), format(RasterFormat::jpg) + : resource(&resource), format(RasterFormat::jpg) {} };