From d39bca111508d30f80792aad40fdb64f59ec760b Mon Sep 17 00:00:00 2001 From: Vaclav Blazek Date: Wed, 27 Nov 2019 18:56:18 +0100 Subject: [PATCH] better generator management to allow generator dependencies (not perfect, tho) --- mapproxy/src/mapproxy/generator.hpp | 18 ++++++--- .../src/mapproxy/generator/generators.cpp | 37 ++++++++++--------- .../src/mapproxy/generator/generators.hpp | 6 ++- .../src/mapproxy/generator/surface-meta.cpp | 13 ++----- mapproxy/src/mapproxy/generator/tms-bing.cpp | 1 - .../src/mapproxy/generator/tms-raster.cpp | 3 -- mapproxy/src/mapproxy/resource.cpp | 9 +++++ mapproxy/src/mapproxy/resource.hpp | 2 + 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/mapproxy/src/mapproxy/generator.hpp b/mapproxy/src/mapproxy/generator.hpp index c7f5f22..187833c 100644 --- a/mapproxy/src/mapproxy/generator.hpp +++ b/mapproxy/src/mapproxy/generator.hpp @@ -72,12 +72,14 @@ class GeneratorFinder { std::shared_ptr findGenerator(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const; + , const Resource::Id &resourceId + , bool mustBeReady = true) const; private: virtual std::shared_ptr findGenerator_impl(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const = 0; + , const Resource::Id &resourceId + , bool mustBeReady) const = 0; }; struct GeneratorNotFound : std::runtime_error { @@ -278,6 +280,7 @@ class Generator : boost::noncopyable { Generator::pointer otherGenerator(Resource::Generator::Type generatorType , const Resource::Id &resourceId + , bool mustBeReady = true , bool mandatory = false) const; void supportFile(const vs::SupportFile &support, Sink &sink @@ -415,17 +418,20 @@ inline Generator::pointer Generators::generator(const FileInfo &fileInfo) const std::shared_ptr inline GeneratorFinder::findGenerator(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const + , const Resource::Id &resourceId + , bool mustBeReady) const { - return findGenerator_impl(generatorType, resourceId); + return findGenerator_impl(generatorType, resourceId, mustBeReady); } inline Generator::pointer Generator::otherGenerator(Resource::Generator::Type generatorType - , const Resource::Id &resourceId, bool mandatory) + , const Resource::Id &resourceId + , bool mustBeReady, bool mandatory) const { - auto other(generatorFinder_->findGenerator(generatorType, resourceId)); + auto other(generatorFinder_->findGenerator + (generatorType, resourceId, mustBeReady)); if (!other && mandatory) { throw GeneratorNotFound(generatorType, resourceId); } diff --git a/mapproxy/src/mapproxy/generator/generators.cpp b/mapproxy/src/mapproxy/generator/generators.cpp index 9cf991f..5c368a6 100644 --- a/mapproxy/src/mapproxy/generator/generators.cpp +++ b/mapproxy/src/mapproxy/generator/generators.cpp @@ -422,26 +422,24 @@ void Generators::Detail::update(const Resource::map &resources) } } - // add stuff - for (const auto &generator : toAdd) { - { - std::unique_lock lock(lock_); - serving_.insert(generator); - } - - if (!generator->ready()) { - prepare(generator); - } - } - // remove stuff for (const auto &generator : toRemove) { { std::unique_lock lock(lock_); serving_.erase(generator); } + } + + // add generators; not prepared yet to make them available to the others + // when prepared + for (const auto &generator : toAdd) { + std::unique_lock lock(lock_); + serving_.insert(generator); + } - // TODO: mark as to be removed for prepare workers + // prepare generators if not ready + for (const auto &generator : toAdd) { + if (!generator->ready()) { prepare(generator); } } // replace stuff (prepare) @@ -489,10 +487,11 @@ Generators::Detail::referenceFrame(const std::string &referenceFrame) Generator::pointer Generators::Detail::generator(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) + , const Resource::Id &resourceId + , bool noReadyCheck) const { - checkReady(); + if (!noReadyCheck) { checkReady(); } // find generator (under lock) auto generator([&]() -> Generator::pointer @@ -518,10 +517,12 @@ Generators::Detail::generator(Resource::Generator::Type generatorType Generator::pointer Generators::Detail::findGenerator_impl(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const + , const Resource::Id &resourceId + , bool mustBeReady) const { - auto g(generator(generatorType, resourceId)); - if (!g || !g->ready()) { return {}; } + auto g(generator(generatorType, resourceId, !mustBeReady)); + if (!g) { return {}; } + if (mustBeReady && !g->ready()) { return {}; } return g; } diff --git a/mapproxy/src/mapproxy/generator/generators.hpp b/mapproxy/src/mapproxy/generator/generators.hpp index 153bbfd..d999e43 100644 --- a/mapproxy/src/mapproxy/generator/generators.hpp +++ b/mapproxy/src/mapproxy/generator/generators.hpp @@ -100,7 +100,8 @@ class Generators::Detail void checkReady() const; Generator::pointer generator(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const; + , const Resource::Id &resourceId + , bool noReadyCheck = false) const; Generator::list referenceFrame(const std::string &referenceFrame) const; @@ -149,7 +150,8 @@ class Generators::Detail virtual Generator::pointer findGenerator_impl(Resource::Generator::Type generatorType - , const Resource::Id &resourceId) const; + , const Resource::Id &resourceId + , bool mustBeReady) const; const Config config_; ResourceBackend::pointer resourceBackend_; diff --git a/mapproxy/src/mapproxy/generator/surface-meta.cpp b/mapproxy/src/mapproxy/generator/surface-meta.cpp index b9fdb7e..06735c2 100644 --- a/mapproxy/src/mapproxy/generator/surface-meta.cpp +++ b/mapproxy/src/mapproxy/generator/surface-meta.cpp @@ -108,18 +108,15 @@ void SurfaceMeta::prepare_impl(Arsenal&) { LOG(info2) << "Preparing <" << id() << ">."; - // find surface and tms generators - + // find surface and tms generators (no need to be ready but fail when not + // present) surface_ = otherGenerator (Resource::Generator::Type::surface - , Resource::Id(referenceFrameId(), definition_.surface), true); + , Resource::Id(referenceFrameId(), definition_.surface), false, true); tms_ = otherGenerator (Resource::Generator::Type::tms - , Resource::Id(referenceFrameId(), definition_.tms), true); - - LOG(info4) << "surface: " << surface_; - LOG(info4) << "tms: " << tms_; + , Resource::Id(referenceFrameId(), definition_.tms), false, true); ts_ = surface_->getProvider(); if (!ts_) { @@ -128,7 +125,6 @@ void SurfaceMeta::prepare_impl(Arsenal&) << Resource::Id(referenceFrameId(), definition_.surface) << "> doesn't provide VTS tileset support."; } - LOG(info4) << "ts: " << ts_; atlas_ = tms_->getProvider(); if (!atlas_) { @@ -137,7 +133,6 @@ void SurfaceMeta::prepare_impl(Arsenal&) << Resource::Id(referenceFrameId(), definition_.surface) << "> doesn't provide VTS atlas support."; } - LOG(info4) << "atlas: " << atlas_; } vts::MapConfig SurfaceMeta::mapConfig_impl(ResourceRoot root) const diff --git a/mapproxy/src/mapproxy/generator/tms-bing.cpp b/mapproxy/src/mapproxy/generator/tms-bing.cpp index 9abfb7b..ab9f26d 100644 --- a/mapproxy/src/mapproxy/generator/tms-bing.cpp +++ b/mapproxy/src/mapproxy/generator/tms-bing.cpp @@ -89,7 +89,6 @@ TmsBing::TmsBing(const Params ¶ms) void TmsBing::prepare_impl(Arsenal&) { LOG(info2) << "Preparing <" << id() << ">."; - makeReady(); } namespace { diff --git a/mapproxy/src/mapproxy/generator/tms-raster.cpp b/mapproxy/src/mapproxy/generator/tms-raster.cpp index 122e68c..23afc09 100644 --- a/mapproxy/src/mapproxy/generator/tms-raster.cpp +++ b/mapproxy/src/mapproxy/generator/tms-raster.cpp @@ -174,7 +174,6 @@ void TmsRaster::prepare_impl(Arsenal&) index_ = boost::in_place(deliveryIndexPath); // done - makeReady(); return; } @@ -205,8 +204,6 @@ void TmsRaster::prepare_impl(Arsenal&) // some invalid pixels hasMetatiles_ = !ds.allValid(); } - - makeReady(); } RasterFormat TmsRaster::format() const diff --git a/mapproxy/src/mapproxy/resource.cpp b/mapproxy/src/mapproxy/resource.cpp index 7ba1a9d..bafe95f 100644 --- a/mapproxy/src/mapproxy/resource.cpp +++ b/mapproxy/src/mapproxy/resource.cpp @@ -716,3 +716,12 @@ std::istream& operator>>(std::istream &is, GeneratorInterface &gi) return is; } +Resource::Id::list Resource::needsResources() const +{ + // fetch list of needed resource IDs and inject reference frame + auto neededIds(definition_->needsResources()); + for (auto &neededId : neededIds) { + neededId.referenceFrame = id.referenceFrame; + } + return neededIds; +} diff --git a/mapproxy/src/mapproxy/resource.hpp b/mapproxy/src/mapproxy/resource.hpp index 3d440a7..fdffe96 100644 --- a/mapproxy/src/mapproxy/resource.hpp +++ b/mapproxy/src/mapproxy/resource.hpp @@ -172,6 +172,8 @@ struct Resource { Changed changed(const Resource &o) const; + Id::list needsResources() const; + private: /** Definition: based on type and driver, created by resource * parser/generator and interpreted by driver.