better generator management to allow generator dependencies (not perf…

…ect, tho)
melowntech · Nov 27, 2019 · d39bca1 · d39bca1
1 parent 48c5a52
commit d39bca1
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 39 deletions.
diff --git a/mapproxy/src/mapproxy/generator.hpp b/mapproxy/src/mapproxy/generator.hpp
@@ -72,12 +72,14 @@ class GeneratorFinder {
 
     std::shared_ptr<Generator>
     findGenerator(Resource::Generator::Type generatorType
-                  , const Resource::Id &resourceId) const;
+                  , const Resource::Id &resourceId
+                  , bool mustBeReady = true) const;
 
 private:
     virtual std::shared_ptr<Generator>
     findGenerator_impl(Resource::Generator::Type generatorType
-                       , const Resource::Id &resourceId) const = 0;
+                       , const Resource::Id &resourceId
+                       , bool mustBeReady) const = 0;
 };
 
 struct GeneratorNotFound : std::runtime_error {
@@ -278,6 +280,7 @@ class Generator : boost::noncopyable {
 
     Generator::pointer otherGenerator(Resource::Generator::Type generatorType
                                       , const Resource::Id &resourceId
+                                      , bool mustBeReady = true
                                       , bool mandatory = false) const;
 
     void supportFile(const vs::SupportFile &support, Sink &sink
@@ -415,17 +418,20 @@ inline Generator::pointer Generators::generator(const FileInfo &fileInfo) const
 
 std::shared_ptr<Generator>
 inline GeneratorFinder::findGenerator(Resource::Generator::Type generatorType
-                                      , const Resource::Id &resourceId) const
+                                      , const Resource::Id &resourceId
+                                      , bool mustBeReady) const
 {
-    return findGenerator_impl(generatorType, resourceId);
+    return findGenerator_impl(generatorType, resourceId, mustBeReady);
 }
 
 inline Generator::pointer
 Generator::otherGenerator(Resource::Generator::Type generatorType
-                          , const Resource::Id &resourceId, bool mandatory)
+                          , const Resource::Id &resourceId
+                          , bool mustBeReady, bool mandatory)
     const
 {
-    auto other(generatorFinder_->findGenerator(generatorType, resourceId));
+    auto other(generatorFinder_->findGenerator
+               (generatorType, resourceId, mustBeReady));
     if (!other && mandatory) {
         throw GeneratorNotFound(generatorType, resourceId);
     }

diff --git a/mapproxy/src/mapproxy/generator/generators.cpp b/mapproxy/src/mapproxy/generator/generators.cpp
@@ -422,26 +422,24 @@ void Generators::Detail::update(const Resource::map &resources)
         }
     }
 
-    // add stuff
-    for (const auto &generator : toAdd) {
-        {
-            std::unique_lock<std::mutex> lock(lock_);
-            serving_.insert(generator);
-        }
-
-        if (!generator->ready()) {
-            prepare(generator);
-        }
-    }
-
     // remove stuff
     for (const auto &generator : toRemove) {
         {
             std::unique_lock<std::mutex> lock(lock_);
             serving_.erase(generator);
         }
+    }
+
+    // add generators; not prepared yet to make them available to the others
+    // when prepared
+    for (const auto &generator : toAdd) {
+        std::unique_lock<std::mutex> lock(lock_);
+        serving_.insert(generator);
+    }
 
-        // TODO: mark as to be removed for prepare workers
+    // prepare generators if not ready
+    for (const auto &generator : toAdd) {
+        if (!generator->ready()) { prepare(generator); }
     }
 
     // replace stuff (prepare)
@@ -489,10 +487,11 @@ Generators::Detail::referenceFrame(const std::string &referenceFrame)
 
 Generator::pointer
 Generators::Detail::generator(Resource::Generator::Type generatorType
-                              , const Resource::Id &resourceId)
+                              , const Resource::Id &resourceId
+                              , bool noReadyCheck)
     const
 {
-    checkReady();
+    if (!noReadyCheck) { checkReady(); }
 
     // find generator (under lock)
     auto generator([&]() -> Generator::pointer
@@ -518,10 +517,12 @@ Generators::Detail::generator(Resource::Generator::Type generatorType
 
 Generator::pointer
 Generators::Detail::findGenerator_impl(Resource::Generator::Type generatorType
-                                       , const Resource::Id &resourceId) const
+                                       , const Resource::Id &resourceId
+                                       , bool mustBeReady) const
 {
-    auto g(generator(generatorType, resourceId));
-    if (!g || !g->ready()) { return {}; }
+    auto g(generator(generatorType, resourceId, !mustBeReady));
+    if (!g) { return {}; }
+    if (mustBeReady && !g->ready()) { return {}; }
     return g;
 }
 

diff --git a/mapproxy/src/mapproxy/generator/generators.hpp b/mapproxy/src/mapproxy/generator/generators.hpp
@@ -100,7 +100,8 @@ class Generators::Detail
     void checkReady() const;
 
     Generator::pointer generator(Resource::Generator::Type generatorType
-                                 , const Resource::Id &resourceId) const;
+                                 , const Resource::Id &resourceId
+                                 , bool noReadyCheck = false) const;
 
     Generator::list referenceFrame(const std::string &referenceFrame) const;
 
@@ -149,7 +150,8 @@ class Generators::Detail
 
     virtual Generator::pointer
     findGenerator_impl(Resource::Generator::Type generatorType
-                       , const Resource::Id &resourceId) const;
+                       , const Resource::Id &resourceId
+                       , bool mustBeReady) const;
 
     const Config config_;
     ResourceBackend::pointer resourceBackend_;

diff --git a/mapproxy/src/mapproxy/generator/surface-meta.cpp b/mapproxy/src/mapproxy/generator/surface-meta.cpp
@@ -108,18 +108,15 @@ void SurfaceMeta::prepare_impl(Arsenal&)
 {
     LOG(info2) << "Preparing <" << id() << ">.";
 
-    // find surface and tms generators
-
+    // find surface and tms generators (no need to be ready but fail when not
+    // present)
     surface_ = otherGenerator
         (Resource::Generator::Type::surface
-         , Resource::Id(referenceFrameId(), definition_.surface), true);
+         , Resource::Id(referenceFrameId(), definition_.surface), false, true);
 
     tms_ = otherGenerator
         (Resource::Generator::Type::tms
-         , Resource::Id(referenceFrameId(), definition_.tms), true);
-
-    LOG(info4) << "surface: " << surface_;
-    LOG(info4) << "tms: " << tms_;
+         , Resource::Id(referenceFrameId(), definition_.tms), false, true);
 
     ts_ = surface_->getProvider<VtsTilesetProvider>();
     if (!ts_) {
@@ -128,7 +125,6 @@ void SurfaceMeta::prepare_impl(Arsenal&)
             << Resource::Id(referenceFrameId(), definition_.surface)
             << "> doesn't provide VTS tileset support.";
     }
-    LOG(info4) << "ts: " << ts_;
 
     atlas_ = tms_->getProvider<VtsAtlasProvider>();
     if (!atlas_) {
@@ -137,7 +133,6 @@ void SurfaceMeta::prepare_impl(Arsenal&)
             << Resource::Id(referenceFrameId(), definition_.surface)
             << "> doesn't provide VTS atlas support.";
     }
-    LOG(info4) << "atlas: " << atlas_;
 }
 
 vts::MapConfig SurfaceMeta::mapConfig_impl(ResourceRoot root) const

diff --git a/mapproxy/src/mapproxy/generator/tms-bing.cpp b/mapproxy/src/mapproxy/generator/tms-bing.cpp
@@ -89,7 +89,6 @@ TmsBing::TmsBing(const Params &params)
 void TmsBing::prepare_impl(Arsenal&)
 {
     LOG(info2) << "Preparing <" << id() << ">.";
-    makeReady();
 }
 
 namespace {

diff --git a/mapproxy/src/mapproxy/generator/tms-raster.cpp b/mapproxy/src/mapproxy/generator/tms-raster.cpp
@@ -174,7 +174,6 @@ void TmsRaster::prepare_impl(Arsenal&)
         index_ = boost::in_place(deliveryIndexPath);
 
         // done
-        makeReady();
         return;
     }
 
@@ -205,8 +204,6 @@ void TmsRaster::prepare_impl(Arsenal&)
         // some invalid pixels
         hasMetatiles_ = !ds.allValid();
     }
-
-    makeReady();
 }
 
 RasterFormat TmsRaster::format() const

diff --git a/mapproxy/src/mapproxy/resource.cpp b/mapproxy/src/mapproxy/resource.cpp
@@ -716,3 +716,12 @@ std::istream& operator>>(std::istream &is, GeneratorInterface &gi)
     return is;
 }
 
+Resource::Id::list Resource::needsResources() const
+{
+    // fetch list of needed resource IDs and inject reference frame
+    auto neededIds(definition_->needsResources());
+    for (auto &neededId : neededIds) {
+        neededId.referenceFrame = id.referenceFrame;
+    }
+    return neededIds;
+}
diff --git a/mapproxy/src/mapproxy/resource.hpp b/mapproxy/src/mapproxy/resource.hpp
@@ -172,6 +172,8 @@ struct Resource {
 
     Changed changed(const Resource &o) const;
 
+    Id::list needsResources() const;
+
 private:
     /** Definition: based on type and driver, created by resource
      *  parser/generator and interpreted by driver.