dcache-chimera: use RemotePoolMonitor to discover file locality

Motivation: The DCacheAwareJdbcFs exposes file locality and uses in instance of RemotePoolMonitor for that. Unfortunately, a new instance is requested for each requests, which is more expensive, than directly querying the PoolManager. Modification: Update DCacheAwareJdbcFs to a single instance of RemotePoolMonitor. Update nfs door to subscribe to pool-monitor topic. Result: A single instance of RemotePoolMonitor is shared among multiple getlocality requests over nfs. Acked-by: Paul Millar Acked-by: Lea Morschel Acked-by: Albert Rossi Target: master, 7.1, 7.0, 6.2 Require-book: no Require-notes: yes (cherry picked from commit 9f69e5c) Signed-off-by: Tigran Mkrtchyan <[email protected]>
paulmillar · May 6, 2021 · c2b910b · c2b910b
1 parent 64bd5d0
commit c2b910b
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 15 deletions.
diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java
@@ -61,6 +61,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 
 import com.google.common.base.Throwables;
 import org.apache.curator.shaded.com.google.common.collect.ImmutableMap;
+import org.dcache.poolmanager.RemotePoolMonitor;
 import org.springframework.beans.factory.annotation.Required;
 import org.springframework.transaction.PlatformTransactionManager;
 
@@ -88,7 +89,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import diskCacheV111.util.PnfsId;
 import diskCacheV111.vehicles.DCapProtocolInfo;
 import diskCacheV111.vehicles.DoorRequestInfoMessage;
-import diskCacheV111.vehicles.PoolManagerGetPoolMonitor;
 import diskCacheV111.vehicles.ProtocolInfo;
 
 import dmg.cells.nucleus.CellAddressCore;
@@ -103,7 +103,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import org.dcache.pinmanager.PinManagerListPinsMessage.Info;
 import org.dcache.pinmanager.PinManagerPinMessage;
 import org.dcache.pinmanager.PinManagerUnpinMessage;
-import org.dcache.poolmanager.PoolMonitor;
 import org.dcache.vehicles.FileAttributes;
 
 
@@ -177,13 +176,17 @@ public boolean isUnpinnable()
             PinManagerListPinsMessage.State.PINNED, PinState.PINNED,
             PinManagerListPinsMessage.State.UNPINNING, PinState.UNPINNING);
 
-    private CellStub poolManagerStub;
     private CellStub pinManagerStub;
     private CellStub billingStub;
     private PnfsHandler pnfsHandler;
     private CellAddressCore myAddress;
     private boolean queryPnfsManagerOnRename;
 
+    /**
+     * Pool manager's runtime configuration.
+     */
+    private RemotePoolMonitor poolMonitor;
+
     @Required
     public void setQueryPnfsManagerOnRename(boolean yes)
     {
@@ -204,10 +207,6 @@ public void setPnfsHandler(PnfsHandler pnfsHandler) {
         this.pnfsHandler = pnfsHandler;
     }
 
-    public void setPoolManagerStub(CellStub poolManagerStub) {
-        this.poolManagerStub = poolManagerStub;
-    }
-
     public void setPinManagerStub(CellStub pinManagerStub) {
         this.pinManagerStub = pinManagerStub;
     }
@@ -216,6 +215,11 @@ public void setBillingStub(CellStub billingStub) {
         this.billingStub = billingStub;
     }
 
+    public void setPoolMonitor(RemotePoolMonitor poolMonitor)
+    {
+        this.poolMonitor = poolMonitor;
+    }
+
     @Override
     public void setCellAddress(CellAddressCore address) {
         myAddress = address;
@@ -326,13 +330,9 @@ public void remove(FsInode directory, String name, FsInode inode) throws Chimera
      * instead of simply its status as recorded in the Chimera database.
      */
     private String getFileLocality(String filePath) throws ChimeraFsException {
-        PoolMonitor _poolMonitor;
         FileLocality locality = FileLocality.UNAVAILABLE;
 
         try {
-            _poolMonitor = poolManagerStub.sendAndWait(
-                            new PoolManagerGetPoolMonitor()).getPoolMonitor();
-
             Set<FileAttribute> requestedAttributes
                 = EnumSet.of(FileAttribute.TYPE,
                              FileAttribute.SIZE,
@@ -347,8 +347,8 @@ private String getFileLocality(String filePath) throws ChimeraFsException {
              * client so that link net masks do not interfere; note that SRM uses
              * "localhost", so it is not a deviation from existing behavior.
              */
-            locality = _poolMonitor.getFileLocality(attributes, "localhost");
-        } catch (CacheException | NoRouteToCellException | InterruptedException t) {
+            locality = poolMonitor.getFileLocality(attributes, "localhost");
+        } catch (CacheException t) {
             throw new ChimeraFsException("getFileLocality", t);
         }
 

diff --git a/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml b/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml
@@ -104,13 +104,19 @@
         <property name="shouldUpdate" value="${nfs.db.schema.auto}"/>
     </bean>
 
+
+    <bean id="pool-monitor" class="org.dcache.poolmanager.RemotePoolMonitor">
+        <description>Maintains runtime information about all pools</description>
+        <property name="poolManagerStub" ref="poolManagerStub"/>
+    </bean>
+
     <bean id="fileSystem" class="org.dcache.chimera.DCacheAwareJdbcFs"
             depends-on="liquibase">
         <description>Chimera Filesystem</description>
         <constructor-arg ref="dataSource" />
         <constructor-arg ref="tx-manager" />
         <property name="pnfsHandler" ref="pnfs"/>
-        <property name="poolManagerStub" ref="poolManagerStub"/>
+        <property name="poolMonitor" ref="pool-monitor"/>
         <property name="pinManagerStub" ref="pinManagerStub"/>
         <property name="billingStub" ref="billing-stub"/>
 	<property name="queryPnfsManagerOnRename" value="${nfs.enable.pnfsmanager-query-on-move}"/>

diff --git a/skel/share/defaults/nfs.properties b/skel/share/defaults/nfs.properties
@@ -15,7 +15,7 @@ nfs.cell.name=NFS-${host.name}
 #   consume from.
 #
 nfs.cell.consume = ${nfs.cell.name}
-nfs.cell.subscribe=${nfs.loginbroker.request-topic},PoolStatusTopic
+nfs.cell.subscribe=${nfs.loginbroker.request-topic},PoolStatusTopic,${nfs.pool-monitor-topic}
 
 
 #
@@ -110,6 +110,11 @@ nfs.loginbroker.update-threshold=${dcache.loginbroker.update-threshold}
 nfs.loginbroker.family = file
 nfs.loginbroker.version = nfs4.1
 
+# ---- Channel on which pool monitor updates are pushed out.
+#
+nfs.pool-monitor-topic=${dcache.pool-monitor.topic}
+
+
 ##  This property is a space-separated list of hostnames or IP
 ##  addresses to publish for this door.  Hostnames are resolved to an
 ##  IP address on start-up, but not subsequently.  Non-wildcard IP

diff --git a/skel/share/services/nfs.batch b/skel/share/services/nfs.batch
@@ -50,6 +50,7 @@ check nfs.db.password.file
 check nfs.domain
 check nfs.mover.queue
 check -strong nfs.enable.access-log
+check -strong nfs.pool-monitor-topic
 
 create org.dcache.cells.UniversalSpringCell ${nfs.cell.name} \
         "classpath:org/dcache/chimera/nfsv41/door/nfsv41-common.xml \