From c2b910bc7a69b461f8620c6eb58334341831fcdb Mon Sep 17 00:00:00 2001 From: Tigran Mkrtchyan Date: Tue, 4 May 2021 20:15:53 +0200 Subject: [PATCH] dcache-chimera: use RemotePoolMonitor to discover file locality Motivation: The DCacheAwareJdbcFs exposes file locality and uses in instance of RemotePoolMonitor for that. Unfortunately, a new instance is requested for each requests, which is more expensive, than directly querying the PoolManager. Modification: Update DCacheAwareJdbcFs to a single instance of RemotePoolMonitor. Update nfs door to subscribe to pool-monitor topic. Result: A single instance of RemotePoolMonitor is shared among multiple getlocality requests over nfs. Acked-by: Paul Millar Acked-by: Lea Morschel Acked-by: Albert Rossi Target: master, 7.1, 7.0, 6.2 Require-book: no Require-notes: yes (cherry picked from commit 9f69e5c4467f36e69ec8a8e9514f0c691a82f5a8) Signed-off-by: Tigran Mkrtchyan --- .../org/dcache/chimera/DCacheAwareJdbcFs.java | 26 +++++++++---------- .../chimera/nfsv41/door/nfsv41-common.xml | 8 +++++- skel/share/defaults/nfs.properties | 7 ++++- skel/share/services/nfs.batch | 1 + 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java index 3cbdb3c409b..ef5c2118a07 100644 --- a/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java +++ b/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java @@ -61,6 +61,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import com.google.common.base.Throwables; import org.apache.curator.shaded.com.google.common.collect.ImmutableMap; +import org.dcache.poolmanager.RemotePoolMonitor; import org.springframework.beans.factory.annotation.Required; import org.springframework.transaction.PlatformTransactionManager; @@ -88,7 +89,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import diskCacheV111.util.PnfsId; import diskCacheV111.vehicles.DCapProtocolInfo; import diskCacheV111.vehicles.DoorRequestInfoMessage; -import diskCacheV111.vehicles.PoolManagerGetPoolMonitor; import diskCacheV111.vehicles.ProtocolInfo; import dmg.cells.nucleus.CellAddressCore; @@ -103,7 +103,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.pinmanager.PinManagerListPinsMessage.Info; import org.dcache.pinmanager.PinManagerPinMessage; import org.dcache.pinmanager.PinManagerUnpinMessage; -import org.dcache.poolmanager.PoolMonitor; import org.dcache.vehicles.FileAttributes; @@ -177,13 +176,17 @@ public boolean isUnpinnable() PinManagerListPinsMessage.State.PINNED, PinState.PINNED, PinManagerListPinsMessage.State.UNPINNING, PinState.UNPINNING); - private CellStub poolManagerStub; private CellStub pinManagerStub; private CellStub billingStub; private PnfsHandler pnfsHandler; private CellAddressCore myAddress; private boolean queryPnfsManagerOnRename; + /** + * Pool manager's runtime configuration. + */ + private RemotePoolMonitor poolMonitor; + @Required public void setQueryPnfsManagerOnRename(boolean yes) { @@ -204,10 +207,6 @@ public void setPnfsHandler(PnfsHandler pnfsHandler) { this.pnfsHandler = pnfsHandler; } - public void setPoolManagerStub(CellStub poolManagerStub) { - this.poolManagerStub = poolManagerStub; - } - public void setPinManagerStub(CellStub pinManagerStub) { this.pinManagerStub = pinManagerStub; } @@ -216,6 +215,11 @@ public void setBillingStub(CellStub billingStub) { this.billingStub = billingStub; } + public void setPoolMonitor(RemotePoolMonitor poolMonitor) + { + this.poolMonitor = poolMonitor; + } + @Override public void setCellAddress(CellAddressCore address) { myAddress = address; @@ -326,13 +330,9 @@ public void remove(FsInode directory, String name, FsInode inode) throws Chimera * instead of simply its status as recorded in the Chimera database. */ private String getFileLocality(String filePath) throws ChimeraFsException { - PoolMonitor _poolMonitor; FileLocality locality = FileLocality.UNAVAILABLE; try { - _poolMonitor = poolManagerStub.sendAndWait( - new PoolManagerGetPoolMonitor()).getPoolMonitor(); - Set requestedAttributes = EnumSet.of(FileAttribute.TYPE, FileAttribute.SIZE, @@ -347,8 +347,8 @@ private String getFileLocality(String filePath) throws ChimeraFsException { * client so that link net masks do not interfere; note that SRM uses * "localhost", so it is not a deviation from existing behavior. */ - locality = _poolMonitor.getFileLocality(attributes, "localhost"); - } catch (CacheException | NoRouteToCellException | InterruptedException t) { + locality = poolMonitor.getFileLocality(attributes, "localhost"); + } catch (CacheException t) { throw new ChimeraFsException("getFileLocality", t); } diff --git a/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml b/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml index 336d86afadc..54141280fae 100644 --- a/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml +++ b/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml @@ -104,13 +104,19 @@ + + + Maintains runtime information about all pools + + + Chimera Filesystem - + diff --git a/skel/share/defaults/nfs.properties b/skel/share/defaults/nfs.properties index b4bd83c515a..95b7e35e75a 100644 --- a/skel/share/defaults/nfs.properties +++ b/skel/share/defaults/nfs.properties @@ -15,7 +15,7 @@ nfs.cell.name=NFS-${host.name} # consume from. # nfs.cell.consume = ${nfs.cell.name} -nfs.cell.subscribe=${nfs.loginbroker.request-topic},PoolStatusTopic +nfs.cell.subscribe=${nfs.loginbroker.request-topic},PoolStatusTopic,${nfs.pool-monitor-topic} # @@ -110,6 +110,11 @@ nfs.loginbroker.update-threshold=${dcache.loginbroker.update-threshold} nfs.loginbroker.family = file nfs.loginbroker.version = nfs4.1 +# ---- Channel on which pool monitor updates are pushed out. +# +nfs.pool-monitor-topic=${dcache.pool-monitor.topic} + + ## This property is a space-separated list of hostnames or IP ## addresses to publish for this door. Hostnames are resolved to an ## IP address on start-up, but not subsequently. Non-wildcard IP diff --git a/skel/share/services/nfs.batch b/skel/share/services/nfs.batch index 2c31ac170ac..a0c57ff940e 100644 --- a/skel/share/services/nfs.batch +++ b/skel/share/services/nfs.batch @@ -50,6 +50,7 @@ check nfs.db.password.file check nfs.domain check nfs.mover.queue check -strong nfs.enable.access-log +check -strong nfs.pool-monitor-topic create org.dcache.cells.UniversalSpringCell ${nfs.cell.name} \ "classpath:org/dcache/chimera/nfsv41/door/nfsv41-common.xml \