From 05d56ff601469733d9c2f74ab48f8babfe4b606a Mon Sep 17 00:00:00 2001 From: Dmitry Litvintsev Date: Fri, 6 Dec 2024 10:32:59 -0600 Subject: [PATCH] bulk: handle absolute/relative paths in uniform fashion Motivation: ----------- Recent change(s) that massaged user input target paths and stored absolute paths on bulk backend lead to ambiguity between user provided and dcache resolved paths and also resulted in inability to use full paths (i.e. only relative paths are supported). At Fermilab we need to use both - relative and absolute paths Modification: ------------- Revert all recent changes that appended prefix to user supplied paths, stored the result and then stripped the prefix so that only "original" paths are exposed to the user. Instead, like before, store user supplied paths but carry over request prefix which is computed from user root and door root. When calling PnfsManager using paths the full paths of the targets are reassembled using the prefix Result: ------ Restored ability to use absolute paths when using REST API. Issue: https://github.com/dCache/dcache/issues/7693 Patch: https://rb.dcache.org/r/14355/ Target: trunk Request: 10.2, 10.1, 10.0, 9.2 Require-book: no Require-notes: yes Acked-by: Lea Morschel, Tigran Mkrtchyan --- .../services/bulk/activity/BulkActivity.java | 5 +- .../plugin/delete/DeleteActivity.java | 5 +- .../plugin/log/LogTargetActivity.java | 2 +- .../bulk/activity/plugin/pin/PinActivity.java | 13 ++-- .../plugin/pin/PinManagerActivity.java | 6 +- .../activity/plugin/pin/ReleaseActivity.java | 6 +- .../activity/plugin/pin/StageActivity.java | 16 ++-- .../activity/plugin/pin/UnpinActivity.java | 6 +- .../plugin/qos/UpdateQoSActivity.java | 28 +++++-- .../bulk/job/BulkRequestContainerJob.java | 76 +++++-------------- .../services/bulk/util/BulkRequestTarget.java | 13 ++-- .../restful/resources/bulk/BulkResources.java | 22 +++++- .../resources/tape/ArchiveInfoResources.java | 12 ++- .../resources/tape/ReleaseResources.java | 3 +- .../resources/tape/StageResources.java | 8 +- .../util/wlcg/ArchiveInfoCollector.java | 23 +++++- .../bulk/BulkRequestJsonParseTest.java | 3 +- 17 files changed, 129 insertions(+), 118 deletions(-) diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/BulkActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/BulkActivity.java index 49f82faabe0..c1e5f00a60a 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/BulkActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/BulkActivity.java @@ -114,7 +114,7 @@ protected BulkActivity(String name, TargetType targetType) { retryPolicy = DEFAULT_RETRY_POLICY; } - public void cancel(BulkRequestTarget target) { + public void cancel(String prefix, BulkRequestTarget target) { target.cancel(); } @@ -173,11 +173,12 @@ public void setDescriptors(Set descriptors) { * * @param rid of the request. * @param tid of the target. + * @param prefix target prefix * @param path of the target on which to perform the activity. * @return future result of the activity. * @throws BulkServiceException */ - public abstract ListenableFuture perform(String rid, long tid, FsPath path, FileAttributes attributes) + public abstract ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) throws BulkServiceException; /** diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/delete/DeleteActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/delete/DeleteActivity.java index 8c3687326b6..7a7d04eae41 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/delete/DeleteActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/delete/DeleteActivity.java @@ -89,9 +89,10 @@ public DeleteActivity(String name, TargetType targetType) { } @Override - public ListenableFuture perform(String rid, long tid, FsPath path, + public ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) { - PnfsDeleteEntryMessage msg = new PnfsDeleteEntryMessage(path.toString()); + FsPath absolutePath = BulkRequestTarget.computeFsPath(prefix, path.toString()); + PnfsDeleteEntryMessage msg = new PnfsDeleteEntryMessage(absolutePath.toString()); msg.setSubject(subject); if (attributes != null && attributes.getFileType() == FileType.DIR && skipDirs) { msg.setSucceeded(); diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/log/LogTargetActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/log/LogTargetActivity.java index 330ac0158ba..fb00b84186a 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/log/LogTargetActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/log/LogTargetActivity.java @@ -89,7 +89,7 @@ protected LogTargetActivity(String name, TargetType targetType) { } @Override - public ListenableFuture perform(String ruid, long tid, FsPath path, + public ListenableFuture perform(String ruid, long tid, String prefix, FsPath path, FileAttributes attributes) { long now = System.currentTimeMillis(); BulkRequestTarget t = BulkRequestTargetBuilder.builder(null).activity(this.getName()).id(tid) diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinActivity.java index 9c237b97cc0..27ed977e9b0 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinActivity.java @@ -91,18 +91,18 @@ public PinActivity(String name, TargetType targetType) { super(name, targetType); } - public void cancel(BulkRequestTarget target) { - super.cancel(target); + public void cancel(String prefix, BulkRequestTarget target) { + super.cancel(prefix, target); try { - pinManager.send(unpinMessage(id, target)); + pinManager.send(unpinMessage(id, prefix, target)); } catch (CacheException e) { - target.setErrorObject(new BulkServiceException("unable to fetch pnfsid of target in " + target.setErrorObject(new BulkServiceException("unable to fetch pnfsid of target in " + "order to cancel pinning.", e)); } } @Override - public ListenableFuture perform(String rid, long tid, FsPath target, + public ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) { if (id == null) { id = rid; @@ -110,7 +110,8 @@ public ListenableFuture perform(String rid, long tid, FsPath target, try { if (attributes == null) { - attributes = getAttributes(target); + FsPath absolutePath = BulkRequestTarget.computeFsPath(prefix, path.toString()); + attributes = getAttributes(absolutePath); } checkPinnable(attributes); diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinManagerActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinManagerActivity.java index 02019cfd741..dcbbd8fbfd9 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinManagerActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/PinManagerActivity.java @@ -61,6 +61,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import static com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly; import static diskCacheV111.util.CacheException.INVALID_ARGS; +import static org.dcache.services.bulk.util.BulkRequestTarget.computeFsPath; import static org.dcache.services.bulk.util.BulkRequestTarget.State.SKIPPED; import com.google.common.util.concurrent.Futures; @@ -130,11 +131,12 @@ protected FileAttributes getAttributes(FsPath path) throws CacheException { return pnfsHandler.getFileAttributes(path, MINIMALLY_REQUIRED_ATTRIBUTES); } - protected PinManagerUnpinMessage unpinMessage(String id, BulkRequestTarget target) + protected PinManagerUnpinMessage unpinMessage(String id, String prefix, BulkRequestTarget target) throws CacheException { PnfsId pnfsId = target.getPnfsId(); if (pnfsId == null) { - pnfsId = getAttributes(target.getPath()).getPnfsId(); + FsPath absolutePath = computeFsPath(prefix, target.getPath().toString()); + pnfsId = getAttributes(absolutePath).getPnfsId(); } return unpinMessage(id, pnfsId); } diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/ReleaseActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/ReleaseActivity.java index be11099b34f..a3e2f157471 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/ReleaseActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/ReleaseActivity.java @@ -60,6 +60,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING package org.dcache.services.bulk.activity.plugin.pin; import static org.dcache.services.bulk.activity.plugin.pin.ReleaseActivityProvider.REQUEST_ID; +import static org.dcache.services.bulk.util.BulkRequestTarget.computeFsPath; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -77,11 +78,12 @@ public ReleaseActivity(String name, TargetType targetType) { } @Override - public ListenableFuture perform(String rid, long tid, FsPath target, + public ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) { try { if (attributes == null) { - attributes = getAttributes(target); + FsPath absolutePath = computeFsPath(prefix, path.toString()); + attributes = getAttributes(absolutePath); } return pinManager.send(unpinMessage(id, attributes.getPnfsId())); } catch (CacheException e) { diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/StageActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/StageActivity.java index b460b261530..d915ec463c1 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/StageActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/StageActivity.java @@ -102,18 +102,18 @@ public StageActivity(String name, TargetType targetType) { super(name, targetType); } - public void cancel(BulkRequestTarget target) { - super.cancel(target); + public void cancel(String prefix, BulkRequestTarget target) { + super.cancel(prefix, target); try { - pinManager.send(unpinMessage(id, target)); + pinManager.send(unpinMessage(id, prefix, target)); } catch (CacheException e) { target.setErrorObject(new BulkServiceException("unable to fetch pnfsid of target in " - + "order to cancel staging.", e)); + + "order to cancel staging.", e)); } } @Override - public ListenableFuture perform(String rid, long tid, FsPath target, + public ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) { id = rid; @@ -121,7 +121,9 @@ public ListenableFuture perform(String rid, long tid, FsPath target, /* * refetch the attributes because RP is not stored in the bulk database. */ - attributes = getAttributes(target); + + FsPath absolutePath = BulkRequestTarget.computeFsPath(prefix, path.toString()); + attributes = getAttributes(absolutePath); checkStageable(attributes); @@ -129,7 +131,7 @@ public ListenableFuture perform(String rid, long tid, FsPath target, = new PinManagerPinMessage(attributes, getProtocolInfo(), pnfsHandler.getRestriction(), id, - getLifetimeInMillis(target)); + getLifetimeInMillis(path)); message.setSubject(subject); Optional> skipOption = skipIfOnline(attributes, message); diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/UnpinActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/UnpinActivity.java index db2106b1b95..44a44473851 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/UnpinActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/pin/UnpinActivity.java @@ -60,6 +60,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING package org.dcache.services.bulk.activity.plugin.pin; import static org.dcache.services.bulk.activity.plugin.pin.UnpinActivityProvider.UNPIN_REQUEST_ID; +import static org.dcache.services.bulk.util.BulkRequestTarget.computeFsPath; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -78,11 +79,12 @@ public UnpinActivity(String name, TargetType targetType) { } @Override - public ListenableFuture perform(String rid, long tid, FsPath target, + public ListenableFuture perform(String rid, long tid, String prefix, FsPath path, FileAttributes attributes) { try { if (attributes == null) { - attributes = getAttributes(target); + FsPath absolutePath = computeFsPath(prefix, path.toString()); + attributes = getAttributes(absolutePath); } return pinManager.send(unpinMessage(id, attributes.getPnfsId())); } catch (CacheException e) { diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/qos/UpdateQoSActivity.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/qos/UpdateQoSActivity.java index e61b169dab0..4df8fb911a3 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/qos/UpdateQoSActivity.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/activity/plugin/qos/UpdateQoSActivity.java @@ -112,10 +112,24 @@ public UpdateQoSActivity(String name, TargetType targetType) { } @Override - public synchronized void cancel(BulkRequestTarget target) { + public synchronized void cancel(String prefix, BulkRequestTarget target) { RemoteQoSRequirementsClient client = new RemoteQoSRequirementsClient(); client.setRequirementsService(qosEngine); - PnfsId pnfsId = target.getAttributes().getPnfsId(); + PnfsId pnfsId = null; + if (target.getAttributes() == null) { + FsPath absolutePath = BulkRequestTarget.computeFsPath(prefix, + target.getPath().toString()); + try { + pnfsId = pnfsHandler.getFileAttributes(absolutePath.toString(), + MINIMALLY_REQUIRED_ATTRIBUTES).getPnfsId(); + } catch (CacheException e) { + LOGGER.error("fileQoSRequirementsModifiedCancelled failed: failed to fetch attributes for {} {}.", + target.getPath().toString(), + e.getMessage()); + } + } else { + pnfsId = target.getAttributes().getPnfsId(); + } try { client.fileQoSRequirementsModifiedCancelled(pnfsId, subject); } catch (QoSException e) { @@ -123,19 +137,20 @@ public synchronized void cancel(BulkRequestTarget target) { e.getMessage()); } responseReceiver.cancel(pnfsId.toString()); - super.cancel(target); + super.cancel(prefix, target); } @Override public ListenableFuture perform(String rid, long tid, - FsPath path, FileAttributes attributes) throws BulkServiceException { + String prefix, FsPath path, FileAttributes attributes) throws BulkServiceException { if (targetQos == null && qosPolicy == null) { return Futures.immediateFailedFuture(new IllegalArgumentException("no target qos or policy given.")); } if (attributes == null) { try { - attributes = pnfsHandler.getFileAttributes(path, MINIMALLY_REQUIRED_ATTRIBUTES); + FsPath absolutePath = BulkRequestTarget.computeFsPath(prefix, path.toString()); + attributes = pnfsHandler.getFileAttributes(absolutePath.toString(), MINIMALLY_REQUIRED_ATTRIBUTES); } catch (CacheException e) { throw new BulkServiceException("failed to retrieve file attributes", e); } @@ -232,6 +247,3 @@ public void setNamespaceHandler(PnfsHandler pnfsHandler) { this.pnfsHandler = pnfsHandler; } } - - - diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java index 7317af31a8d..c7d0269ff24 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java @@ -120,7 +120,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.util.list.ListDirectoryHandler; import org.dcache.vehicles.FileAttributes; import org.dcache.vehicles.PnfsGetFileAttributes; -import org.dcache.vehicles.PnfsResolveSymlinksMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -140,18 +139,7 @@ public final class BulkRequestContainerJob static final AtomicLong taskCounter = new AtomicLong(0L); public static FsPath findAbsolutePath(String prefix, String path) { - FsPath absPath = computeFsPath(null, path); - if (prefix == null) { - return absPath; - } - - FsPath pref = FsPath.create(prefix); - - if (!absPath.hasPrefix(pref)) { - absPath = computeFsPath(prefix, path); - } - - return absPath; + return computeFsPath(prefix, path); } /** @@ -235,7 +223,7 @@ enum ContainerState { * proper paths and attributes from listing. */ enum TaskState { - RESOLVE_PATH, FETCH_ATTRIBUTES, HANDLE_TARGET, HANDLE_DIR_TARGET + FETCH_ATTRIBUTES, HANDLE_TARGET, HANDLE_DIR_TARGET } /** @@ -452,7 +440,7 @@ void cancel() { } if (target != null) { - activity.cancel(target); + activity.cancel(targetPrefix, target); LOGGER.debug("{} - target cancelled for task {}.", ruid, seqNo); } @@ -464,9 +452,6 @@ void doInner() { try { checkForRequestCancellation(); switch (state) { - case RESOLVE_PATH: - resolvePath(); - break; case FETCH_ATTRIBUTES: fetchAttributes(); break; @@ -512,43 +497,15 @@ void performSync() throws InterruptedException { } /** - * (1) symlink resolution on initial targets; bypassed for discovered targets. + * (1) retrieval of required file attributes. */ - private void resolvePath() { - LOGGER.debug("{} - resolvePath, resolving {}", ruid, target.getPath()); - PnfsResolveSymlinksMessage message = new PnfsResolveSymlinksMessage( - target.getPath().toString(), null); - ListenableFuture requestFuture = pnfsHandler.requestAsync( - message); - CellStub.addCallback(requestFuture, new AbstractMessageCallback<>() { - @Override - public void success(PnfsResolveSymlinksMessage message) { - LOGGER.debug("{} - resolvePath {}, callback success.", ruid, target.getPath()); - FsPath path = FsPath.create(message.getResolvedPath()); - if (targetPrefix != null && !path.contains(targetPrefix)) { - path = computeFsPath(targetPrefix, path.toString()); - } - LOGGER.debug("{} - resolvePath, resolved path {}", ruid, path); - target.setPath(path); - state = TaskState.FETCH_ATTRIBUTES; - taskFuture = executor.submit(TargetTask.this); - } + private void fetchAttributes() { + FsPath absolutePath = findAbsolutePath(targetPrefix, + target.getPath().toString()); + LOGGER.debug("{} - fetchAttributes for path {}, prefix {}, absolute path {} ", ruid, target.getPath(), targetPrefix, absolutePath); - @Override - public void failure(int rc, Object error) { - LOGGER.error("{} - resolvePath, callback failure for {}.", ruid, target); - storeOrUpdate(CacheExceptionFactory.exceptionOf( - rc, Objects.toString(error, null))); - } - }, callbackExecutor); - } - /** - * (2) retrieval of required file attributes. - */ - private void fetchAttributes() { - LOGGER.debug("{} - fetchAttributes for path {}", ruid, target.getPath()); - PnfsGetFileAttributes message = new PnfsGetFileAttributes(target.getPath().toString(), + PnfsGetFileAttributes message = new PnfsGetFileAttributes(absolutePath.toString(), MINIMALLY_REQUIRED_ATTRIBUTES); ListenableFuture requestFuture = pnfsHandler.requestAsync( message); @@ -573,7 +530,7 @@ public void failure(int rc, Object error) { } /** - * (3b) either recurs on directory or performs activity on file. + * (2b) either recurs on directory or performs activity on file. */ private void handleTarget() throws InterruptedException { LOGGER.debug("{} - handleTarget for {}, path {}.", ruid, target.getActivity(), @@ -611,17 +568,20 @@ private void performActivity(boolean async) throws InterruptedException { FsPath path = target.getPath(); FileAttributes attributes = target.getAttributes(); LOGGER.debug("{} - performActivity {} on {}.", ruid, activity, path); - storeOrUpdate(null); if (hasBeenSpecificallyCancelled(this)) { LOGGER.debug("{} - performActivity hasBeenSpecificallyCancelled for {}.", ruid, - path); + path); remove(); } try { - activityFuture = activity.perform(ruid, id == null ? seqNo : id, path, attributes); + activityFuture = activity.perform(ruid, + id == null ? seqNo : id, + targetPrefix, + path, + attributes); if (async) { activityFuture.addListener(() -> handleCompletion(), callbackExecutor); permitHolder.throttledRelease(); @@ -1077,7 +1037,7 @@ private void processFileTargets() { for (BulkRequestTarget target : requestTargets) { try { checkForRequestCancellation(); - new TargetTask(target, TaskState.RESOLVE_PATH).submitAsync(); + new TargetTask(target, TaskState.FETCH_ATTRIBUTES).submitAsync(); } catch (InterruptedException e) { /* * Cancel most likely called; stop processing. @@ -1128,4 +1088,4 @@ private void update() { signalStateChange(); } -} \ No newline at end of file +} diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkRequestTarget.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkRequestTarget.java index de7d3632be0..2bbbd1b56be 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkRequestTarget.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkRequestTarget.java @@ -87,12 +87,15 @@ public final class BulkRequestTarget { RUNNING}; public static FsPath computeFsPath(String prefix, String target) { - if (prefix == null) { - return FsPath.create(FsPath.ROOT + target); - } else { - return FsPath.create( - FsPath.ROOT + (prefix.endsWith("/") ? prefix : prefix + "/") + target); + FsPath absolutePath = FsPath.create(FsPath.ROOT + target); + if (prefix != null) { + FsPath pref = FsPath.create(prefix); + if (!absolutePath.hasPrefix(pref)) { + absolutePath = FsPath.create( + FsPath.ROOT + (prefix.endsWith("/") ? prefix : prefix + "/") + target); + } } + return absolutePath; } public static final FsPath ROOT_REQUEST_PATH = computeFsPath(null, "=request_target="); diff --git a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/bulk/BulkResources.java b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/bulk/BulkResources.java index 6a0bf16da2d..40f3b5e0a01 100644 --- a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/bulk/BulkResources.java +++ b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/bulk/BulkResources.java @@ -59,6 +59,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING */ package org.dcache.restful.resources.bulk; +import static org.dcache.http.AuthenticationHandler.getLoginAttributes; import static org.dcache.restful.util.HttpServletRequests.getUserRootAwareTargetPrefix; import static org.dcache.restful.util.JSONUtils.newBadRequestException; @@ -67,6 +68,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.JsonParseException; +import diskCacheV111.util.FsPath; import diskCacheV111.util.PnfsHandler; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -93,6 +95,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.ForbiddenException; import javax.ws.rs.NotAuthorizedException; import javax.ws.rs.PATCH; import javax.ws.rs.POST; @@ -103,9 +106,11 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.dcache.auth.attributes.LoginAttributes; import org.dcache.auth.attributes.Restriction; import org.dcache.auth.attributes.Restrictions; import org.dcache.cells.CellStub; +import org.dcache.http.PathMapper; import org.dcache.restful.util.HandlerBuilders; import org.dcache.restful.util.RequestUser; import org.dcache.restful.util.bulk.BulkServiceCommunicator; @@ -144,6 +149,9 @@ public final class BulkResources { @Inject private BulkServiceCommunicator service; + @Inject + private PathMapper pathMapper; + @Inject @Named("pnfs-stub") private CellStub pnfsmanager; @@ -235,11 +243,14 @@ public Response submit( Subject subject = getSubject(); Restriction restriction = getRestriction(); PnfsHandler handler = HandlerBuilders.unrestrictedPnfsHandler(pnfsmanager); - BulkRequest request = toBulkRequest(requestPayload, this.request, handler); + FsPath userRoot = LoginAttributes.getUserRoot(getLoginAttributes(request)); + FsPath rootPath = pathMapper.effectiveRoot(userRoot, ForbiddenException::new); + BulkRequest request = toBulkRequest(requestPayload, this.request, handler, rootPath); /* * Frontend sets the URL. The backend service provides the UUID. */ + request.setUrlPrefix(this.request.getRequestURL().toString()); BulkRequestMessage message = new BulkRequestMessage(request, restriction); @@ -498,7 +509,7 @@ public static Restriction getRestriction() { * they are defined in the Bulk service as well. */ @VisibleForTesting - static BulkRequest toBulkRequest(String requestPayload, HttpServletRequest httpServletRequest, PnfsHandler handler) { + static BulkRequest toBulkRequest(String requestPayload, HttpServletRequest httpServletRequest, PnfsHandler handler, FsPath rootPath) { if (Strings.emptyToNull(requestPayload) == null) { throw new BadRequestException("empty request payload."); } @@ -531,10 +542,13 @@ static BulkRequest toBulkRequest(String requestPayload, HttpServletRequest httpS string = removeEntry(map, String.class, "target_prefix", "target-prefix", "targetPrefix"); + if (httpServletRequest != null) { - request.setTargetPrefix(getUserRootAwareTargetPrefix(httpServletRequest, string, handler)); + request.setTargetPrefix(getUserRootAwareTargetPrefix(httpServletRequest, + string != null ? string : rootPath.toString(), + handler)); } else { - request.setTargetPrefix(string); + request.setTargetPrefix(string != null ? string : rootPath.toString()); } string = removeEntry(map, String.class, "expand_directories", "expand-directories", diff --git a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ArchiveInfoResources.java b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ArchiveInfoResources.java index 718c871a58b..314f9ece1ff 100644 --- a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ArchiveInfoResources.java +++ b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ArchiveInfoResources.java @@ -160,19 +160,17 @@ public List getArchiveInfo( paths = new ArrayList<>(); for (int i = 0; i < len; ++i) { String requestedPath = jsonArray.getString(i); - String dcachePath = rootPath.chroot(requestedPath).toString(); - paths.add(dcachePath); + paths.add(requestedPath); } } catch (JSONException e) { throw newBadRequestException(requestPayload, e); } var archiveInfos = archiveInfoCollector.getInfo(HandlerBuilders.roleAwarePnfsHandler(pnfsManager), - paths); - - archiveInfos.forEach(ai -> - ai.setPath(FsPath.create(ai.getPath()).stripPrefix(rootPath))); - + rootPath.toString(), + paths); return archiveInfos; } + + } diff --git a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ReleaseResources.java b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ReleaseResources.java index 57750c66a01..44bd7a816ae 100644 --- a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ReleaseResources.java +++ b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ReleaseResources.java @@ -173,8 +173,7 @@ public Response release( int len = paths.length(); targetPaths = new ArrayList<>(); for (int i = 0; i < len; ++i) { - String requestPath = paths.getString(i); - String path = rootPath.chroot(requestPath).toString(); + String path = paths.getString(i); targetPaths.add(path); } } catch (JSONException e) { diff --git a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/StageResources.java b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/StageResources.java index 72f39bb11a1..8ac882b6144 100644 --- a/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/StageResources.java +++ b/modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/StageResources.java @@ -127,6 +127,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING @Api(value = "tape", authorizations = {@Authorization("basicAuth")}) @Path("tape/stage") public final class StageResources { + private static final String STAGE = "STAGE"; @Context @@ -189,7 +190,6 @@ public StageRequestInfo getStageInfo(@ApiParam("The unique id of the request.") offset = lastInfo.getNextId(); } - targetInfos.forEach(ti -> ti.setTarget(FsPath.create(ti.getTarget()).stripPrefix(rootPath))); lastInfo.setTargets(targetInfos); return new StageRequestInfo(lastInfo); @@ -238,8 +238,7 @@ public Response cancel( List targetPaths = new ArrayList<>(); int len = paths.length(); for (int i = 0; i < len; ++i) { - String requestPath = paths.getString(i); - String path = rootPath.chroot(requestPath).toString(); + String path = paths.getString(i); targetPaths.add(path); } @@ -397,8 +396,7 @@ private BulkRequest toBulkRequest(String requestPayload, FsPath rootPath) { if (!file.has("path")) { throw new BadRequestException("file object " + i + " has no path."); } - String requestPath = file.getString("path"); - String path = rootPath.chroot(requestPath).toString(); + String path = file.getString("path"); paths.add(path); if (file.has("diskLifetime")) { jsonLifetimes.put(path, diff --git a/modules/dcache-frontend/src/main/java/org/dcache/restful/util/wlcg/ArchiveInfoCollector.java b/modules/dcache-frontend/src/main/java/org/dcache/restful/util/wlcg/ArchiveInfoCollector.java index 67ab5298c50..3298156d8f9 100644 --- a/modules/dcache-frontend/src/main/java/org/dcache/restful/util/wlcg/ArchiveInfoCollector.java +++ b/modules/dcache-frontend/src/main/java/org/dcache/restful/util/wlcg/ArchiveInfoCollector.java @@ -64,6 +64,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import com.google.common.base.Throwables; import diskCacheV111.util.CacheException; import diskCacheV111.util.FileLocality; +import diskCacheV111.util.FsPath; import diskCacheV111.util.PnfsHandler; import dmg.cells.nucleus.CellCommandListener; import dmg.util.command.Argument; @@ -117,12 +118,12 @@ public String call() throws Exception { private ExecutorService service; private int maxPaths; - public List getInfo(PnfsHandler pnfsHandler, List paths) { + public List getInfo(PnfsHandler pnfsHandler, String prefix, List paths) { Map> futures = new HashMap<>(); List infoList = new ArrayList<>(); for (String path : paths) { - futures.put(path, service.submit(() -> getInfo(path, pnfsHandler))); + futures.put(path, service.submit(() -> getInfo(path, prefix, pnfsHandler))); } for (Entry> future : futures.entrySet()) { @@ -162,9 +163,23 @@ public void setService(ExecutorService service) { this.service = service; } - private FileLocality getInfo(String path, PnfsHandler pnfsHandler) throws CacheException { - FileAttributes attributes = pnfsHandler.getFileAttributes(path, REQUIRED_ATTRIBUTES, + private FileLocality getInfo(String path, String prefix, PnfsHandler pnfsHandler) throws CacheException { + String absolutePath = computeFsPath(prefix, path).toString(); + FileAttributes attributes = pnfsHandler.getFileAttributes(absolutePath, REQUIRED_ATTRIBUTES, ACCESS_MASK, false); return poolMonitor.getFileLocality(attributes, "localhost"); } + + public static FsPath computeFsPath(String prefix, String target) { + FsPath absolutePath = FsPath.create(FsPath.ROOT + target); + if (prefix != null) { + FsPath pref = FsPath.create(prefix); + if (!absolutePath.hasPrefix(pref)) { + absolutePath = FsPath.create( + FsPath.ROOT + (prefix.endsWith("/") ? prefix : prefix + "/") + target); + } + } + return absolutePath; + } + } diff --git a/modules/dcache-frontend/src/test/java/org/dcache/restful/resources/bulk/BulkRequestJsonParseTest.java b/modules/dcache-frontend/src/test/java/org/dcache/restful/resources/bulk/BulkRequestJsonParseTest.java index 33e78a20f3f..fb8d8af9ea7 100644 --- a/modules/dcache-frontend/src/test/java/org/dcache/restful/resources/bulk/BulkRequestJsonParseTest.java +++ b/modules/dcache-frontend/src/test/java/org/dcache/restful/resources/bulk/BulkRequestJsonParseTest.java @@ -59,6 +59,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING */ package org.dcache.restful.resources.bulk; +import static diskCacheV111.util.FsPath.create; import static org.dcache.restful.resources.bulk.BulkResources.toBulkRequest; import static org.junit.Assert.assertEquals; @@ -182,6 +183,6 @@ private void givenJsonWithArrayTargetAttribute() { } private void whenParsed() { - bulkRequest = toBulkRequest(requestJson, null, handler); + bulkRequest = toBulkRequest(requestJson, null, handler, create("/")); } }