From a4d5914162c202a471091652626835930236befe Mon Sep 17 00:00:00 2001 From: David An Date: Wed, 29 Nov 2023 13:50:01 -0500 Subject: [PATCH 1/3] entity APIs throw error for Azure workspaces --- .../broadinstitute/dsde/rawls/entities/EntityManager.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala index d1ab218de3..07ff1ada5e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala @@ -1,5 +1,6 @@ package org.broadinstitute.dsde.rawls.entities +import bio.terra.workspace.model.CloudPlatform import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.config.DataRepoEntityProviderConfig import org.broadinstitute.dsde.rawls.dataaccess.datarepo.DataRepoDAO @@ -9,7 +10,7 @@ import org.broadinstitute.dsde.rawls.entities.base.{EntityProvider, EntityProvid import org.broadinstitute.dsde.rawls.entities.datarepo.{DataRepoEntityProvider, DataRepoEntityProviderBuilder} import org.broadinstitute.dsde.rawls.entities.exceptions.DataEntityException import org.broadinstitute.dsde.rawls.entities.local.{LocalEntityProvider, LocalEntityProviderBuilder} -import org.broadinstitute.dsde.rawls.model.ErrorReport +import org.broadinstitute.dsde.rawls.model.{ErrorReport, WorkspaceType} import scala.concurrent.{ExecutionContext, Future} import scala.reflect.runtime.universe._ @@ -42,6 +43,10 @@ class EntityManager(providerBuilders: Set[EntityProviderBuilder[_ <: EntityProvi def resolveProvider(requestArguments: EntityRequestArguments): Try[EntityProvider] = { + if (!WorkspaceType.RawlsWorkspace.equals(requestArguments.workspace.workspaceType)) { + throw new DataEntityException(s"This functionality only available to ${CloudPlatform.GCP} workspaces.") + } + // soon: look up the reference name to ensure it exists. // for now, this simplistic logic illustrates the approach: choose the right builder for the job. val targetTag = if (requestArguments.dataReference.isDefined) { From 2fe90bda7ea1f8a32b37b8edfb7797ea9dd0749a Mon Sep 17 00:00:00 2001 From: David An Date: Wed, 29 Nov 2023 15:51:44 -0500 Subject: [PATCH 2/3] copyEntities not available for Azure either --- .../dsde/rawls/entities/EntityService.scala | 63 ++++++++----------- .../rawls/entities/base/EntityProvider.scala | 13 +++- .../datarepo/DataRepoEntityProvider.scala | 14 ++++- .../entities/local/LocalEntityProvider.scala | 25 +++++++- .../dsde/rawls/util/WorkspaceSupport.scala | 6 +- .../rawls/webservice/EntityApiService.scala | 2 +- 6 files changed, 78 insertions(+), 45 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityService.scala index d4c5a5a15b..5ae7a15a2d 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityService.scala @@ -540,44 +540,31 @@ class EntityService(protected val ctx: RawlsRequestContext, } } - def copyEntities(entityCopyDef: EntityCopyDefinition, - uri: Uri, - linkExistingEntities: Boolean - ): Future[EntityCopyResponse] = - getV2WorkspaceContextAndPermissions(entityCopyDef.destinationWorkspace, - SamWorkspaceActions.write, - Some(WorkspaceAttributeSpecs(all = false)) - ) flatMap { destWorkspaceContext => - getV2WorkspaceContextAndPermissions(entityCopyDef.sourceWorkspace, - SamWorkspaceActions.read, - Some(WorkspaceAttributeSpecs(all = false)) - ) flatMap { sourceWorkspaceContext => - dataSource.inTransaction { dataAccess => - for { - sourceAD <- DBIO.from( - samDAO.getResourceAuthDomain(SamResourceTypeNames.workspace, sourceWorkspaceContext.workspaceId, ctx) - ) - destAD <- DBIO.from( - samDAO.getResourceAuthDomain(SamResourceTypeNames.workspace, destWorkspaceContext.workspaceId, ctx) - ) - result <- authDomainCheck(sourceAD.toSet, destAD.toSet) flatMap { _ => - val entityNames = entityCopyDef.entityNames - val entityType = entityCopyDef.entityType - val copyResults = traceDBIOWithParent("checkAndCopyEntities", ctx)(s1 => - dataAccess.entityQuery.checkAndCopyEntities(sourceWorkspaceContext, - destWorkspaceContext, - entityType, - entityNames, - linkExistingEntities, - s1 - ) - ) - copyResults - } - } yield result - } - } - } + def copyEntities(entityCopyDef: EntityCopyDefinition, linkExistingEntities: Boolean): Future[EntityCopyResponse] = + for { + destWsCtx <- getV2WorkspaceContextAndPermissions(entityCopyDef.destinationWorkspace, + SamWorkspaceActions.write, + Some(WorkspaceAttributeSpecs(all = false)) + ) + sourceWsCtx <- getV2WorkspaceContextAndPermissions(entityCopyDef.sourceWorkspace, + SamWorkspaceActions.read, + Some(WorkspaceAttributeSpecs(all = false)) + ) + sourceAD <- samDAO.getResourceAuthDomain(SamResourceTypeNames.workspace, sourceWsCtx.workspaceId, ctx) + destAD <- samDAO.getResourceAuthDomain(SamResourceTypeNames.workspace, destWsCtx.workspaceId, ctx) + _ = authDomainCheck(sourceAD.toSet, destAD.toSet) + entityRequestArguments = EntityRequestArguments(destWsCtx, ctx, None, None) + entityProvider <- entityManager.resolveProviderFuture(entityRequestArguments) + entityCopyResponse <- entityProvider + .copyEntities(sourceWsCtx, + destWsCtx, + entityCopyDef.entityType, + entityCopyDef.entityNames, + linkExistingEntities, + ctx + ) + .recover(bigQueryRecover) + } yield entityCopyResponse def batchUpdateEntitiesInternal(workspaceName: WorkspaceName, entityUpdates: Seq[EntityUpdateDefinition], diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/base/EntityProvider.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/base/EntityProvider.scala index 1d7364ea76..dda9c0879f 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/base/EntityProvider.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/base/EntityProvider.scala @@ -7,11 +7,14 @@ import org.broadinstitute.dsde.rawls.model.{ AttributeEntityReference, AttributeValue, Entity, + EntityCopyDefinition, + EntityCopyResponse, EntityQuery, EntityQueryResponse, EntityTypeMetadata, RawlsRequestContext, - SubmissionValidationEntityInputs + SubmissionValidationEntityInputs, + Workspace } import scala.concurrent.Future @@ -70,4 +73,12 @@ trait EntityProvider { def batchUpdateEntities(entityUpdates: Seq[EntityUpdateDefinition]): Future[Traversable[Entity]] def batchUpsertEntities(entityUpdates: Seq[EntityUpdateDefinition]): Future[Traversable[Entity]] + + def copyEntities(sourceWorkspaceContext: Workspace, + destWorkspaceContext: Workspace, + entityType: String, + entityNames: Seq[String], + linkExistingEntities: Boolean, + parentContext: RawlsRequestContext + ): Future[EntityCopyResponse] } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala index 4c8d74e9c1..a8f872332e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala @@ -38,13 +38,16 @@ import org.broadinstitute.dsde.rawls.model.{ AttributeValueList, AttributeValueRawJson, Entity, + EntityCopyDefinition, + EntityCopyResponse, EntityQuery, EntityQueryResponse, EntityTypeMetadata, ErrorReport, GoogleProjectId, RawlsRequestContext, - SubmissionValidationEntityInputs + SubmissionValidationEntityInputs, + Workspace } import org.broadinstitute.dsde.rawls.util.CollectionUtils import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} @@ -524,4 +527,13 @@ class DataRepoEntityProvider(snapshotModel: SnapshotModel, override def batchUpsertEntities(entityUpdates: Seq[EntityUpdateDefinition]): Future[Traversable[Entity]] = throw new UnsupportedEntityOperationException("batch-upsert entities not supported by this provider.") + + override def copyEntities(sourceWorkspaceContext: Workspace, + destWorkspaceContext: Workspace, + entityType: String, + entityNames: Seq[String], + linkExistingEntities: Boolean, + parentContext: RawlsRequestContext + ): Future[EntityCopyResponse] = + throw new UnsupportedEntityOperationException("copy entities not supported by this provider.") } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/local/LocalEntityProvider.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/local/LocalEntityProvider.scala index 476a097973..d05ba0d24c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/local/LocalEntityProvider.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/local/LocalEntityProvider.scala @@ -26,15 +26,20 @@ import org.broadinstitute.dsde.rawls.model.{ AttributeEntityReference, AttributeValue, Entity, + EntityCopyDefinition, + EntityCopyResponse, EntityQuery, EntityQueryResponse, EntityQueryResultMetadata, EntityTypeMetadata, ErrorReport, RawlsRequestContext, + SamResourceTypeNames, + SamWorkspaceActions, SubmissionValidationEntityInputs, SubmissionValidationValue, - Workspace + Workspace, + WorkspaceAttributeSpecs } import org.broadinstitute.dsde.rawls.util.TracingUtils._ import org.broadinstitute.dsde.rawls.util.{AttributeSupport, CollectionUtils, EntitySupport} @@ -386,4 +391,22 @@ class LocalEntityProvider(requestArguments: EntityRequestArguments, override def batchUpsertEntities(entityUpdates: Seq[EntityUpdateDefinition]): Future[Traversable[Entity]] = batchUpdateEntitiesImpl(entityUpdates, upsert = true) + override def copyEntities(sourceWorkspaceContext: Workspace, + destWorkspaceContext: Workspace, + entityType: String, + entityNames: Seq[String], + linkExistingEntities: Boolean, + parentContext: RawlsRequestContext + ): Future[EntityCopyResponse] = + traceWithParent("checkAndCopyEntities", parentContext)(_ => + dataSource.inTransaction { dataAccess => + dataAccess.entityQuery.checkAndCopyEntities(sourceWorkspaceContext, + destWorkspaceContext, + entityType, + entityNames, + linkExistingEntities, + parentContext + ) + } + ) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala index 3ec8f61380..db62ca8fb6 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala @@ -132,14 +132,14 @@ trait WorkspaceSupport { } yield () // can't use withClonedAuthDomain because the Auth Domain -> no Auth Domain logic is different - def authDomainCheck(sourceWorkspaceADs: Set[String], destWorkspaceADs: Set[String]): ReadWriteAction[Boolean] = + def authDomainCheck(sourceWorkspaceADs: Set[String], destWorkspaceADs: Set[String]): Boolean = // if the source has any auth domains, the dest must also *at least* have those auth domains - if (sourceWorkspaceADs.subsetOf(destWorkspaceADs)) DBIO.successful(true) + if (sourceWorkspaceADs.subsetOf(destWorkspaceADs)) true else { val missingGroups = sourceWorkspaceADs -- destWorkspaceADs val errorMsg = s"Source workspace has an Authorization Domain containing the groups ${missingGroups.mkString(", ")}, which are missing on the destination workspace" - DBIO.failed(new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.UnprocessableEntity, errorMsg))) + throw new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.UnprocessableEntity, errorMsg)) } // WorkspaceContext helpers diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/EntityApiService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/EntityApiService.scala index 96b51d6797..dc14e7ca30 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/EntityApiService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/EntityApiService.scala @@ -317,7 +317,7 @@ trait EntityApiService extends UserInfoDirectives { entity(as[EntityCopyDefinition]) { copyDefinition => complete { entityServiceConstructor(ctx) - .copyEntities(copyDefinition, request.uri, linkExistingEntitiesBool) + .copyEntities(copyDefinition, linkExistingEntitiesBool) .map { response => if ( response.hardConflicts.isEmpty && (response.softConflicts.isEmpty || linkExistingEntitiesBool) From 518e06f0476a6d414e4a449ea7e365c8901c9399 Mon Sep 17 00:00:00 2001 From: David An Date: Fri, 8 Dec 2023 10:15:15 -0500 Subject: [PATCH 3/3] Tweak error message; remove stack trace --- .../broadinstitute/dsde/rawls/entities/EntityManager.scala | 4 +++- .../dsde/rawls/webservice/RawlsApiService.scala | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala index 07ff1ada5e..f8b5a54ac0 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/EntityManager.scala @@ -44,7 +44,9 @@ class EntityManager(providerBuilders: Set[EntityProviderBuilder[_ <: EntityProvi def resolveProvider(requestArguments: EntityRequestArguments): Try[EntityProvider] = { if (!WorkspaceType.RawlsWorkspace.equals(requestArguments.workspace.workspaceType)) { - throw new DataEntityException(s"This functionality only available to ${CloudPlatform.GCP} workspaces.") + throw new DataEntityException( + s"This API is disabled for ${CloudPlatform.AZURE} workspaces. Contact support for alternatives." + ) } // soon: look up the reference name to ensure it exists. diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala index c1a3e7fb54..868b1ac20a 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala @@ -20,6 +20,7 @@ import org.broadinstitute.dsde.rawls.billing.BillingProjectOrchestrator import org.broadinstitute.dsde.rawls.bucketMigration.BucketMigrationService import org.broadinstitute.dsde.rawls.dataaccess.{ExecutionServiceCluster, SamDAO} import org.broadinstitute.dsde.rawls.entities.EntityService +import org.broadinstitute.dsde.rawls.entities.exceptions.DataEntityException import org.broadinstitute.dsde.rawls.genomics.GenomicsService import org.broadinstitute.dsde.rawls.metrics.InstrumentationDirectives import org.broadinstitute.dsde.rawls.model.{ApplicationVersion, ErrorReport, RawlsRequestContext, UserInfo} @@ -62,6 +63,9 @@ object RawlsApiService extends LazyLogging { Sentry.captureException(wsmApiException) } complete(wsmApiException.getCode -> ErrorReport(wsmApiException).copy(stackTrace = Seq())) + case dataEntityException: DataEntityException => + // propagate only the message; don't include the stack trace + complete(dataEntityException.code -> ErrorReport(dataEntityException.getMessage)) case e: Throwable => // so we don't log the error twice when debug is enabled if (logger.underlying.isDebugEnabled) {