From d88757c0e38f1d61969116c43b4456e65fb837be Mon Sep 17 00:00:00 2001 From: arpit-at Date: Mon, 12 Aug 2024 22:42:18 +0530 Subject: [PATCH 1/5] Adding new API for shifting --- .../bootstrap_relationship_policies.json | 2 +- .../apache/atlas/repository/Constants.java | 2 + .../instance/MoveBusinessPolicyRequest.java | 90 +++++++++++++++++++ .../store/graph/AtlasEntityStore.java | 1 + .../store/graph/v2/AtlasEntityStoreV2.java | 28 ++++++ .../store/graph/v2/EntityGraphMapper.java | 47 ++++++++-- .../atlas/web/rest/BusinessPolicyREST.java | 42 +++++++++ 7 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/MoveBusinessPolicyRequest.java diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json index 2c123bec6b..8c07922f66 100644 --- a/addons/policies/bootstrap_relationship_policies.json +++ b/addons/policies/bootstrap_relationship_policies.json @@ -604,7 +604,7 @@ ] } }, - { + {xf "typeName": "AuthPolicy", "attributes": { diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 9d3c7a3fde..ab19d31562 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -370,6 +370,8 @@ public final class Constants { public static final String STORM_SOURCE = "storm"; public static final String FILE_SPOOL_SOURCE = "file_spool"; public static final String ASSET_POLICY_GUIDS = "assetPolicyGUIDs"; + + public static final String NON_COMPLIANT_ASSET_POLICY_GUIDS = "nonComplaintAssetPolicyGUIDs"; public static final String ASSET_POLICIES_COUNT = "assetPoliciesCount"; diff --git a/intg/src/main/java/org/apache/atlas/model/instance/MoveBusinessPolicyRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/MoveBusinessPolicyRequest.java new file mode 100644 index 0000000000..6fc640030c --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/MoveBusinessPolicyRequest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Request to link/unlink policies from asset. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class MoveBusinessPolicyRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private Set policyIds; + private String type; + + public Set getPolicyIds() { + return policyIds; + } + + public void setPolicyIds(Set policyIds) { + this.policyIds = policyIds; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("MoveBusinessPolicyRequest{"); + sb.append("policyIds=").append(policyIds); + sb.append(", type='").append(type).append('\''); + sb.append('}'); + return sb.toString(); + } + + public enum Type { + GOVERNED ("governed"), NON_COMPLIANT ("non_compliant"); + private final String description; + Type(String description) { + this.description = description; + } + + // Getter method to retrieve the description + public String getDescription() { + return description; + } + } + + + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 24ca97d731..9f1a3f7db1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -371,4 +371,5 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void unlinkBusinessPolicy(String policyId, Set unlinkGuids) throws AtlasBaseException; + void moveBusinessPolicy(Set policyId, String assetId, String type) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index c0bb42298c..180efa3af2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2822,6 +2822,34 @@ private void handleBusinessPolicyMutation(List vertices) throws Atl } + @Override + @GraphTransaction + public void moveBusinessPolicy(Set policyIds, String assetId, String type) throws AtlasBaseException { + // Start performance metric recording + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("moveBusinessPolicy.GraphTransaction"); + + try { + // Attempt to move the business policy using the entityGraphMapper + AtlasVertex vertex = entityGraphMapper.moveBusinessPolicy(policyIds, assetId, type); + + if (vertex == null) { + LOG.warn("No vertex found for assetId: {}", assetId); + return; + } + + handleBusinessPolicyMutation(Collections.singletonList(vertex)); + } catch (Exception e) { + // Log the error with context and rethrow it wrapped in an AtlasBaseException + LOG.error("Error during moveBusinessPolicy for assetId: {}", assetId, e); + throw new AtlasBaseException(e); + } finally { + // End the performance metric recording + RequestContext.get().endMetricRecord(metric); + } + } + + + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 865d58a2b2..7a3b33af74 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -28,15 +28,7 @@ import org.apache.atlas.exception.EntityNotFoundException; import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.TypeCategory; -import org.apache.atlas.model.instance.AtlasClassification; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; -import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.model.instance.*; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.model.typedef.AtlasEntityDef; @@ -4654,6 +4646,40 @@ public List unlinkBusinessPolicy(String policyId, Set unlin }).collect(Collectors.toList()); } + public AtlasVertex moveBusinessPolicy(Set policyIds, String assetId, String type) { + // Retrieve the AtlasVertex for the given assetId + AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(graph, assetId); + + // Get the sets of governed and non-compliant policy GUIDs + Set governedPolicies = assetVertex.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); + Set nonCompliantPolicies = assetVertex.getMultiValuedSetProperty(NON_COMPLIANT_ASSET_POLICY_GUIDS, String.class); + + // Move the policy between governed and non-compliant sets based on the type + if (MoveBusinessPolicyRequest.Type.GOVERNED.getDescription().equals(type)) { + nonCompliantPolicies.removeAll(policyIds); + governedPolicies.addAll(policyIds); + } else if (MoveBusinessPolicyRequest.Type.NON_COMPLIANT.getDescription().equals(type)) { + governedPolicies.removeAll(policyIds); + nonCompliantPolicies.addAll(policyIds); + } + + // Update the modification metadata + updateModificationMetadata(assetVertex); + + // Create a differential AtlasEntity to reflect the changes + AtlasEntity diffEntity = new AtlasEntity(assetVertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); + diffEntity.setGuid(assetVertex.getProperty(GUID_PROPERTY_KEY, String.class)); + diffEntity.setAttribute(ASSET_POLICY_GUIDS, governedPolicies); + diffEntity.setAttribute(NON_COMPLIANT_ASSET_POLICY_GUIDS, nonCompliantPolicies); + diffEntity.setUpdatedBy(assetVertex.getProperty(MODIFIED_BY_KEY, String.class)); + diffEntity.setUpdateTime(new Date(RequestContext.get().getRequestTime())); + + // Cache the differential entity for further processing + RequestContext.get().cacheDifferentialEntity(diffEntity); + + return assetVertex; + } + private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues) { AtlasEntity diffEntity = new AtlasEntity(ev.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); @@ -4667,4 +4693,7 @@ private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues) requestContext.cacheDifferentialEntity(diffEntity); } + + + } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index ac0a52f9f5..990fd0c25d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -5,6 +5,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.LinkBusinessPolicyRequest; +import org.apache.atlas.model.instance.MoveBusinessPolicyRequest; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; @@ -112,4 +113,45 @@ public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, RequestContext.get().endMetricRecord(metric); } } + + + @POST + @Path("/move/{assetId}") + @Timed + public void moveBusinessPolicy( + @PathParam("assetId") String assetId, + final MoveBusinessPolicyRequest request + ) throws AtlasBaseException { + // Start performance metric recording + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("moveBusinessPolicy"); + + // Ensure the current user is authorized to move the policy + String currentUser = RequestContext.getCurrentUser(); + if (!ARGO_SERVICE_USER_NAME.equals(currentUser)) { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, currentUser, "moveBusinessPolicy"); + } + + // Set request context parameters for the current operation + RequestContext requestContext = RequestContext.get(); + requestContext.setIncludeClassifications(false); + requestContext.setIncludeMeanings(false); + requestContext.getRequestContextHeaders().put("x-atlan-route", "business-policy-rest"); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "BusinessPolicyREST.moveBusinessPolicy(" + assetId + ")"); + } + + // Move the business policy using the entitiesStore + entitiesStore.moveBusinessPolicy(request.getPolicyIds(), assetId, request.getType()); + } finally { + // Log performance trace and end metric recording + AtlasPerfTracer.log(perf); + requestContext.endMetricRecord(metric); + } + } + } + From 3615a2d125f19365d8f6affc5c75707896db5c88 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Mon, 12 Aug 2024 22:42:53 +0530 Subject: [PATCH 2/5] Adding new API for shifting --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..189de0b9e2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - policyendpointsmasterss jobs: build: From c714b54ba2139ad341ba60a0e645b22ab9e7a7ba Mon Sep 17 00:00:00 2001 From: arpit-at Date: Tue, 13 Aug 2024 14:22:07 +0530 Subject: [PATCH 3/5] Adding new API for shifting --- .../store/graph/v2/EntityGraphMapper.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 7a3b33af74..fd47b8b846 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -4654,13 +4654,26 @@ public AtlasVertex moveBusinessPolicy(Set policyIds, String assetId, Str Set governedPolicies = assetVertex.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); Set nonCompliantPolicies = assetVertex.getMultiValuedSetProperty(NON_COMPLIANT_ASSET_POLICY_GUIDS, String.class); - // Move the policy between governed and non-compliant sets based on the type - if (MoveBusinessPolicyRequest.Type.GOVERNED.getDescription().equals(type)) { - nonCompliantPolicies.removeAll(policyIds); + // Determine if the type is governed or non-compliant and move policies accordingly + boolean isGoverned = MoveBusinessPolicyRequest.Type.GOVERNED.getDescription().equals(type); + + policyIds.forEach(policyId -> { + if (isGoverned) { + AtlasGraphUtilsV2.addItemToListProperty(assetVertex, ASSET_POLICY_GUIDS, policyId); + AtlasGraphUtilsV2.removeItemFromListPropertyValue(assetVertex, NON_COMPLIANT_ASSET_POLICY_GUIDS, policyId); + } else { + AtlasGraphUtilsV2.addItemToListProperty(assetVertex, NON_COMPLIANT_ASSET_POLICY_GUIDS, policyId); + AtlasGraphUtilsV2.removeItemFromListPropertyValue(assetVertex, ASSET_POLICY_GUIDS, policyId); + } + }); + + // Update the sets after processing + if (isGoverned) { governedPolicies.addAll(policyIds); - } else if (MoveBusinessPolicyRequest.Type.NON_COMPLIANT.getDescription().equals(type)) { - governedPolicies.removeAll(policyIds); + nonCompliantPolicies.removeAll(policyIds); + } else { nonCompliantPolicies.addAll(policyIds); + governedPolicies.removeAll(policyIds); } // Update the modification metadata @@ -4681,6 +4694,8 @@ public AtlasVertex moveBusinessPolicy(Set policyIds, String assetId, Str } + + private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues) { AtlasEntity diffEntity = new AtlasEntity(ev.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); diffEntity.setGuid(ev.getProperty(GUID_PROPERTY_KEY, String.class)); From b73dfaaf7027633d44ca8b6087834101f7927d42 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Tue, 13 Aug 2024 14:49:24 +0530 Subject: [PATCH 4/5] Adding new API for shifting --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index fd47b8b846..16d080c286 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -4659,10 +4659,10 @@ public AtlasVertex moveBusinessPolicy(Set policyIds, String assetId, Str policyIds.forEach(policyId -> { if (isGoverned) { - AtlasGraphUtilsV2.addItemToListProperty(assetVertex, ASSET_POLICY_GUIDS, policyId); + assetVertex.setProperty(ASSET_POLICY_GUIDS, policyId); AtlasGraphUtilsV2.removeItemFromListPropertyValue(assetVertex, NON_COMPLIANT_ASSET_POLICY_GUIDS, policyId); } else { - AtlasGraphUtilsV2.addItemToListProperty(assetVertex, NON_COMPLIANT_ASSET_POLICY_GUIDS, policyId); + assetVertex.setProperty(NON_COMPLIANT_ASSET_POLICY_GUIDS, policyId); AtlasGraphUtilsV2.removeItemFromListPropertyValue(assetVertex, ASSET_POLICY_GUIDS, policyId); } }); From aa6d17132dd0deac256ab5e6819c468afa581f0c Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 11 Sep 2024 13:41:22 +0530 Subject: [PATCH 5/5] DG-1697 API endpoint to move the policy --- .github/workflows/maven.yml | 1 - addons/policies/bootstrap_relationship_policies.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 189de0b9e2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - policyendpointsmasterss jobs: build: diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json index 8c07922f66..2c123bec6b 100644 --- a/addons/policies/bootstrap_relationship_policies.json +++ b/addons/policies/bootstrap_relationship_policies.json @@ -604,7 +604,7 @@ ] } }, - {xf + { "typeName": "AuthPolicy", "attributes": {