diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..c2e9e24610 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - ns/DG-1716-auth-benchmarking jobs: build: @@ -48,7 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: | + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" id: get_branch - name: Create Maven Settings @@ -58,12 +61,12 @@ jobs: [{ "id": "github", "username": "atlan-ci", - "password": "${{ secrets.my_pat }}" + "password": "${{ secrets.org_pat_github }}" }] - name: Build with Maven run: | - branch_name=${{ steps.get_branch.outputs.branch }} + branch_name='${{ steps.get_branch.outputs.branch }}' if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] then echo "build without dashboard" @@ -77,7 +80,7 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.org_pat_github }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx @@ -89,7 +92,7 @@ jobs: with: registry: ghcr.io username: $GITHUB_ACTOR - password: ${{ secrets.my_pat }} + password: ${{ secrets.org_pat_github }} - name: Build and push id: docker_build @@ -102,8 +105,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index b224cccc7e..3ce076b910 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -52,6 +52,7 @@ import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil; import org.apache.atlas.plugin.util.*; +import org.openjdk.jol.info.GraphLayout; import java.util.ArrayList; import java.util.Arrays; @@ -379,6 +380,7 @@ public void setPolicies(ServicePolicies policies) { RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + LOG.info("AuthPerformance: using old policies to create engine" + CollectionUtils.isNotEmpty(policies.getPolicyDeltas())); } if (newPolicyEngine != null) { @@ -403,6 +405,7 @@ public void setPolicies(ServicePolicies policies) { } if (newPolicyEngine != null) { + LOG.info("AuthPerformance: PolicyEngine object size" + GraphLayout.parseInstance(newPolicyEngine).toFootprint()); if (!isPolicyEngineShared) { newPolicyEngine.setUseForwardedIPAddress(pluginConfig.isUseForwardedIPAddress()); newPolicyEngine.setTrustedProxyAddresses(pluginConfig.getTrustedProxyAddresses()); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index aae09a7d26..1cfe0d5963 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -30,6 +30,7 @@ import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig; import org.apache.atlas.plugin.policyengine.RangerPluginContext; import org.apache.atlas.plugin.service.RangerBasePlugin; +import org.openjdk.jol.info.GraphLayout; import java.io.File; import java.io.FileReader; @@ -326,6 +327,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } + LOG.info("AuthPerformance: PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin() svcPolicies object size" + GraphLayout.parseInstance(svcPolicies).toFootprint()); boolean isUpdated = svcPolicies != null; diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index accea8ec88..b525f1f987 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -366,6 +366,7 @@ public final class Constants { public static final String IMPALA_SOURCE = "impala"; public static final String STORM_SOURCE = "storm"; public static final String FILE_SPOOL_SOURCE = "file_spool"; + public static final String ASSET_PRODUCT_GUIDS = "assetProductGUIDs"; /* * All supported file-format extensions for Bulk Imports through file upload diff --git a/intg/src/main/java/org/apache/atlas/model/instance/LinkDataProductRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/LinkDataProductRequest.java new file mode 100644 index 0000000000..d7962dc150 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/LinkDataProductRequest.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Request to link/unlink policies from asset. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class LinkDataProductRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private Set linkGuids; + private Set unlinkGuids; + + public Set getLinkGuids() { + return linkGuids; + } + + public void setLinkGuids(Set linkGuids) { + this.linkGuids = linkGuids; + } + + public Set getUnlinkGuids() { + return unlinkGuids; + } + + public void setUnlinkGuids(Set unlinkGuids) { + this.unlinkGuids = unlinkGuids; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); + sb.append("linkGuids=").append(linkGuids); + sb.append(", unlinkGuids=").append(unlinkGuids); + sb.append('}'); + return sb.toString(); + } +} diff --git a/pom.xml b/pom.xml index 1cc9aa70dc..8168839fce 100644 --- a/pom.xml +++ b/pom.xml @@ -1718,6 +1718,12 @@ 2.2.2 + + org.openjdk.jol + jol-core + 0.17 + + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 912799cdd6..04a7cd2ba2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -365,4 +365,10 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairAccesscontrolAlias(String guid) throws AtlasBaseException; + void linkProductToAsset(String productId, Set linkGuids) throws AtlasBaseException; + + void unlinkProductFromAsset(String productId, Set unlinkGuids) throws AtlasBaseException; + + void linkProductWithNotification(String productId, Set unlinkGuids) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 83df0fe2ef..33f185e696 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -150,11 +150,12 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final FeatureFlagStore featureFlagStore; private final ESAliasStore esAliasStore; + private final IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier; @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, - AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore) { + AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore, IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier) { this.graph = graph; this.deleteDelegate = deleteDelegate; this.restoreHandlerV1 = restoreHandlerV1; @@ -168,6 +169,7 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate this.atlasRelationshipStore = atlasRelationshipStore; this.featureFlagStore = featureFlagStore; this.esAliasStore = new ESAliasStore(graph, entityRetriever); + this.atlasAlternateChangeNotifier = atlasAlternateChangeNotifier; try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -2737,6 +2739,73 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { RequestContext.get().endMetricRecord(metric); } + + @Override + @GraphTransaction + public void linkProductToAsset(String productGuid, Set linkGuids) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkProductToAsset.GraphTransaction"); + + try { + List vertices = this.entityGraphMapper.linkProductToAsset(productGuid, linkGuids); + if (CollectionUtils.isEmpty(vertices)) { + return; + } + + LOG.info("linkProductToAsset: productGuid={}, linkGuids={}", productGuid, linkGuids); + + //handleProductMutation(vertices); + } catch (Exception e) { + LOG.error("Error during linkProduct for productGuid: {}", productGuid, e); + throw e; + } finally { + RequestContext.get().endMetricRecord(metric); + } + } + + @Override + @GraphTransaction + public void linkProductWithNotification(String productGuid, Set linkGuids) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkProductToAsset.GraphTransaction"); + + try { + List vertices = this.entityGraphMapper.linkProductWithNotification(productGuid, linkGuids); + if (CollectionUtils.isEmpty(vertices)) { + return; + } + + handleProductMutation(vertices); + } catch (Exception e) { + LOG.error("Error during linkProduct for productGuid: {}", productGuid, e); + throw e; + } finally { + RequestContext.get().endMetricRecord(metric); + } + } + + @Override + @GraphTransaction + public void unlinkProductFromAsset(String productGuid, Set unlinkGuids) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkProductFromAsset.GraphTransaction"); + try { + List vertices = this.entityGraphMapper.unlinkProductFromAsset(productGuid, unlinkGuids); + if (CollectionUtils.isEmpty(vertices)) { + return; + } + + handleProductMutation(vertices); + } catch (Exception e) { + LOG.error("Error during unlinkProduct for productGuid: {}", productGuid, e); + throw e; + } finally { + RequestContext.get().endMetricRecord(metric); + } + } + + private void handleProductMutation(List vertices) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("handleBusinessPolicyMutation"); + this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices); + RequestContext.get().endMetricRecord(metricRecorder); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductNotifierImpl.java new file mode 100644 index 0000000000..4c9115dec8 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductNotifierImpl.java @@ -0,0 +1,64 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.listener.EntityChangeListenerV2; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.GraphHelper.*; + + +@Component +public class DataProductNotifierImpl implements IAtlasMinimalChangeNotifier { + + private final Set entityChangeListenersV2; + + @Inject + public DataProductNotifierImpl(Set entityChangeListenersV2) { + this.entityChangeListenersV2 = entityChangeListenersV2; + + } + + @Override + public void onEntitiesMutation(final List vertices) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); + final List entities = new ArrayList<>(0); + vertices.forEach(item -> entities.add(createAtlasEntity(item))); + for (EntityChangeListenerV2 listener : entityChangeListenersV2) { + listener.onEntitiesUpdated(entities, false); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private AtlasEntity createAtlasEntity(AtlasVertex vertex) { + AtlasEntity atlasEntity = new AtlasEntity(); + atlasEntity.setAttribute(QUALIFIED_NAME, vertex.getProperty(QUALIFIED_NAME, String.class)); + atlasEntity.setAttribute(NAME, vertex.getProperty(NAME, String.class)); + + atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class)); + atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); + atlasEntity.setCreatedBy(vertex.getProperty(CREATED_BY_KEY, String.class)); + atlasEntity.setUpdatedBy(vertex.getProperty(MODIFIED_BY_KEY, String.class)); + atlasEntity.setCreateTime(new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); + atlasEntity.setUpdateTime(new Date(vertex.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class))); + atlasEntity.setIsProxy(vertex.getProperty(IS_PROXY_KEY, Boolean.class)); + atlasEntity.setIsIncomplete(vertex.getProperty(IS_INCOMPLETE_PROPERTY_KEY, Boolean.class)); + atlasEntity.setStatus(getStatus(vertex)); + atlasEntity.setProvenanceType(getProvenanceType(vertex)); + atlasEntity.setHomeId(getHomeId(vertex)); + atlasEntity.setVersion(getVersion(vertex)); + + + return atlasEntity; + } + + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 596420696d..fc46ca0b15 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -4567,4 +4567,62 @@ public void addHasLineage(Set inputOutputEdges, boolean isRestoreEnti RequestContext.get().endMetricRecord(metricRecorder); } + public List linkProductToAsset(String productId, Set linkGuids) { + return linkGuids.stream().map(guid -> findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + return !existingValues.contains(productId); + }).peek(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + existingValues.add(productId); + ev.setProperty(ASSET_PRODUCT_GUIDS, productId); + + updateModificationMetadata(ev); + + //cacheDifferentialEntity(ev, existingValues); + }).collect(Collectors.toList()); + } + + public List linkProductWithNotification(String productId, Set linkGuids) { + return linkGuids.stream().map(guid -> findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + return !existingValues.contains(productId); + }).peek(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + existingValues.add(productId); + ev.setProperty(ASSET_PRODUCT_GUIDS, productId); + + updateModificationMetadata(ev); + + cacheDifferentialEntity(ev, existingValues); + }).collect(Collectors.toList()); + } + + + public List unlinkProductFromAsset(String productId, Set unlinkGuids) { + return unlinkGuids.stream().map(guid -> AtlasGraphUtilsV2.findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + return existingValues.contains(productId); + }).peek(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_PRODUCT_GUIDS, String.class); + existingValues.remove(productId); + ev.removePropertyValue(ASSET_PRODUCT_GUIDS, productId); + + updateModificationMetadata(ev); + + cacheDifferentialEntity(ev, existingValues); + }).collect(Collectors.toList()); + } + + + private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues) { + AtlasEntity diffEntity = new AtlasEntity(ev.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); + diffEntity.setGuid(ev.getProperty(GUID_PROPERTY_KEY, String.class)); + diffEntity.setAttribute(ASSET_PRODUCT_GUIDS, existingValues); + diffEntity.setUpdatedBy(ev.getProperty(MODIFIED_BY_KEY, String.class)); + diffEntity.setUpdateTime(new Date(RequestContext.get().getRequestTime())); + + RequestContext requestContext = RequestContext.get(); + requestContext.cacheDifferentialEntity(diffEntity); + } + } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java new file mode 100644 index 0000000000..35c2d9e757 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.AtlasVertex; + +import java.util.List; + +public interface IAtlasMinimalChangeNotifier { + void onEntitiesMutation(final List vertices) throws AtlasBaseException; +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java index 9edbfc1cc3..84e7948790 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java @@ -129,7 +129,7 @@ private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null); AtlasRelationshipStoreV2 atlasRelationshipStoreV2 = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); - return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null); + return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null, null); } private void shutdownEntityCreationManager(EntityCreationManager creationManager) { diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java index de9fb0718a..86c292a40b 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java @@ -84,7 +84,7 @@ public void setup() throws IOException, AtlasBaseException { loadHiveModel(typeDefStore, typeRegistry); RequestContext.get().setImportInProgress(true); - entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); + entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null, null); createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db", "table-columns", "table-view", "table-table-lineage"}); final String[] entityGuids = {DB_GUID, TABLE_GUID, TABLE_TABLE_GUID, TABLE_VIEW_GUID}; verifyCreatedEntities(entityStore, entityGuids, 4); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java index cda9d5658b..d9c8fa28be 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java @@ -128,7 +128,7 @@ public void setUp() throws Exception { } @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); + entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java index 60973b308b..c6ffef6797 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java @@ -115,7 +115,7 @@ public void clear() throws Exception { @BeforeTest public void init() throws Exception { - entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null); + entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, atlasRelationshipStore, null, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java index 796da31b65..a554a08245 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java @@ -135,7 +135,7 @@ public void setUp() throws Exception { @BeforeTest public void init() throws Exception { relationshipStore = new AtlasRelationshipStoreV2(atlasGraph, typeRegistry, deleteDelegate, entityNotifier); - entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, relationshipStore, null); + entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, restoreHandlerV1, typeRegistry, mockChangeNotifier, graphMapper, null, relationshipStore, null, null); RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..ccf327ac02 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -148,14 +148,15 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s AtlasPerfTracer perf = null; try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); - } if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; } + // only measure the performance when update is required, full request time anyway logged in audits + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); + } ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime); updateLastSync(serviceName); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/ProductAssetLinkREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/ProductAssetLinkREST.java new file mode 100644 index 0000000000..ae90ece72c --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/ProductAssetLinkREST.java @@ -0,0 +1,147 @@ +package org.apache.atlas.web.rest; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.LinkDataProductRequest; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.Servlets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; + +import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; + +@Path("product-asset-link") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class ProductAssetLinkREST { + + private static final Logger LOG = LoggerFactory.getLogger(ProductAssetLinkREST.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.ProductAssetLinkREST"); + + private final AtlasEntityStore entitiesStore; + + @Inject + public ProductAssetLinkREST(AtlasEntityStore entitiesStore) { + this.entitiesStore = entitiesStore; + } + + /** + * Links a product to entities. + * + * @param productGuid the ID of the product to be linked + * @param request the request containing the GUIDs of the assets to link the product to + * @throws AtlasBaseException if there is an error during the linking process + */ + + @POST + @Path("/{productId}/link-product-to-asset") + @Timed + public void linkProductToAsset(@PathParam("productId") final String productGuid, final LinkDataProductRequest request) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkDataProductToAsset"); + // Ensure the current user is authorized to link policies +// if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { +// throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); +// } + + // Set request context parameters + RequestContext.get().setIncludeClassifications(false); + RequestContext.get().setIncludeMeanings(false); + RequestContext.get().getRequestContextHeaders().put("route", "product-asset-link"); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ProductAssetLinkREST.linkProductToAsset(" + productGuid + ")"); + } + + // Link the product to the specified entities + entitiesStore.linkProductToAsset(productGuid, request.getLinkGuids()); + } finally { + // Log performance metrics + AtlasPerfTracer.log(perf); + RequestContext.get().endMetricRecord(metric); + } + } + + @POST + @Path("/{productId}/link-product-with-notification") + @Timed + public void linkProductWithNotification(@PathParam("productId") final String productGuid, final LinkDataProductRequest request) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkDataProductToAsset"); + // Ensure the current user is authorized to link policies +// if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { +// throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); +// } + + // Set request context parameters + RequestContext.get().setIncludeClassifications(false); + RequestContext.get().setIncludeMeanings(false); + RequestContext.get().getRequestContextHeaders().put("route", "product-asset-link"); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ProductAssetLinkREST.linkProductWithNotification(" + productGuid + ")"); + } + + // Link the product to the specified entities + entitiesStore.linkProductWithNotification(productGuid, request.getLinkGuids()); + } finally { + // Log performance metrics + AtlasPerfTracer.log(perf); + RequestContext.get().endMetricRecord(metric); + } + } + + /** + * Unlinks a product from entities. + * + * @param productGuid the ID of the policy to be unlinked + * @param request the request containing the GUIDs of the assets to unlink the policy from + * @throws AtlasBaseException if there is an error during the unlinking process + */ + @POST + @Path("/{policyId}/unlink-product-to-asset") + @Timed + public void unlinkProductFromAsset(@PathParam("policyId") final String productGuid, final LinkDataProductRequest request) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkProductFromAsset"); + // Ensure the current user is authorized to unlink policies +// if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { +// throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); +// } + + // Set request context parameters + RequestContext.get().setIncludeClassifications(false); + RequestContext.get().setIncludeMeanings(false); + RequestContext.get().getRequestContextHeaders().put("route", "product-asset-link"); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ProductAssetLinkREST.unlinkProductFromAsset(" + productGuid + ")"); + } + + // Unlink the business policy from the specified entities + entitiesStore.unlinkProductFromAsset(productGuid, request.getUnlinkGuids()); + } finally { + // Log performance metrics + AtlasPerfTracer.log(perf); + RequestContext.get().endMetricRecord(metric); + } + } +} \ No newline at end of file