Ns/dg 1716 auth benchmarking

.github/workflows/maven.yml

@@ -26,6 +26,7 @@ on:
       - development
       - master
       - lineageondemand
+      - ns/DG-1716-auth-benchmarking
 
 jobs:
   build:
@@ -48,7 +49,9 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Get branch name
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        run: |
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+          echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')"
         id: get_branch
 
       - name: Create Maven Settings
@@ -58,12 +61,12 @@ jobs:
             [{
                 "id": "github",
                 "username": "atlan-ci",
-                "password": "${{ secrets.my_pat }}"
+                "password": "${{ secrets.org_pat_github }}"
             }]
 
       - name: Build with Maven
         run: |
-          branch_name=${{ steps.get_branch.outputs.branch }} 
+          branch_name='${{ steps.get_branch.outputs.branch }}'
           if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]]
           then
               echo "build without dashboard"
@@ -77,7 +80,7 @@ jobs:
         shell: bash
 
       - name: Get version tag
-        run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
+        run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.org_pat_github }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
         id: get_version
 
       - name: Set up Buildx 
@@ -89,7 +92,7 @@ jobs:
         with:
           registry: ghcr.io
           username: $GITHUB_ACTOR
-          password: ${{ secrets.my_pat }}
+          password: ${{ secrets.org_pat_github }}
 
       - name: Build and push
         id: docker_build
@@ -102,8 +105,8 @@ jobs:
           provenance: true
           push: true
           tags: |
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }}
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }}
 
       - name: Scan Image
         uses: aquasecurity/trivy-action@master

auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java

@@ -52,6 +52,7 @@
 import org.apache.atlas.plugin.policyevaluator.RangerPolicyEvaluator;
 import org.apache.atlas.plugin.store.EmbeddedServiceDefsUtil;
 import org.apache.atlas.plugin.util.*;
+import org.openjdk.jol.info.GraphLayout;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -379,6 +380,7 @@ public void setPolicies(ServicePolicies policies) {
 							RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine;
 
 							newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies);
+							LOG.info("AuthPerformance: using old policies to create engine" + CollectionUtils.isNotEmpty(policies.getPolicyDeltas()));
 						}
 
 						if (newPolicyEngine != null) {
@@ -403,6 +405,7 @@ public void setPolicies(ServicePolicies policies) {
 				}
 
 				if (newPolicyEngine != null) {
+					LOG.info("AuthPerformance: PolicyEngine object size" + GraphLayout.parseInstance(newPolicyEngine).toFootprint());
 					if (!isPolicyEngineShared) {
 						newPolicyEngine.setUseForwardedIPAddress(pluginConfig.isUseForwardedIPAddress());
 						newPolicyEngine.setTrustedProxyAddresses(pluginConfig.getTrustedProxyAddresses());

auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java

@@ -30,6 +30,7 @@
 import org.apache.atlas.authorization.hadoop.config.RangerPluginConfig;
 import org.apache.atlas.plugin.policyengine.RangerPluginContext;
 import org.apache.atlas.plugin.service.RangerBasePlugin;
+import org.openjdk.jol.info.GraphLayout;
 
 import java.io.File;
 import java.io.FileReader;
@@ -326,6 +327,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound
 			} else {
 				svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis);
 			}
+			LOG.info("AuthPerformance: PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin() svcPolicies object size" + GraphLayout.parseInstance(svcPolicies).toFootprint());
 
 			boolean isUpdated = svcPolicies != null;
 

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -366,6 +366,7 @@ public final class Constants {
     public static final String IMPALA_SOURCE      = "impala";
     public static final String STORM_SOURCE       = "storm";
     public static final String FILE_SPOOL_SOURCE  = "file_spool";
+    public static final String ASSET_PRODUCT_GUIDS = "assetProductGUIDs";
 
     /*
      * All supported file-format extensions for Bulk Imports through file upload

intg/src/main/java/org/apache/atlas/model/instance/LinkDataProductRequest.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.model.instance;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Set;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+/**
+ * Request to link/unlink policies from asset.
+ */
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.PROPERTY)
+public class LinkDataProductRequest implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private Set<String> linkGuids;
+    private Set<String> unlinkGuids;
+
+    public Set<String> getLinkGuids() {
+        return linkGuids;
+    }
+
+    public void setLinkGuids(Set<String> linkGuids) {
+        this.linkGuids = linkGuids;
+    }
+
+    public Set<String> getUnlinkGuids() {
+        return unlinkGuids;
+    }
+
+    public void setUnlinkGuids(Set<String> unlinkGuids) {
+        this.unlinkGuids = unlinkGuids;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{");
+        sb.append("linkGuids=").append(linkGuids);
+        sb.append(", unlinkGuids=").append(unlinkGuids);
+        sb.append('}');
+        return sb.toString();
+    }
+}

pom.xml

@@ -1718,6 +1718,12 @@
             <version>2.2.2</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.openjdk.jol</groupId>
+            <artifactId>jol-core</artifactId>
+            <version>0.17</version>
+        </dependency>
+
     </dependencies>
 
     <build>

repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java

@@ -365,4 +365,10 @@ EntityMutationResponse deleteByUniqueAttributes(List<AtlasObjectId> objectIds)
 
     void repairAccesscontrolAlias(String guid) throws AtlasBaseException;
 
+    void linkProductToAsset(String productId, Set<String> linkGuids) throws AtlasBaseException;
+
+    void unlinkProductFromAsset(String productId, Set<String> unlinkGuids) throws AtlasBaseException;
+
+    void linkProductWithNotification(String productId, Set<String> unlinkGuids) throws AtlasBaseException;
+
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java

@@ -150,11 +150,12 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
     private final FeatureFlagStore featureFlagStore;
 
     private final ESAliasStore esAliasStore;
+    private final IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier;
 
     @Inject
     public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry,
                               IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement,
-                              AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore) {
+                              AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore, IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier) {
         this.graph                = graph;
         this.deleteDelegate       = deleteDelegate;
         this.restoreHandlerV1     = restoreHandlerV1;
@@ -168,6 +169,7 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate
         this.atlasRelationshipStore = atlasRelationshipStore;
         this.featureFlagStore = featureFlagStore;
         this.esAliasStore = new ESAliasStore(graph, entityRetriever);
+        this.atlasAlternateChangeNotifier = atlasAlternateChangeNotifier;
 
         try {
             this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null);
@@ -2737,6 +2739,73 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException {
 
         RequestContext.get().endMetricRecord(metric);
     }
+
+    @Override
+    @GraphTransaction
+    public void linkProductToAsset(String productGuid, Set<String> linkGuids) throws AtlasBaseException {
+        AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkProductToAsset.GraphTransaction");
+
+        try {
+            List<AtlasVertex> vertices = this.entityGraphMapper.linkProductToAsset(productGuid, linkGuids);
+            if (CollectionUtils.isEmpty(vertices)) {
+                return;
+            }
+
+            LOG.info("linkProductToAsset: productGuid={}, linkGuids={}", productGuid, linkGuids);
+
+            //handleProductMutation(vertices);
+        } catch (Exception e) {
+            LOG.error("Error during linkProduct for productGuid: {}", productGuid, e);
+            throw e;
+        } finally {
+            RequestContext.get().endMetricRecord(metric);
+        }
+    }
+
+    @Override
+    @GraphTransaction
+    public void linkProductWithNotification(String productGuid, Set<String> linkGuids) throws AtlasBaseException {
+        AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkProductToAsset.GraphTransaction");
+
+        try {
+            List<AtlasVertex> vertices = this.entityGraphMapper.linkProductWithNotification(productGuid, linkGuids);
+            if (CollectionUtils.isEmpty(vertices)) {
+                return;
+            }
+
+            handleProductMutation(vertices);
+        } catch (Exception e) {
+            LOG.error("Error during linkProduct for productGuid: {}", productGuid, e);
+            throw e;
+        } finally {
+            RequestContext.get().endMetricRecord(metric);
+        }
+    }
+
+    @Override
+    @GraphTransaction
+    public void unlinkProductFromAsset(String productGuid, Set<String> unlinkGuids) throws AtlasBaseException {
+        AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkProductFromAsset.GraphTransaction");
+        try {
+            List<AtlasVertex> vertices = this.entityGraphMapper.unlinkProductFromAsset(productGuid, unlinkGuids);
+            if (CollectionUtils.isEmpty(vertices)) {
+                return;
+            }
+
+            handleProductMutation(vertices);
+        } catch (Exception e) {
+            LOG.error("Error during unlinkProduct for productGuid: {}", productGuid, e);
+            throw e;
+        } finally {
+            RequestContext.get().endMetricRecord(metric);
+        }
+    }
+
+    private void handleProductMutation(List<AtlasVertex> vertices) throws AtlasBaseException {
+        AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("handleBusinessPolicyMutation");
+        this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices);
+        RequestContext.get().endMetricRecord(metricRecorder);
+    }
 }
 
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductNotifierImpl.java

@@ -0,0 +1,64 @@
+package org.apache.atlas.repository.store.graph.v2;
+
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.listener.EntityChangeListenerV2;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.utils.AtlasPerfMetrics;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.*;
+
+import static org.apache.atlas.repository.Constants.*;
+import static org.apache.atlas.repository.graph.GraphHelper.*;
+
+
+@Component
+public class DataProductNotifierImpl implements IAtlasMinimalChangeNotifier {
+
+    private final Set<EntityChangeListenerV2> entityChangeListenersV2;
+
+    @Inject
+    public DataProductNotifierImpl(Set<EntityChangeListenerV2> entityChangeListenersV2) {
+        this.entityChangeListenersV2 = entityChangeListenersV2;
+
+    }
+
+    @Override
+    public void onEntitiesMutation(final List<AtlasVertex> vertices) throws AtlasBaseException {
+        AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation");
+        final List<AtlasEntity> entities = new ArrayList<>(0);
+        vertices.forEach(item -> entities.add(createAtlasEntity(item)));
+        for (EntityChangeListenerV2 listener : entityChangeListenersV2) {
+            listener.onEntitiesUpdated(entities, false);
+        }
+
+        RequestContext.get().endMetricRecord(metricRecorder);
+    }
+
+    private AtlasEntity createAtlasEntity(AtlasVertex vertex) {
+        AtlasEntity atlasEntity = new AtlasEntity();
+        atlasEntity.setAttribute(QUALIFIED_NAME, vertex.getProperty(QUALIFIED_NAME, String.class));
+        atlasEntity.setAttribute(NAME, vertex.getProperty(NAME, String.class));
+
+        atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class));
+        atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class));
+        atlasEntity.setCreatedBy(vertex.getProperty(CREATED_BY_KEY, String.class));
+        atlasEntity.setUpdatedBy(vertex.getProperty(MODIFIED_BY_KEY, String.class));
+        atlasEntity.setCreateTime(new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class)));
+        atlasEntity.setUpdateTime(new Date(vertex.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class)));
+        atlasEntity.setIsProxy(vertex.getProperty(IS_PROXY_KEY, Boolean.class));
+        atlasEntity.setIsIncomplete(vertex.getProperty(IS_INCOMPLETE_PROPERTY_KEY, Boolean.class));
+        atlasEntity.setStatus(getStatus(vertex));
+        atlasEntity.setProvenanceType(getProvenanceType(vertex));
+        atlasEntity.setHomeId(getHomeId(vertex));
+        atlasEntity.setVersion(getVersion(vertex));
+
+
+        return atlasEntity;
+    }
+
+
+}