Bug fixes, v 2.1 release

force-app/main/algorithms/classes/ClusterCentroidDataAssignmentStep.cls

@@ -84,15 +84,20 @@ public with sharing class ClusterCentroidDataAssignmentStep extends ClusterBatch
             //Calculating min distance to centroids
             Integer nearestCentroidIndex = 0;
             Integer previousCentroidIndex = 0;
-            Double minDistance = this.runner.calculateDistance(currentDataPoint.values, centroids[nearestCentroidIndex].values);
-            centroidDistances[0] = minDistance;
-            for (Integer cindex = 1; cindex < centroidSize; cindex++) {
-                Double distance = this.runner.calculateDistance(currentDataPoint.values, centroids[cindex].values);
+            Double minDistance = ClusterDataHelper.DOUBLE_MAX_VALUE;
+            Double prevMinDistance = ClusterDataHelper.DOUBLE_MAX_VALUE;
+            for (Integer cindex = 0; cindex < centroidSize; cindex++) {
+                Double distance = Math.abs(this.runner.calculateDistance(currentDataPoint.values, centroids[cindex].values));
                 centroidDistances[cindex] = distance;
-                if (Math.abs(distance) < Math.abs(minDistance)) {
+                if (distance < minDistance) {
+                    previousCentroidIndex = nearestCentroidIndex;
+                    prevMinDistance = minDistance;
                     minDistance = distance;
                     nearestCentroidIndex = cindex;
-                    previousCentroidIndex = nearestCentroidIndex;
+                }
+                else if (distance < prevMinDistance) {
+                    prevMinDistance = distance;
+                    previousCentroidIndex = cindex;
                 }
             }
             //We found closest and second closest centroids to the current data point

force-app/main/algorithms/classes/ClusterKMeansJobState.cls

@@ -12,12 +12,16 @@ public with sharing class ClusterKMeansJobState extends ClusterJobState {
     public Boolean hasSwapped;
     public List<Id> sampleResultsIds;
     public Integer iterationsCount;
+    private Boolean hasNearestCentroidsCache;
+    private Boolean hasNearestCentroidsCalculated;
 
     public ClusterKMeansJobState() {
         this.centroids = new List<ClusterDataPoint>();
         this.hasAssignmentChanged = false;
         this.hasSwapped = false;
         this.iterationsCount = 0;
+        this.hasNearestCentroidsCache = false;
+        this.hasNearestCentroidsCalculated = false;
     }
 
     public override void loadFromMap(Map<String, Object> stateValues) {
@@ -73,6 +77,10 @@ public with sharing class ClusterKMeansJobState extends ClusterJobState {
     }
 
     public boolean hasNearestCentroids() {
+        if (this.hasNearestCentroidsCalculated) {
+            return this.hasNearestCentroidsCache;
+        }
+        this.hasNearestCentroidsCalculated = true;
         if (this.nearestCentroids != null) {
             Boolean hasNC = true;
             for (Integer nnIndex:this.nearestCentroids) {
@@ -81,9 +89,11 @@ public with sharing class ClusterKMeansJobState extends ClusterJobState {
                     break;
                 }
             }
+            this.hasNearestCentroidsCache = hasNC;
             return hasNC;
         }
-        return false;
+        this.hasNearestCentroidsCache = false;
+        return this.hasNearestCentroidsCache;
     }
 
 }

force-app/main/algorithms/classes/ClusterKNNPredictor.cls

@@ -72,7 +72,6 @@ public with sharing class ClusterKNNPredictor {
         for (ClusterPredictionResult.FieldPrediction prediction:predictionResult.fieldPredictions) {
             prediction.aggregateValues(nearestNeighbors.size());
         }
-        log.debug(JSON.serialize(clusterPrediction));
         predictionResult.clusterIndex = Integer.valueOf(clusterPrediction.fieldValuePredictions[0].value);
         return predictionResult;
     }
@@ -83,14 +82,14 @@ public with sharing class ClusterKNNPredictor {
         Id jobId = state.clusterJob.Id;
         ClusterAccessCheck.checkCRUDPermission(Schema.SObjectType.ClusterJobNeighbor__c);
         ClusterAccessCheck.checkCRUDPermission(Schema.SObjectType.ClusterJobResult__c);
-        List<ClusterNeighbor> clusterNeighbors = this.runner.findNearestClusters(recordId, ClusterConstants.NUM_NEAREST_CLUSTERS);
+        ClusterDataPoint dataPoint = this.runner.getDataPoint(recordId);
+        List<ClusterNeighbor> clusterNeighbors = this.runner.findNearestClusters(dataPoint, ClusterConstants.NUM_NEAREST_CLUSTERS);
         List<Integer> nearestClusters = new List<Integer>();
         for (ClusterNeighbor cn:clusterNeighbors) {
             nearestClusters.add(cn.clusterIndex);
         }
         log.debug('Retrieved ' + nearestClusters.size() + ' nearest clusters');
         //We will retrieve MAX_NEIGHBORS neighbors and then return first numNeighbors
-        ClusterDataPoint dataPoint = this.runner.getDataPoint(recordId);
         List<ClusterDataPointNeighbor> neighbors = this.findNearestNeighbors(dataPoint, nearestClusters, ClusterConstants.MAX_NEIGHBORS);
         List<ClusterJobNeighbor__c> neighborRecords = new List<ClusterJobNeighbor__c>();
         for (ClusterDataPointNeighbor neighbor:neighbors) {
@@ -167,7 +166,7 @@ public with sharing class ClusterKNNPredictor {
             if (dataPoint.clusterIndex == nearestClusters[i]) {
                 currentCentroidDistance = distance;
             }
-            Integer nextClusterIndex = jobState.getNextClusterIndex(i);
+            Integer nextClusterIndex = jobState.getNextClusterIndex(nearestClusters[i]);
             Double nextClusterDistance = this.runner.calculateDPDistance(dataPoint, jobState.centroids[nextClusterIndex]);
             nearestDataPoints.addAll(this.getRandomDataPoints(ClusterConstants.MIN_KNN_DP_COUNT, nearestClusters[i], distance, nextClusterDistance, jobState));
         }
@@ -242,29 +241,35 @@ public with sharing class ClusterKNNPredictor {
     }
 
     public ClusterDataPoint[] getRandomDataPoints(Integer count, Integer clusterIndex, Double distanceToCenter, Double distanceToNextCluster, ClusterJobState jobState) {
-        log.debug('Retrieving nearest random data points in cluster ' + clusterIndex);
+        log.debug('Retrieving nearest random data points in cluster ' + clusterIndex + ', distanceToCenter: ' + distanceToCenter + ', distanceToNextCluster: ' + distanceToNextCluster);
         ClusterAccessCheck.checkReadPermission(Schema.SObjectType.ClusterJobResult__c);
         List<ClusterJobResult__c> jobResults = null;
         if (distanceToCenter != null) {
-            Double[] distanceTolerances = new Double[6];
-            distanceTolerances[0] = distanceToCenter * 0.0001;  //0.001%
-            distanceTolerances[1] = distanceToCenter * 0.001;  //0.01%
-            distanceTolerances[2] = distanceToCenter * 0.01; //1%
-            distanceTolerances[3] = distanceToCenter * 0.05; //5%
-            distanceTolerances[4] = distanceToCenter * 0.1; //10%
-            distanceTolerances[5] = distanceToCenter * 0.25; //25%
+            Double[] distanceTolerances = new Double[10];
+            distanceTolerances[0] = 0.001;  //0.1%
+            distanceTolerances[1] = 0.002;  //0.2%
+            distanceTolerances[2] = 0.004;  //0.2%
+            distanceTolerances[3] = 0.008;  //0.2%
+            distanceTolerances[4] = 0.01; //1%
+            distanceTolerances[5] = 0.02; //1%
+            distanceTolerances[6] = 0.04; //1%
+            distanceTolerances[7] = 0.05; //5%
+            distanceTolerances[8] = 0.1; //10%
+            distanceTolerances[9] = 0.25; //25%
             for (Integer i=0; i<distanceTolerances.size(); i++) {
-                Decimal distanceMin = Decimal.valueOf(distanceToCenter - distanceTolerances[i]);
-                Decimal distanceMax = Decimal.valueOf(distanceToCenter + distanceTolerances[i]);
-                Decimal nextDistanceMax = Decimal.valueOf(distanceToNextCluster + distanceTolerances[i]);
+                Decimal distanceMin = Decimal.valueOf(distanceToCenter - distanceToCenter * distanceTolerances[i]);
+                Decimal distanceMax = Decimal.valueOf(distanceToCenter + distanceToCenter * distanceTolerances[i]);
+                Decimal nextDistanceMin = Decimal.valueOf(distanceToNextCluster - distanceToCenter * distanceTolerances[i]);
+                Decimal nextDistanceMax = Decimal.valueOf(distanceToNextCluster + distanceToCenter * distanceTolerances[i]);
                 //We are trying to retrieve results with similar distanceToCenter value to reduce KNN calculations
                 //Here we are looking for an intersection of the current and nearest clusters
                 jobResults = [SELECT Id, Cluster__c, Json__c, Json2__c, Json3__c, Json4__c, Json5__c, RecordId__c, RecordName__c, ClusterNumber__c, ClusterJob__c, DistanceToCluster__c, DistanceToNNCluster__c 
                     FROM ClusterJobResult__c WHERE ClusterJob__c = :jobState.clusterJob.Id AND ClusterNumber__c = :clusterIndex 
-                    AND (DistanceToCluster__c >= :distanceMin AND DistanceToCluster__c<= :distanceMax AND DistanceToNNCluster__c<=:nextDistanceMax) 
-                    ORDER BY Random__c LIMIT :count];
+                    AND (DistanceToCluster__c >= :distanceMin AND DistanceToCluster__c<= :distanceMax AND DistanceToNNCluster__c<=:nextDistanceMax AND DistanceToNNCluster__c>=:nextDistanceMin) 
+                    ORDER BY DistanceToCluster__c DESC LIMIT :count];
                 if (jobResults.size() > ClusterConstants.MIN_KNN_NEIGHBOR_SIZE) {
                     log.debug('Retrieved ' + jobResults.size() + ' nearest data points with tolerance ' + distanceTolerances[i]);
+                    log.debug('distanceMin: ' + distanceMin + ', distanceMax: ' + distanceMax + ', nextDistanceMax: ' + nextDistanceMax);
                     break;
                 }            
             }

force-app/main/algorithms/classes/ClusterPAMDataAssignmentStep.cls

@@ -43,22 +43,30 @@ public with sharing class ClusterPAMDataAssignmentStep extends ClusterIterableBa
             ClusterDataPoint currentRecord = dataPoints[sindex];
             //ClusterDataHelper.normalizeObject(currentObject, jobState);
             //Calculating min distance to centroids
-            Integer nearestCentroidIndex = 0;
+            Integer nearestCentroidIndex = -1;
+            Integer prevNearestCentroidIndex = -1;
             Boolean isCentroid = false;
             Integer centroidIndex;
-            Double minDistance = this.runner.calculateDPDistance(currentRecord, centroids[nearestCentroidIndex]);
-            for (Integer cindex = 1; cindex < centroidSize; cindex++) {
+            Double minDistance = ClusterDataHelper.DOUBLE_MAX_VALUE;
+            Double prevMinDistance = ClusterDataHelper.DOUBLE_MAX_VALUE;
+            for (Integer cindex = 0; cindex < centroidSize; cindex++) {
                 Boolean isCurrentCentroid = centroids[cindex].recordId == currentRecord.recordId;
                 isCentroid = isCentroid || isCurrentCentroid;
                 if (isCurrentCentroid) {
                     currentRecord.clusterIndex = null;
                     centroidIndex = cindex;
                 }
-                Double distance = this.runner.calculateDPDistance(currentRecord, centroids[cindex]);
-                if (Math.abs(distance) < Math.abs(minDistance)) {
+                Double distance = Math.abs(this.runner.calculateDPDistance(currentRecord, centroids[cindex]));
+                if (distance < minDistance) {
+                    prevMinDistance = minDistance;
+                    prevNearestCentroidIndex = nearestCentroidIndex;
                     minDistance = distance;
                     nearestCentroidIndex = cindex;
                 }
+                else if (distance < prevMinDistance) {
+                    prevMinDistance = distance;
+                    prevNearestCentroidIndex = cindex;
+                }
             }
             if (!isCentroid) {
                 //Reassigning to another cluster if needed
@@ -72,7 +80,7 @@ public with sharing class ClusterPAMDataAssignmentStep extends ClusterIterableBa
             }
             else {
                 //If current dp is a centroid store the nearest centroid
-                jobState.nearestCentroids[centroidIndex] = nearestCentroidIndex;
+                jobState.nearestCentroids[centroidIndex] = prevNearestCentroidIndex;
             }
         }        
         //Aggregating cost for each centroid/medoid

force-app/main/api/classes/ClusterApi.cls

@@ -98,5 +98,16 @@ global with sharing class ClusterApi {
         return this.runner.getDataPoint(externalRecordId);
     }
 
+    /**
+     * Converts an SObject to a data point
+     * @param record
+     *        SObject record to convert
+     * @returns ClusterDataPoint object with converted data
+     */
+    global ClusterDataPoint convertToDataPoint(SObject record) {
+        this.checkRunnerInitialized();
+        ClusterSObjectProcessor objectProcessor = this.runner.getSObjectProcessor();
+        return objectProcessor.processSObject(record);
+    }
 
 }

force-app/main/default/aura/ClusterResultDetailsView/ClusterResultDetailsView.cmp

@@ -1,7 +1,8 @@
 <aura:component implements="force:hasRecordId,lightning:actionOverride"
     controller="ClusterJobDetailsController" extends="c:ClusterUiBaseComponent" access="global">
     <aura:attribute name="jobResultDetails" type="Object" access="public" />
-    <aura:handler name="init" value="{!this}" action="{!c.onInit}" />
+    <ltng:require scripts="{!join(',',
+        $Resource.clustanUtils + '/clustanUtils.js')}" afterScriptsLoaded="{!c.onInit}"  />    
     <lightning:notificationsLibrary aura:id="notifLib" />
     <div class="c-container slds-scope slds-container slds-panel">
 

force-app/main/default/aura/ClusterResultDetailsView/ClusterResultDetailsViewHelper.js

@@ -8,6 +8,8 @@
                 if (uiModel.jobStateString && uiModel.jobStateString !== '') {
                     uiModel.jobState = JSON.parse(uiModel.jobStateString);
                 }
+                clustanUtils.decompressJobState(uiModel.jobState);
+                clustanUtils.decompressDataPointValues(uiModel.jobState, uiModel.dataPoint.values);
                 component.set("v.jobResultDetails", uiModel);
                 let dpDetails = component.find('dataPointDetails');
                 dpDetails.set('v.dataPoint', uiModel.dataPoint);

force-app/main/default/classes/ClusterConstants.cls

@@ -180,4 +180,10 @@ public with sharing class ClusterConstants {
         return (String[])JSON.deserialize(getLongTextSettingValue('DefaultClusterColors','[]'), String[].class);
     }
 
+    public static Boolean getStorePredictions() {
+        Integer storePredictions = getIntegerSettingValue('StorePredictedDataPoints', 0);
+        return storePredictions == 1;
+
+    }
+
 }

force-app/main/default/classes/ClusterPredictController.cls

@@ -24,7 +24,7 @@ public with sharing class ClusterPredictController {
             ClusterAccessCheck.checkReadPermission(Schema.SObjectType.ClusterModel__c);
             ClusterAccessCheck.checkReadPermission(Schema.SObjectType.ClusterJob__c);
             uiModel.models = new List<ClusterModelWrapper>();
-            if (objectDescribe.getSObjectType() == ClusterModel__c.getSObjectType()) {
+            if (objectDescribe.getKeyPrefix() == ClusterModel__c.getSObjectType().getDescribe().getKeyPrefix()) {
                 ClusterModelWrapper model = ClusterModelBuilderController.loadModel(recordId);
                 uiModel.models.add(model);
                 uiModel.jobId = getLastCompletedJobId(model.modelId);
@@ -33,7 +33,7 @@ public with sharing class ClusterPredictController {
                 }
                 uiModel.recordIdNeeded = true;
             }
-            else if (objectDescribe.getSObjectType() == ClusterJob__c.getSObjectType()) {
+            else if (objectDescribe.getKeyPrefix() == ClusterJob__c.getSObjectType().getDescribe().getKeyPrefix()) {
                 List<ClusterJob__c> jobs = [SELECT Id,ClusterModel__c FROM ClusterJob__c WHERE Id = :recordId AND JobStatus__c = :ClusterConstants.JOBSTATUS_COMPLETED  
                     WITH SECURITY_ENFORCED ORDER BY CreatedDate DESC LIMIT 1];
                 if (jobs.size() == 1) {
@@ -86,10 +86,10 @@ public with sharing class ClusterPredictController {
             ClusterAccessCheck.checkReadPermission(Schema.SObjectType.ClusterModel__c);
             ClusterAccessCheck.checkReadPermission(Schema.SObjectType.ClusterJob__c);
             ClusterModelWrapper model;
-            if (objectDescribe.getSObjectType() == ClusterModel__c.getSObjectType()) {
+            if (objectDescribe.getKeyPrefix() == ClusterModel__c.getSObjectType().getDescribe().getKeyPrefix()) {
                 model = ClusterModelBuilderController.loadModel(jobOrModelId);
             }
-            else if (objectDescribe.getSObjectType() == ClusterJob__c.getSObjectType()) {
+            else if (objectDescribe.getKeyPrefix() == ClusterJob__c.getSObjectType().getDescribe().getKeyPrefix()) {
                 List<ClusterJob__c> jobs = [SELECT Id,ClusterModel__c FROM ClusterJob__c WHERE Id = :jobOrModelId AND JobStatus__c = :ClusterConstants.JOBSTATUS_COMPLETED  
                     WITH SECURITY_ENFORCED ORDER BY CreatedDate DESC LIMIT 1];
                 if (jobs.size() == 1) {
@@ -285,7 +285,7 @@ public with sharing class ClusterPredictController {
             log.debug('Starting k nearest neighbor calculations for record id: ' + recordId);
             ClusterAccessCheck.checkCRUDPermission(Schema.SObjectType.ClusterJobNeighbor__c);
             ClusterAlgorithmRunner runner = ClusterAlgorithmFactory.getRunnerFromJobId(jobId);
-            List<ClusterDataPointNeighbor> neighbors = runner.getPredictor().findNearestNeighbors(recordId, numNeighbors, true);
+            List<ClusterDataPointNeighbor> neighbors = runner.getPredictor().findNearestNeighbors(recordId, numNeighbors, ClusterConstants.getStorePredictions());
             return neighbors;
         }
         catch (Exception ex) {

force-app/main/default/customMetadata/ClusterSetting.StorePredictedDataPoints.md-meta.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CustomMetadata xmlns="http://soap.sforce.com/2006/04/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+    <label>Store Predicted Data Points</label>
+    <protected>false</protected>
+    <values>
+        <field>Description__c</field>
+        <value xsi:type="xsd:string">If 0 new records will not be included into predictions</value>
+    </values>
+    <values>
+        <field>LongTextValue__c</field>
+        <value xsi:nil="true"/>
+    </values>
+    <values>
+        <field>Value__c</field>
+        <value xsi:type="xsd:string">0</value>
+    </values>
+</CustomMetadata>

force-app/main/test/classes/ClusterApiTest.cls

@@ -11,7 +11,7 @@ public with sharing class ClusterApiTest {
         User clusterUser = ClusterTestData.createClusterUser();
         System.runAs(clusterUser) {
             Test.startTest();
-            List<Lead> leads = [SELECT Id, Name FROM Lead LIMIT 10];
+            List<Lead> leads = [SELECT Id, Name, AnnualRevenue, NumberOfEmployees FROM Lead LIMIT 10];
             List<ClusterJob__c> jobs = [SELECT Id FROM ClusterJob__c LIMIT 1];
             Id jobId = jobs[0].Id;
             Id recordId = leads[0].Id;
@@ -23,6 +23,8 @@ public with sharing class ClusterApiTest {
             System.assertEquals(true, predictionResult.getClusterIndex() >= 0, 'Incorrect prediction cluster index');
             ClusterDataPoint dataPoint = api.getDataPoint(recordId);
             System.assertEquals(recordId, dataPoint.getExternalId(), 'Incorrect data point id');
+            ClusterDataPoint dataPoint2 = api.convertToDataPoint(leads[0]);
+            System.assertEquals(dataPoint.getRecordName(), dataPoint2.getRecordName(), 'Incorrect data point name');
             Double distance = api.calculateDistance(dataPoint.getValues(), dataPoint.getValues());
             System.assertEquals(true, ClusterDataHelper.doublesEqual(distance, ClusterDataHelper.DOUBLE_ZERO), 'Incorrect distance');
             Test.stopTest();
@@ -35,7 +37,7 @@ public with sharing class ClusterApiTest {
         User clusterUser = ClusterTestData.createClusterUser();
         System.runAs(clusterUser) {
             Test.startTest();
-            List<Lead> leads = [SELECT Id, Name FROM Lead LIMIT 11];
+            List<Lead> leads = [SELECT Id, Name, AnnualRevenue, NumberOfEmployees FROM Lead LIMIT 11];
             List<ClusterJob__c> jobs = [SELECT Id FROM ClusterJob__c LIMIT 1];
             Id jobId = jobs[0].Id;
             Id recordId = leads[0].Id;