ImlCopy: catalog/map adjustments (heremaps#1411)

* ImlCopy: catalog/map adjustments Signed-off-by: qGYdXbY2 <[email protected]> * contentUpdatedAt on featurecount Signed-off-by: qGYdXbY2 <[email protected]> * loadSpaceStatistics - explicit EXTENSION Signed-off-by: qGYdXbY2 <[email protected]> * output key [Layer|Space] Signed-off-by: qGYdXbY2 <[email protected]> --------- Signed-off-by: qGYdXbY2 <[email protected]>
qGYdXbY2 · Dec 9, 2024 · 3c2a676 · 3c2a676
1 parent 9354d2a
commit 3c2a676
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 8 deletions.
diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/SpaceCopy.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/SpaceCopy.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 
 import com.here.xyz.events.PropertiesQuery;
+import com.here.xyz.events.ContextAwareEvent.SpaceContext;
 import com.here.xyz.jobs.Job;
 import com.here.xyz.jobs.datasets.DatasetDescription;
 import com.here.xyz.jobs.datasets.filters.Filters;
@@ -34,6 +35,7 @@
 import com.here.xyz.jobs.steps.impl.transport.CopySpacePre;
 import com.here.xyz.jobs.steps.impl.transport.CopySpacePost;
 import com.here.xyz.models.hub.Ref;
+import com.here.xyz.models.hub.Space;
 import com.here.xyz.responses.StatisticsResponse;
 import com.here.xyz.util.web.HubWebClient;
 import com.here.xyz.util.web.XyzWebClient.WebClientException;
@@ -63,15 +65,23 @@ private static int threadCountCalc( long sourceFeatureCount, long targetFeatureC
     return PARALLELIZTATION_THREAD_MAX; 
   }
 
-  public static CompilationStepGraph compileSteps(String sourceId, String targetId, String jobId, Filters filters, Ref versionRef) 
+
+  private static StatisticsResponse _loadSpaceStatistics(String spaceId) throws WebClientException
+  {
+   Space sourceSpace = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpace(spaceId);
+   boolean isExtended = sourceSpace.getExtension() != null;
+   return HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(spaceId, isExtended ? SpaceContext.EXTENSION : null);  
+  }
+
+  public static CompilationStepGraph compileSteps(String sourceId, String targetId, String jobId, Filters filters, Ref versionRef, String targetType) 
   {
     final String sourceSpaceId = sourceId,
                  targetSpaceId = targetId;
 
     StatisticsResponse sourceStatistics = null, targetStatistics = null;
     try {
-      sourceStatistics = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(sourceSpaceId);
-      targetStatistics = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(targetSpaceId);
+      sourceStatistics = _loadSpaceStatistics(sourceSpaceId);
+      targetStatistics = _loadSpaceStatistics(targetSpaceId);
     } catch (WebClientException e) {
       String errMsg = String.format("Unable to get Staistics for %s", sourceStatistics == null ? sourceSpaceId : targetSpaceId );
       throw new JobCompiler.CompilationError(errMsg);
@@ -114,7 +124,8 @@ public static CompilationStepGraph compileSteps(String sourceId, String targetId
     startGraph.addExecution(cGraph);
 
     CopySpacePost postCopySpace = new CopySpacePost().withSpaceId(targetSpaceId)
-                                                     .withJobId(jobId);
+                                                     .withJobId(jobId)
+                                                     .withTargetType(targetType);
 
     startGraph.addExecution(postCopySpace);
 
@@ -127,9 +138,10 @@ public CompilationStepGraph compile(Job job) {
 
     return compileSteps(job.getSource().getKey(), 
                         job.getTarget().getKey(), 
-                        job.getId(), 
+                        job.getId(),
                         ((DatasetDescription.Space<?>) job.getSource()).getFilters(),
-                        ((DatasetDescription.Space<?>) job.getSource()).getVersionRef() );
+                        ((DatasetDescription.Space<?>) job.getSource()).getVersionRef(),
+                        job.getTarget().getClass().getSimpleName() );
 
   }
 }
diff --git a/...obs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java b/...obs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java
@@ -84,6 +84,8 @@ public class CopySpacePost extends SpaceBasedStep<CopySpacePost> {
   @JsonView({Internal.class, Static.class})
   private long copiedByteSize = 0;
 
+  @JsonView({Internal.class, Static.class})
+  private String targetType = "Space";
 
   public long getCopiedByteSize() {
     return copiedByteSize;
@@ -97,6 +99,20 @@ public long getFetchedVersion() {
     return fetchedVersion;
   }
 
+  public String getTargetType() {
+    return targetType;
+  }
+
+  public void setTargetType(String targetType) {
+    this.targetType = targetType;
+  }
+
+  public CopySpacePost withTargetType(String targetType) {
+    setTargetType( targetType );
+    return this;
+  }
+
+
   @Override
   public List<Load> getNeededResources() {
     try {
@@ -184,16 +200,18 @@ public void execute() throws Exception {
      if( output instanceof FetchedVersions f )
       fetchedVersion = f.getFetchtedSequence();
 
-    infoLog(STEP_EXECUTE, this,String.format("Get stats for version %d - %s", fetchedVersion ,getSpaceId() )); 
+    infoLog(STEP_EXECUTE, this,String.format("Get stats for version %d - %s", fetchedVersion, getSpaceId() )); 
 
     FeatureStatistics statistics = getCopiedFeatures(fetchedVersion);
+    statistics.withMetadata( getTargetType(), getSpaceId() );  // key = [Layer(iml)|Space(xyz)] 
 
     infoLog(STEP_EXECUTE, this,"Job Statistics: bytes=" + statistics.getByteSize() + " rows=" + statistics.getFeatureCount());
     registerOutputs(List.of(statistics), true);
 
     setCopiedByteSize( statistics.getByteSize() );
 
-    hubWebClient().patchSpace(getSpaceId(), Map.of("contentUpdatedAt", Core.currentTimeMillis()));
+    if( statistics.getFeatureCount() > 0 )
+     hubWebClient().patchSpace(getSpaceId(), Map.of("contentUpdatedAt", Core.currentTimeMillis()));
 
   }