From f9618d1c7c6d20fd14ddd4578927226433913117 Mon Sep 17 00:00:00 2001 From: qGYdXbY2 <47661341+qGYdXbY2@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:02:15 +0100 Subject: [PATCH] ImlCopy: Catalog + Map, contentUpdatedAt Signed-off-by: qGYdXbY2 <47661341+qGYdXbY2@users.noreply.github.com> --- .../com/here/xyz/jobs/steps/JobCompiler.java | 4 +- .../{CopySpaceToSpace.java => SpaceCopy.java} | 37 ++++++++++++------- .../steps/impl/transport/CopySpacePost.java | 4 ++ 3 files changed, 29 insertions(+), 16 deletions(-) rename xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/{CopySpaceToSpace.java => SpaceCopy.java} (83%) diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/JobCompiler.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/JobCompiler.java index aa49d8a057..d2b6fced2e 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/JobCompiler.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/JobCompiler.java @@ -21,7 +21,7 @@ import com.here.xyz.jobs.Job; import com.here.xyz.jobs.steps.compiler.ExportToFiles; -import com.here.xyz.jobs.steps.compiler.CopySpaceToSpace; +import com.here.xyz.jobs.steps.compiler.SpaceCopy; import com.here.xyz.jobs.steps.compiler.ImportFromFiles; import com.here.xyz.jobs.steps.compiler.JobCompilationInterceptor; import com.here.xyz.util.Async; @@ -41,7 +41,7 @@ public class JobCompiler { static { registerCompilationInterceptor(ImportFromFiles.class); registerCompilationInterceptor(ExportToFiles.class); - registerCompilationInterceptor(CopySpaceToSpace.class); + registerCompilationInterceptor(SpaceCopy.class); } public Future compile(Job job) { diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/CopySpaceToSpace.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/SpaceCopy.java similarity index 83% rename from xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/CopySpaceToSpace.java rename to xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/SpaceCopy.java index 328a256889..c05a68c2f6 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/CopySpaceToSpace.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/SpaceCopy.java @@ -19,6 +19,7 @@ package com.here.xyz.jobs.steps.compiler; +import java.util.Map; import java.util.Set; import com.here.xyz.events.PropertiesQuery; @@ -37,7 +38,7 @@ import com.here.xyz.util.web.HubWebClient; import com.here.xyz.util.web.XyzWebClient.WebClientException; -public class CopySpaceToSpace implements JobCompilationInterceptor { +public class SpaceCopy implements JobCompilationInterceptor { protected boolean validSubType( String subType ) { return "Space".equals(subType); } @@ -50,7 +51,7 @@ && validSubType( job.getSource().getClass().getSimpleName() ) && validSubType( job.getTarget().getClass().getSimpleName() ); } - private int threadCountCalc( long sourceFeatureCount, long targetFeatureCount ) + private static int threadCountCalc( long sourceFeatureCount, long targetFeatureCount ) { long PARALLELIZTATION_THRESHOLD = 100000; int PARALLELIZTATION_THREAD_MAX = 8; @@ -62,22 +63,21 @@ private int threadCountCalc( long sourceFeatureCount, long targetFeatureCount ) return PARALLELIZTATION_THREAD_MAX; } - @Override - public CompilationStepGraph compile(Job job) { - final String sourceSpaceId = job.getSource().getKey(); - final String targetSpaceId = job.getTarget().getKey(); - + public static CompilationStepGraph compileSteps(String sourceId, String targetId, String jobId, Filters filters, Ref versionRef) + { + final String sourceSpaceId = sourceId, + targetSpaceId = targetId; StatisticsResponse sourceStatistics = null, targetStatistics = null; try { sourceStatistics = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(sourceSpaceId); targetStatistics = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(targetSpaceId); } catch (WebClientException e) { - String errMsg = String.format("Unable to get Staistics for %s", sourceStatistics != null ? sourceSpaceId : targetSpaceId ); + String errMsg = String.format("Unable to get Staistics for %s", sourceStatistics == null ? sourceSpaceId : targetSpaceId ); throw new JobCompiler.CompilationError(errMsg); } - CopySpacePre preCopySpace = new CopySpacePre().withSpaceId(targetSpaceId).withJobId(job.getId()); + CopySpacePre preCopySpace = new CopySpacePre().withSpaceId(targetSpaceId).withJobId(jobId); CompilationStepGraph startGraph = new CompilationStepGraph(); startGraph.addExecution(preCopySpace); @@ -87,9 +87,6 @@ public CompilationStepGraph compile(Job job) { int threadCount = threadCountCalc(sourceFeatureCount, targetFeatureCount); - Filters filters = ((DatasetDescription.Space) job.getSource()).getFilters(); - Ref versionRef = ((DatasetDescription.Space) job.getSource()).getVersionRef(); - SpatialFilter spatialFilter = filters != null ? filters.getSpatialFilter() : null; PropertiesQuery propertyFilter = filters != null ? filters.getPropertyFilter() : null; @@ -103,7 +100,7 @@ public CompilationStepGraph compile(Job job) { .withSourceVersionRef(versionRef) .withPropertyFilter(propertyFilter) .withThreadInfo(new int[]{ threadId, threadCount }) - .withJobId(job.getId()); + .withJobId(jobId); if (spatialFilter != null) { copySpaceStep.setGeometry(spatialFilter.getGeometry()); @@ -117,10 +114,22 @@ public CompilationStepGraph compile(Job job) { startGraph.addExecution(cGraph); CopySpacePost postCopySpace = new CopySpacePost().withSpaceId(targetSpaceId) - .withJobId(job.getId()); + .withJobId(jobId); startGraph.addExecution(postCopySpace); return startGraph; + + } + + @Override + public CompilationStepGraph compile(Job job) { + + return compileSteps(job.getSource().getKey(), + job.getTarget().getKey(), + job.getId(), + ((DatasetDescription.Space) job.getSource()).getFilters(), + ((DatasetDescription.Space) job.getSource()).getVersionRef() ); + } } diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java index 5f431d52bf..040743948c 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/CopySpacePost.java @@ -29,6 +29,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,6 +46,7 @@ import com.here.xyz.models.hub.Space; import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.service.BaseHttpServerVerticle.ValidationException; +import com.here.xyz.util.service.Core; import com.here.xyz.util.web.XyzWebClient.WebClientException; @@ -190,6 +192,8 @@ public void execute() throws Exception { registerOutputs(List.of(statistics), true); setCopiedByteSize( statistics.getByteSize() ); + + hubWebClient().patchSpace(getSpaceId(), Map.of("contentUpdatedAt", Core.currentTimeMillis())); }