diff --git a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/dao/ProjectMapperWrapper.java b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/dao/ProjectMapperWrapper.java index 9b44fd3e..7633d078 100644 --- a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/dao/ProjectMapperWrapper.java +++ b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/dao/ProjectMapperWrapper.java @@ -86,16 +86,17 @@ public void updateSingleJobStatus(String user, String agency, JobDO jobDO, JobSt batchUpdateJobStatus(user, agency, new ArrayList<>(Arrays.asList(jobDO)), status); } - public List queryJobDetail(String jobID, String user, String agency) { + public List queryJobDetail(String jobID, Boolean onlyMeta, String user, String agency) { JobDO condition = new JobDO(jobID); - List jobDOList = this.queryJobByCondition(false, user, agency, condition); + List jobDOList = this.queryJobByCondition(onlyMeta, user, agency, condition); // try to query the follower information if (jobDOList == null || jobDOList.isEmpty()) { // reset the condition condition.setOwner(null); condition.setOwnerAgency(null); jobDOList = - this.projectMapper.queryFollowerJobByCondition(false, user, agency, condition); + this.projectMapper.queryFollowerJobByCondition( + onlyMeta, user, agency, condition); } return jobDOList; } diff --git a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/model/JobRequest.java b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/model/JobRequest.java index 2aaf5905..1822ee2d 100644 --- a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/model/JobRequest.java +++ b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/model/JobRequest.java @@ -15,6 +15,7 @@ package com.webank.wedpr.components.project.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.webank.wedpr.common.utils.Common; import com.webank.wedpr.common.utils.PageRequest; import com.webank.wedpr.common.utils.WeDPRException; @@ -23,12 +24,16 @@ import com.webank.wedpr.components.project.dao.ProjectDO; import com.webank.wedpr.components.project.dao.ProjectMapper; import java.util.List; +import lombok.Data; import lombok.SneakyThrows; +@Data +@JsonIgnoreProperties(ignoreUnknown = true) public class JobRequest extends PageRequest { private JobDO job = new JobDO(true); private List taskParties; private List datasetList; + private Boolean onlyMeta = Boolean.TRUE; public JobRequest() {} @@ -60,10 +65,6 @@ public void setJob(JobDO job) { this.job = job; } - public List getTaskParties() { - return taskParties; - } - public void setTaskParties(List taskParties) { this.taskParties = taskParties; checkAndConfigTaskParities(taskParties); @@ -92,6 +93,13 @@ public void setDatasetList(List datasetList) { this.datasetList = datasetList; } + public void setOnlyMeta(Boolean onlyMeta) { + if (onlyMeta == null) { + return; + } + this.onlyMeta = onlyMeta; + } + @Override public String toString() { return "JobRequest{" diff --git a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/service/impl/ProjectServiceImpl.java b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/service/impl/ProjectServiceImpl.java index f571e230..9cc0af10 100644 --- a/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/service/impl/ProjectServiceImpl.java +++ b/wedpr-components/meta/project/src/main/java/com/webank/wedpr/components/project/service/impl/ProjectServiceImpl.java @@ -327,7 +327,10 @@ public WeDPRResponse queryJobByCondition(String user, JobRequest request) { // query with using owner identity List jobDOList = this.projectMapperWrapper.queryJobByCondition( - false, user, WeDPRCommonConfig.getAgency(), request.getJob()); + request.getOnlyMeta(), + user, + WeDPRCommonConfig.getAgency(), + request.getJob()); response.setData( new BatchJobList(new PageInfo(jobDOList).getTotal(), jobDOList)); return response; @@ -431,7 +434,10 @@ public WeDPRResponse queryFollowerJobByCondition(String user, JobRequest request this.projectMapperWrapper .getProjectMapper() .queryFollowerJobByCondition( - false, user, WeDPRCommonConfig.getAgency(), request.getJob()); + request.getOnlyMeta(), + user, + WeDPRCommonConfig.getAgency(), + request.getJob()); if (jobDOList == null) { return response; } diff --git a/wedpr-components/meta/project/src/main/resources/mapper/ProjectMapper.xml b/wedpr-components/meta/project/src/main/resources/mapper/ProjectMapper.xml index cb778a14..de3421a2 100644 --- a/wedpr-components/meta/project/src/main/resources/mapper/ProjectMapper.xml +++ b/wedpr-components/meta/project/src/main/resources/mapper/ProjectMapper.xml @@ -198,7 +198,7 @@ select - `id`, `owner`, `owner_agency`, `job_type`, `parties` + `id`, `owner`, `owner_agency`, `job_type`, `parties`, `create_time`, `last_update_time`, `status` * @@ -297,7 +297,8 @@ select - job_table.id, job_table.owner, job_table.owner_agency, job_table.job_type + job_table.id, job_table.owner, job_table.owner_agency, + job_table.job_type, job_table.create_time, job_table.last_update_time, job_table.status job_table.* @@ -307,7 +308,16 @@ (select `user_name`, `agency`, `resource_id` from `wedpr_follower_table` where `user_name` = #{followerUser} and `agency` = #{followerAgency})as follower_table left join - (select * from `wedpr_job_table` where 1 = 1 + (select + + + `id`, `owner`, `owner_agency`, `job_type`, `last_update_time`, `create_time`, `status` + + + * + + + from `wedpr_job_table` where 1 = 1 @@ -351,7 +361,7 @@ (select `user_name`, `agency`, `resource_id` from `wedpr_follower_table` where `user_name` = #{followerUser} and `agency` = #{followerAgency}) as follower_table left join - (select * from `wedpr_job_table` where 1 = 1 + (select `id`, `owner` from `wedpr_job_table` where 1 = 1 diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/SchedulerService.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/SchedulerService.java index a1f4cda3..621f0f4e 100644 --- a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/SchedulerService.java +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/SchedulerService.java @@ -15,8 +15,10 @@ package com.webank.wedpr.components.scheduler; +import com.webank.wedpr.components.scheduler.impl.JobDetailRequest; + public interface SchedulerService { // query the job detail - public abstract Object queryJobDetail(String user, String agency, String jobID) - throws Exception; + public abstract Object queryJobDetail( + String user, String agency, JobDetailRequest jobDetailRequest) throws Exception; } diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/MLExecutorClient.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/MLExecutorClient.java index b15e6108..b3dcddce 100644 --- a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/MLExecutorClient.java +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/MLExecutorClient.java @@ -24,8 +24,12 @@ import com.webank.wedpr.components.scheduler.executor.impl.ml.model.ModelJobResult; import com.webank.wedpr.components.scheduler.executor.impl.ml.request.GetTaskResultRequest; import com.webank.wedpr.sdk.jni.transport.model.ServiceMeta; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MLExecutorClient { + private static final Logger logger = LoggerFactory.getLogger(MLExecutorClient.class); + public static Object getJobResult(LoadBalancer loadBalancer, GetTaskResultRequest request) throws Exception { @@ -57,9 +61,6 @@ public static Object getJobResult(LoadBalancer loadBalancer, GetTaskResultReques if (modelJobResult.getData() == null) { return null; } - if (modelJobResult.getData().getJobPlanetResult() == null) { - return null; - } return modelJobResult.getData(); } } diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/model/ModelJobResult.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/model/ModelJobResult.java index 8a8c651a..3e796ce1 100644 --- a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/model/ModelJobResult.java +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/model/ModelJobResult.java @@ -15,6 +15,7 @@ package com.webank.wedpr.components.scheduler.executor.impl.ml.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.wedpr.common.utils.ObjectMapperFactory; import lombok.Data; @@ -22,9 +23,11 @@ @Data @NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) public class ModelJobResult { @Data @NoArgsConstructor + @JsonIgnoreProperties(ignoreUnknown = true) public static class ModelJobData { private Object jobPlanetResult; private String modelData; diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/request/GetTaskResultRequest.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/request/GetTaskResultRequest.java index 178c869a..4ad99db8 100644 --- a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/request/GetTaskResultRequest.java +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/executor/impl/ml/request/GetTaskResultRequest.java @@ -28,7 +28,8 @@ public class GetTaskResultRequest implements BaseRequest { String jobID; String user; String jobType; - Boolean onlyFetchLog; + Boolean fetchLog = Boolean.FALSE; + Boolean fetchJobResult = Boolean.FALSE; public GetTaskResultRequest() {} diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/JobDetailRequest.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/JobDetailRequest.java new file mode 100644 index 00000000..0ec04094 --- /dev/null +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/JobDetailRequest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.scheduler.impl; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +@NoArgsConstructor +public class JobDetailRequest { + private String jobID; + private Boolean fetchJobDetail = Boolean.TRUE; + private Boolean fetchJobResult = Boolean.FALSE; + private Boolean fetchLog = Boolean.FALSE; + + public void setFetchJobDetail(Boolean fetchJobDetail) { + if (fetchJobDetail == null) { + return; + } + this.fetchJobDetail = fetchJobDetail; + } + + public void setFetchJobResult(Boolean fetchJobResult) { + if (fetchJobResult == null) { + return; + } + this.fetchJobResult = fetchJobResult; + } + + public void setFetchLog(Boolean fetchLog) { + if (fetchLog == null) { + return; + } + this.fetchLog = fetchLog; + } +} diff --git a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/SchedulerServiceImpl.java b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/SchedulerServiceImpl.java index 47742c05..fe240362 100644 --- a/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/SchedulerServiceImpl.java +++ b/wedpr-components/scheduler/src/main/java/com/webank/wedpr/components/scheduler/impl/SchedulerServiceImpl.java @@ -50,46 +50,71 @@ public class SchedulerServiceImpl implements SchedulerService { private LoadBalancer loadBalancer; @Override - public Object queryJobDetail(String user, String agency, String jobID) throws Exception { - List jobDOList = this.projectMapperWrapper.queryJobDetail(jobID, user, agency); + public Object queryJobDetail(String user, String agency, JobDetailRequest jobDetailRequest) + throws Exception { + Boolean onlyMeta = Boolean.TRUE; + // query the jobDetail + if (jobDetailRequest.getFetchJobDetail()) { + onlyMeta = Boolean.FALSE; + } + List jobDOList = + this.projectMapperWrapper.queryJobDetail( + jobDetailRequest.getJobID(), onlyMeta, user, agency); if (jobDOList == null || jobDOList.isEmpty()) { - throw new WeDPRException("queryJobDetail failed for the job " + jobID + " not exist!"); + throw new WeDPRException( + "queryJobDetail failed for the job " + + jobDetailRequest.getJobID() + + " not exist!"); } - // query the jobDetail JobDO jobDO = jobDOList.get(0); + JobDetailResponse response = new JobDetailResponse(jobDO); // run failed, no need to fetch the result, only fetch the log - if (jobDO.getType().mlJob() && !JobStatus.success(jobDO.getStatus())) { + if (jobDetailRequest.getFetchLog() + && jobDO.getType().mlJob() + && !JobStatus.success(jobDO.getStatus())) { Object logDetail = null; if (jobDO.getJobStatus().finished()) { GetTaskResultRequest getTaskResultRequest = new GetTaskResultRequest(user, jobDO.getId(), jobDO.getJobType()); - getTaskResultRequest.setOnlyFetchLog(Boolean.TRUE); + getTaskResultRequest.setFetchLog(Boolean.TRUE); + getTaskResultRequest.setFetchJobResult(Boolean.FALSE); ModelJobResult.ModelJobData modelJobData = (ModelJobResult.ModelJobData) MLExecutorClient.getJobResult(loadBalancer, getTaskResultRequest); logDetail = modelJobData.getLogDetail(); } - return new JobDetailResponse(jobDO, null, null, logDetail); + response.setLog(logDetail); + return response; } // the ml job if (jobDO.getType().mlJob()) { + // no need to fetch log and fetch job result + if (!jobDetailRequest.getFetchLog() && !jobDetailRequest.getFetchJobResult()) { + return response; + } + GetTaskResultRequest getTaskResultRequest = new GetTaskResultRequest(user, jobDO.getId(), jobDO.getJobType()); + getTaskResultRequest.setFetchJobResult(jobDetailRequest.getFetchJobResult()); + getTaskResultRequest.setFetchLog(jobDetailRequest.getFetchLog()); ModelJobResult.ModelJobData modelJobData = (ModelJobResult.ModelJobData) MLExecutorClient.getJobResult(loadBalancer, getTaskResultRequest); + if (modelJobData == null) { + return new JobDetailResponse(jobDO, null, null, null); + } return new JobDetailResponse( jobDO, modelJobData.getJobPlanetResult(), modelJobData.getModelData(), modelJobData.getLogDetail()); } - JobDetailResponse response = new JobDetailResponse(jobDO); // the psi job, parse the output if (JobType.isPSIJob(jobDO.getJobType())) { PSIJobParam psiJobParam = PSIJobParam.deserialize(jobDO.getParam()); response.setResultFileInfo( - psiJobParam.getResultPath(datasetMapper, fileMetaBuilder, jobID)); + psiJobParam.getResultPath( + datasetMapper, fileMetaBuilder, jobDetailRequest.getJobID())); } // the pir job, get result files if (JobType.isPirJob(jobDO.getJobType())) { @@ -107,7 +132,9 @@ public Object queryJobDetail(String user, String agency, String jobID) throws Ex mpcJobParam.check(datasetMapper); response.setResultFileInfo( mpcJobParam.getMpcPath( - fileMetaBuilder, jobID, ExecutorConfig.getMpcResultFileName())); + fileMetaBuilder, + jobDetailRequest.getJobID(), + ExecutorConfig.getMpcResultFileName())); } return response; diff --git a/wedpr-site/src/main/java/com/webank/wedpr/site/controller/SchedulerController.java b/wedpr-site/src/main/java/com/webank/wedpr/site/controller/SchedulerController.java index 85a82ced..bdf9f913 100644 --- a/wedpr-site/src/main/java/com/webank/wedpr/site/controller/SchedulerController.java +++ b/wedpr-site/src/main/java/com/webank/wedpr/site/controller/SchedulerController.java @@ -19,15 +19,13 @@ import com.webank.wedpr.common.utils.Constant; import com.webank.wedpr.common.utils.WeDPRResponse; import com.webank.wedpr.components.scheduler.SchedulerService; +import com.webank.wedpr.components.scheduler.impl.JobDetailRequest; import com.webank.wedpr.components.token.auth.TokenUtils; import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; @RestController @RequestMapping( @@ -39,8 +37,9 @@ public class SchedulerController { @Autowired private SchedulerService schedulerService; // create the authorization request - @GetMapping("/queryJobDetail") - public WeDPRResponse queryJobDetail(@RequestParam String jobID, HttpServletRequest request) { + @PostMapping("/queryJobDetail") + public WeDPRResponse queryJobDetail( + @RequestBody JobDetailRequest jobDetailRequest, HttpServletRequest request) { try { WeDPRResponse response = new WeDPRResponse(Constant.WEDPR_SUCCESS, Constant.WEDPR_SUCCESS_MSG); @@ -49,13 +48,17 @@ public WeDPRResponse queryJobDetail(@RequestParam String jobID, HttpServletReque this.schedulerService.queryJobDetail( TokenUtils.getLoginUser(request).getUsername(), WeDPRCommonConfig.getAgency(), - jobID)); + jobDetailRequest)); return response; } catch (Exception e) { - logger.warn("queryJobDetail exception, job: {}, error: ", jobID, e); + logger.warn( + "queryJobDetail exception, job: {}, error: ", jobDetailRequest.getJobID(), e); return new WeDPRResponse( Constant.WEDPR_FAILED, - "queryJobDetail for job " + jobID + " failed for " + e.getMessage()); + "queryJobDetail for job " + + jobDetailRequest.getJobID() + + " failed for " + + e.getMessage()); } } }