Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BIGTOP-4181: Refactor factory/runner for job/stage #36

Merged
merged 8 commits into from
Aug 5, 2024
Merged

BIGTOP-4181: Refactor factory/runner for job/stage #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c20e390
refactor job factory from persist objects to domain objects
kevinw66 Aug 2, 2024
68b1f6f
fix bug
kevinw66 Aug 2, 2024
76afa7b
fix job retry function
kevinw66 Aug 4, 2024
745ec87
reorganize directories
kevinw66 Aug 4, 2024
44b1209
add license
kevinw66 Aug 4, 2024
d35aef2
fix code style
kevinw66 Aug 4, 2024
8eb7150
fix eslint
kevinw66 Aug 4, 2024
660409d
more friendly task name
kevinw66 Aug 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions ...agent/src/main/java/org/apache/bigtop/manager/agent/executor/AbstractCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,8 @@ public CommandReply execute(CommandRequest request) {
log.error("Run command failed, {}", request, e);
}

commandReplyBuilder.setType(request.getType());
commandReplyBuilder.setHostname(request.getHostname());
commandReplyBuilder.setTaskId(request.getTaskId());
commandReplyBuilder.setStageId(request.getStageId());
commandReplyBuilder.setJobId(request.getJobId());
return commandReplyBuilder.build();
}

Expand Down
7 changes: 4 additions & 3 deletions ...cutor/CacheDistributeCommandExecutor.java → ...cutor/CacheFileUpdateCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

Expand All @@ -41,13 +42,13 @@
import static org.apache.bigtop.manager.common.constants.CacheFiles.USERS_INFO;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class CacheDistributeCommandExecutor extends AbstractCommandExecutor {
public class CacheFileUpdateCommandExecutor extends AbstractCommandExecutor {

@Override
public CommandType getCommandType() {
return CommandType.CACHE_DISTRIBUTE;
return CommandType.UPDATE_CACHE_FILES;
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion ...gent/src/main/java/org/apache/bigtop/manager/agent/executor/ComponentCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentCommandExecutor extends AbstractCommandExecutor {

Expand Down
14 changes: 4 additions & 10 deletions bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/TaskPO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@ public class TaskPO extends BasePO {
@Column(name = "name")
private String name;

@Column(name = "message_id")
private String messageId;
@Lob
@Basic(fetch = FetchType.LAZY)
@Column(name = "context", length = 16777216)
private String context;

@Column(name = "state")
private JobState state;
Expand All @@ -79,14 +81,6 @@ public class TaskPO extends BasePO {
@Column(name = "custom_command")
private String customCommand;

@Lob
@Basic(fetch = FetchType.LAZY)
@Column(name = "custom_commands", length = 16777216)
private String customCommands;

@Column(name = "command_script")
private String commandScript;

@Column(name = "hostname")
private String hostname;

Expand Down
2 changes: 1 addition & 1 deletion bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/utils/ProtobufUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public static <T extends MessageOrBuilder> T fromJson(String json, Class<T> claz

public static <T extends Message> String toJson(T message) {
try {
return JsonFormat.printer().print(message);
return JsonFormat.printer().omittingInsignificantWhitespace().print(message);
} catch (Exception e) {
throw new RuntimeException(e);
}
Expand Down
13 changes: 4 additions & 9 deletions bigtop-manager-grpc/src/main/resources/proto/command.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,19 @@ service CommandService {
enum CommandType {
COMPONENT = 0;
HOST_CHECK = 1;
CACHE_DISTRIBUTE = 2;
UPDATE_CACHE_FILES = 2;
}

message CommandRequest {
string payload = 1;
string hostname = 2;
int64 job_id = 3;
int64 stage_id = 4;
int64 task_id = 5;
CommandType type = 6;
int64 task_id = 3;
CommandType type = 4;
}

message CommandReply {
int32 code = 1;
string result = 2;
string hostname = 3;
int64 job_id = 4;
int64 stage_id = 5;
int64 task_id = 6;
CommandType type = 7;
int64 task_id = 4;
}
8 changes: 2 additions & 6 deletions ...manager/server/enums/JobStrategyType.java → ...r/command/factory/AbstractJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.enums;
package org.apache.bigtop.manager.server.command.factory;

public enum JobStrategyType {
OVER_ON_FAIL,

CONTINUE_ON_FAIL,
}
public abstract class AbstractJobFactory implements JobFactory {}
2 changes: 1 addition & 1 deletion ...ver/command/job/factory/JobFactories.java → .../server/command/factory/JobFactories.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory;
package org.apache.bigtop.manager.server.command.factory;

import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.enums.ApiExceptionEnum;
Expand Down
7 changes: 4 additions & 3 deletions ...erver/command/job/factory/JobFactory.java → ...er/server/command/factory/JobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory;
package org.apache.bigtop.manager.server.command.factory;

import org.apache.bigtop.manager.dao.po.JobPO;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;

public interface JobFactory {

CommandIdentifier getCommandIdentifier();

JobPO createJob(JobContext jobContext);
Job createJob(JobContext jobContext);
}
4 changes: 2 additions & 2 deletions ...ry/cluster/AbstractClusterJobFactory.java → ...ry/cluster/AbstractClusterJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.cluster;
package org.apache.bigtop.manager.server.command.factory.cluster;

import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory;
import org.apache.bigtop.manager.server.command.factory.AbstractJobFactory;

public abstract class AbstractClusterJobFactory extends AbstractJobFactory {}
18 changes: 13 additions & 5 deletions .../component/ComponentRestartJobRunner.java → ...tory/cluster/ClusterCreateJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,33 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.runner.component;
package org.apache.bigtop.manager.server.command.factory.cluster;

import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner;
import org.apache.bigtop.manager.server.command.job.ClusterCreateJob;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;
import org.apache.bigtop.manager.server.enums.CommandLevel;

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentRestartJobRunner extends AbstractJobRunner {
public class ClusterCreateJobFactory extends AbstractClusterJobFactory {

@Override
public CommandIdentifier getCommandIdentifier() {
return new CommandIdentifier(CommandLevel.COMPONENT, Command.RESTART);
return new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE);
}

@Override
public Job createJob(JobContext jobContext) {
return new ClusterCreateJob(jobContext);
}
}
4 changes: 2 additions & 2 deletions ...omponent/AbstractComponentJobFactory.java → ...omponent/AbstractComponentJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.component;
package org.apache.bigtop.manager.server.command.factory.component;

import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory;
import org.apache.bigtop.manager.server.command.factory.AbstractJobFactory;

public abstract class AbstractComponentJobFactory extends AbstractJobFactory {}
11 changes: 8 additions & 3 deletions ...component/ComponentInstallJobFactory.java → ...component/ComponentInstallJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.component;
package org.apache.bigtop.manager.server.command.factory.component;

import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;
import org.apache.bigtop.manager.server.enums.CommandLevel;

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentInstallJobFactory extends AbstractComponentJobFactory {

Expand All @@ -38,5 +41,7 @@ public CommandIdentifier getCommandIdentifier() {
}

@Override
public void createStagesAndTasks() {}
public Job createJob(JobContext jobContext) {
return null;
}
}
11 changes: 8 additions & 3 deletions ...component/ComponentRestartJobFactory.java → ...component/ComponentRestartJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.component;
package org.apache.bigtop.manager.server.command.factory.component;

import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;
import org.apache.bigtop.manager.server.enums.CommandLevel;

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentRestartJobFactory extends AbstractComponentJobFactory {

Expand All @@ -38,5 +41,7 @@ public CommandIdentifier getCommandIdentifier() {
}

@Override
public void createStagesAndTasks() {}
public Job createJob(JobContext jobContext) {
return null;
}
}
11 changes: 8 additions & 3 deletions ...y/component/ComponentStartJobFactory.java → ...y/component/ComponentStartJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.component;
package org.apache.bigtop.manager.server.command.factory.component;

import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;
import org.apache.bigtop.manager.server.enums.CommandLevel;

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentStartJobFactory extends AbstractComponentJobFactory {

Expand All @@ -38,5 +41,7 @@ public CommandIdentifier getCommandIdentifier() {
}

@Override
public void createStagesAndTasks() {}
public Job createJob(JobContext jobContext) {
return null;
}
}
11 changes: 8 additions & 3 deletions ...ry/component/ComponentStopJobFactory.java → ...ry/component/ComponentStopJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.component;
package org.apache.bigtop.manager.server.command.factory.component;

import org.apache.bigtop.manager.common.enums.Command;
import org.apache.bigtop.manager.server.command.CommandIdentifier;
import org.apache.bigtop.manager.server.command.job.Job;
import org.apache.bigtop.manager.server.command.job.JobContext;
import org.apache.bigtop.manager.server.enums.CommandLevel;

import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;

@Slf4j
@org.springframework.stereotype.Component
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class ComponentStopJobFactory extends AbstractComponentJobFactory {

Expand All @@ -38,5 +41,7 @@ public CommandIdentifier getCommandIdentifier() {
}

@Override
public void createStagesAndTasks() {}
public Job createJob(JobContext jobContext) {
return null;
}
}
4 changes: 2 additions & 2 deletions .../factory/host/AbstractHostJobFactory.java → .../factory/host/AbstractHostJobFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.bigtop.manager.server.command.job.factory.host;
package org.apache.bigtop.manager.server.command.factory.host;

import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory;
import org.apache.bigtop.manager.server.command.factory.AbstractJobFactory;

public abstract class AbstractHostJobFactory extends AbstractJobFactory {}
Loading
Loading