Skip to content

Commit

Permalink
Changes for Performance/Fix for logs spam (#99)
Browse files Browse the repository at this point in the history 
* Changes for Performance/Fix for logs spam
* Commented unused methods

Co-authored-by: Madalin Tiutiu <[email protected]>
  • Loading branch information
@TiutiuMadalin
TiutiuMadalin and Madalin Tiutiu authored Sep 26, 2023
1 parent ff9cb51 commit 5f3f0db
Show file tree
Hide file tree
Showing 8 changed files with 49 additions and 46 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

<groupId>com.microfocus.octane.ciplugins</groupId>
<artifactId>octane-gitlab-service</artifactId>
<version>23.3.0</version>
<version>23.3.1</version>

<name>ALM Octane GitLab CI Service</name>
<description>
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/microfocus/octane/gitlab/api/EventListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ private void sendCodeCoverage(long projectId, Project project, Job job) throws G
Optional<Variable> coverageReportFilePathVar = VariablesHelper.getProjectVariable(gitLabApi, project.getId(),
applicationSettings.getConfig().getGeneratedCoverageReportFilePathVariableName());

Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project);
Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project, applicationSettings.getConfig());

if (coverageReportFilePathVar.isEmpty() && !projectGroupVariables.containsKey(
applicationSettings.getConfig().getGeneratedCoverageReportFilePathVariableName())) {
Expand Down Expand Up @@ -335,7 +335,7 @@ private Response handleMergeRequestEvent(JSONObject event) throws GitLabApiExcep
}

Project project = gitLabApi.getProjectApi().getProject(event.getJSONObject("project").getLong("id"));
Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project);
Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project, applicationSettings.getConfig());

Optional<Variable> publishMergeRequests = VariablesHelper.getProjectVariable(gitLabApi, project.getId(),
config.getPublishMergeRequestsVariableName());
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/microfocus/octane/gitlab/api/MergeRequestHistoryHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public void executeFirstScan() {
List<Project> gitLabProjects = gitLabApi.getProjectApi().getProjects(filter).stream()
.filter(project -> {
Map<String, String> projectGroupVariables =
VariablesHelper.getProjectGroupVariables(gitLabApi, project);
VariablesHelper.getProjectGroupVariables(gitLabApi, project, applicationSettings.getConfig());

Optional<Variable> shouldPublishToOctane =
VariablesHelper.getProjectVariable(gitLabApi, project.getId(),
Expand Down Expand Up @@ -174,7 +174,7 @@ public void startListening() {
private void sendMergeRequestsToOctane(Project project) throws GitLabApiException {
log.info("Sending merge request history for project with id " + project.getId() + " to Octane.");
List<MergeRequest> mergeRequests = gitLabApi.getMergeRequestApi().getMergeRequests(project.getId());
Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project);
Map<String, String> projectGroupVariables = VariablesHelper.getProjectGroupVariables(gitLabApi, project, applicationSettings.getConfig());

Optional<Variable> destinationWSVar =
VariablesHelper.getProjectVariable(gitLabApi, project.getId(),
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/microfocus/octane/gitlab/helpers/GitLabApiWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public GitLabApi getGitLabApi() {
return gitLabApi;
}

public boolean isUserHasPermissionForProject(Project project,User currentUser) {
/* public boolean isUserHasPermissionForProject(Project project,User currentUser) {
try {
Optional<Member> currentMember = gitLabApi.getProjectApi().getAllMembers(project.getId())
.stream().filter(member -> member.getId().equals(currentUser.getId())).findFirst();
Expand All @@ -128,6 +128,6 @@ public boolean isUserHasPermissionForProject(Project project,User currentUser) {
log.error("failed to get user permissions" + e.getMessage());
}
return false;
}
}*/

}
6 changes: 2 additions & 4 deletions src/main/java/com/microfocus/octane/gitlab/helpers/HooksHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,9 @@ public static Boolean addWebHookToProject(GitLabApi gitLabApi,URL webhookURL, Ob
}


public static void deleteWebHooks(List<Project> projects,URL webhookURL,GitLabApiWrapper gitLabApiWrapper,GitLabApi gitLabApi, User currentUser) throws GitLabApiException {
public static void deleteWebHooks(List<Project> projects,URL webhookURL,GitLabApi gitLabApi) throws GitLabApiException {
for (Project project : projects) {
if (gitLabApiWrapper.isUserHasPermissionForProject(project, currentUser)) {
HooksHelper.deleteWebHooks(gitLabApi,webhookURL,project.getId());
}
HooksHelper.deleteWebHooks(gitLabApi,webhookURL,project.getId());
}
}
public static void deleteWebHooks(GitLabApi gitLabApi,URL webhookURL, Object projectIdOrPath) throws GitLabApiException {
Expand Down
43 changes: 25 additions & 18 deletions src/main/java/com/microfocus/octane/gitlab/helpers/VariablesHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,15 @@ public static List<Variable> getVariables(ParsedPath project, GitLabApi gitLabAp
for(String group :groupsFullPath){
try {
List<Variable> variablesOnGroup = gitLabApi.getGroupApi().getVariables(group);
if(variablesOnGroup.isEmpty()){
if (log.isDebugEnabled()) {
log.warn("can not find variables for the group:" + group);
}
}
variables.addAll(variablesOnGroup);
}catch (GitLabApiException e){
if("root".equalsIgnoreCase(group)){
if(log.isDebugEnabled()){log.warn("can not find variables for the group:" + group);}
}else {
log.error("can not find variables for the group:" + group);
if (log.isDebugEnabled()) {
log.warn("can not find variables for the group:" + group);
}
}
}
Expand Down Expand Up @@ -156,23 +159,27 @@ public static Optional<Variable> getProjectVariable(GitLabApi gitLabApi, long pr
return Optional.ofNullable(variable);
}

public static Map<String, String> getProjectGroupVariables(GitLabApi gitLabApi, Project project) {
public static Map<String, String> getProjectGroupVariables(GitLabApi gitLabApi, Project project, ConfigStructure appConfig) {
Map<String, String> variablesKeyValuePairs = new HashMap<>();

List<String> groupsFullPath = ParsedPath.getGroupFullPathFromProject(project.getPathWithNamespace());
groupsFullPath.forEach(group -> {
try {
List<Variable> variablesOnGroup = gitLabApi.getGroupApi().getVariables(group);
variablesOnGroup.forEach(variable -> variablesKeyValuePairs.put(variable.getKey(), variable.getValue()));
} catch (GitLabApiException e) {
if("root".equalsIgnoreCase(group) ){
if (log.isDebugEnabled()) {log.warn("can not find variables for the group:" + group);}
}else {
log.error("can not find variables for the group:" + group);
if(appConfig.getGitlabVariablesPipelineUsage().contains(VARS_ON_GROUPS)) {
List<String> groupsFullPath = ParsedPath.getGroupFullPathFromProject(project.getPathWithNamespace());
groupsFullPath.forEach(group -> {
try {
List<Variable> variablesOnGroup = gitLabApi.getGroupApi().getVariables(group);
if(variablesOnGroup.isEmpty()){
if (log.isDebugEnabled()) {
log.warn("can not find variables for the group:" + group);
}
}
variablesOnGroup.forEach(variable -> variablesKeyValuePairs.put(variable.getKey(), variable.getValue()));
} catch (GitLabApiException e) {
if (log.isDebugEnabled()) {
log.warn("can not find variables for the group:" + group);
}
}
}
});

});
}
return variablesKeyValuePairs;
}
}
22 changes: 13 additions & 9 deletions src/main/java/com/microfocus/octane/gitlab/services/GitlabServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,19 +106,21 @@ private void init() throws MalformedURLException {
gitLabApi = gitLabApiWrapper.getGitLabApi();

try {
List<Project> projects = isCurrentUserAdmin() ? gitLabApi.getProjectApi().getProjects() : gitLabApi.getProjectApi().getMemberProjects();
User currentUser = gitLabApi.getUserApi().getCurrentUser();
ProjectFilter filter = new ProjectFilter().withMembership(true)
.withMinAccessLevel(AccessLevel.MAINTAINER);

List<Project> projects = gitLabApi.getProjectApi().getProjects(filter);

if(cleanupOnly){
log.info("start with cleanup process");
HooksHelper.deleteWebHooks(projects,webhookURL,gitLabApiWrapper,gitLabApi,currentUser);
HooksHelper.deleteWebHooks(projects,webhookURL,gitLabApi);
}else {

//start hooks' update thread
updateHooksExecutor =
Executors.newSingleThreadScheduledExecutor();
updateHooksScheduledFuture = updateHooksExecutor.scheduleAtFixedRate(
new HooksUpdateRunnable(gitLabApiWrapper,webhookURL,currentUser),0 , HooksUpdateRunnable.INTERVAL, TimeUnit.MINUTES);
new HooksUpdateRunnable(gitLabApiWrapper,webhookURL),0 , HooksUpdateRunnable.INTERVAL, TimeUnit.MINUTES);

}
} catch (GitLabApiException e) {
Expand Down Expand Up @@ -150,9 +152,11 @@ private void stop() {

log.info("Destroying GitLab webhooks ...");

List<Project> projects = isCurrentUserAdmin() ? gitLabApi.getProjectApi().getProjects() : gitLabApi.getProjectApi().getMemberProjects();
User currentUser = gitLabApi.getUserApi().getCurrentUser();
HooksHelper.deleteWebHooks(projects,webhookURL,gitLabApiWrapper,gitLabApi,currentUser);
ProjectFilter filter = new ProjectFilter().withMembership(true)
.withMinAccessLevel(AccessLevel.MAINTAINER);

List<Project> projects = gitLabApi.getProjectApi().getProjects(filter);
HooksHelper.deleteWebHooks(projects,webhookURL,gitLabApi);

} catch (Exception e) {
log.warn("Failed to destroy GitLab webhooks", e);
Expand Down Expand Up @@ -245,9 +249,9 @@ CIJobsList getJobList(boolean includeParameters, Long workspaceId) {
return ciJobsList;
}

private Boolean isCurrentUserAdmin() throws GitLabApiException {
/*private Boolean isCurrentUserAdmin() throws GitLabApiException {
return gitLabApi.getUserApi().getCurrentUser().getIsAdmin() != null && gitLabApi.getUserApi().getCurrentUser().getIsAdmin();
}
}*/

PipelineNode createStructure(String buildId, boolean isMultiBranchParent) {

Expand Down
10 changes: 2 additions & 8 deletions src/main/java/com/microfocus/octane/gitlab/testresults/HooksUpdateRunnable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,21 +47,17 @@
public class HooksUpdateRunnable implements Runnable {

static public final int INTERVAL = 60;
private final GitLabApiWrapper gitLabApiWrapper;
private final User currentUser;
GitLabApi gitLabApi;
Date lastUpdateTime;
private URL webhookURL;
private long lastUpdatedProjectId = 0;
static final Logger log = LogManager.getLogger(HooksUpdateRunnable.class);

public HooksUpdateRunnable(GitLabApiWrapper gitLabApiWrapper, URL webhookURL, User currentUser) {
public HooksUpdateRunnable(GitLabApiWrapper gitLabApiWrapper, URL webhookURL) {

this.gitLabApi = gitLabApiWrapper.getGitLabApi();
this.gitLabApiWrapper = gitLabApiWrapper;
this.lastUpdateTime = new Date(System.currentTimeMillis());
this.webhookURL = webhookURL;
this.currentUser = currentUser;
}

/*
Expand All @@ -81,9 +77,7 @@ public void addHooksToNewProjects() {
if (projects.size() > 0) {
projects.stream().forEach(project -> {
try {
if (gitLabApiWrapper.isUserHasPermissionForProject(project, currentUser)) {
HooksHelper.addWebHookToProject(gitLabApi, webhookURL, project.getId(), true);
}
HooksHelper.addWebHookToProject(gitLabApi, webhookURL, project.getId(), true);
} catch (GitLabApiException e) {
log.warn("Failed to create GitLab web hooks", e);
}
Expand Down

0 comments on commit 5f3f0db

Please sign in to comment.