Skip to content

Commit

Permalink
Reduce database round-trips during BOM processing
Browse files Browse the repository at this point in the history 
In the previous implementation, a `SELECT` query was issued for every single component and service in a BOM, in order to find existing components that match their identity.

In retrospect, this causes a lot of unnecessary database round-trips and puts the database under unnecessary stress, in particular for new projects where no components and services exist yet.

Now, we query all existing components and services of the project once in bulk.

A situation where this approach can perform worse, is when a BOM is uploaded to an existing project, and the content differs wildly between BOM and project. We would then load many components into memory, only to delete them shortly after. However, this scenario should be less common. Usually, projects are either empty, or have significant overlap with the uploaded BOM.

Backports DependencyTrack/hyades-apiserver#1006

Signed-off-by: nscuro <[email protected]>
  • Loading branch information
@nscuro
nscuro committed Dec 21, 2024
1 parent ee5cbce commit f886f41
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 219 deletions.
14 changes: 14 additions & 0 deletions src/main/java/org/dependencytrack/model/ComponentIdentity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ public ComponentIdentity(final Component component) {
this.objectType = ObjectType.COMPONENT;
}

public ComponentIdentity(final Component component, final boolean excludeUuid) {
this(component);
if (excludeUuid) {
this.uuid = null;
}
}

public ComponentIdentity(final org.cyclonedx.model.Component component) {
try {
this.purl = new PackageURL(component.getPurl());
Expand All @@ -95,6 +102,13 @@ public ComponentIdentity(final ServiceComponent service) {
this.objectType = ObjectType.SERVICE;
}

public ComponentIdentity(final ServiceComponent service, final boolean excludeUuid) {
this(service);
if (excludeUuid) {
this.uuid = null;
}
}

public ComponentIdentity(final org.cyclonedx.model.Service service) {
this.group = service.getGroup();
this.name = service.getName();
Expand Down
130 changes: 0 additions & 130 deletions src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,55 +460,6 @@ public void recursivelyDelete(Component component, boolean commitIndex) {

}

/**
* Returns a component by matching its identity information.
* <p>
* Note that this method employs a stricter matching logic than {@link #matchIdentity(ComponentIdentity)}.
* For example, if {@code purl} of the given {@link ComponentIdentity} is {@code null},
* this method will use a query that explicitly checks for the {@code purl} column to be {@code null}.
* Whereas other methods will simply not include {@code purl} in the query in such cases.
*
* @param project the Project the component is a dependency of
* @param cid the identity values of the component
* @return a Component object, or null if not found
* @since 4.11.0
*/
public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
query.setNamedParameters(queryFilterParamsPair.getRight());
try {
return query.executeUnique();
} finally {
query.closeAll();
}
}

/**
* Returns the first component matching a given {@link ComponentIdentity} in a {@link Project}.
*
* @param project the Project the component is a dependency of
* @param cid the identity values of the component
* @return a Component object, or null if not found
* @since 4.11.0
*/
public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
query.setNamedParameters(queryFilterParamsPair.getRight());
query.setRange(0, 1);
try {
final List<Component> result = query.executeList();
if (result.isEmpty()) {
return null;
}

return result.get(0);
} finally {
query.closeAll();
}
}

/**
* Returns a list of components by matching its identity information.
* @param project the Project the component is a dependency of
Expand Down Expand Up @@ -597,87 +548,6 @@ private static Pair<String, Map<String, Object>> buildComponentIdentityQuery(fin
return Pair.of(filter, params);
}

private static Pair<String, Map<String, Object>> buildExactComponentIdentityQuery(final Project project, final ComponentIdentity cid) {
var filterParts = new ArrayList<String>();
final var params = new HashMap<String, Object>();

if (cid.getPurl() != null) {
filterParts.add("(purl != null && purl == :purl)");
params.put("purl", cid.getPurl().canonicalize());
} else {
filterParts.add("purl == null");
}

if (cid.getCpe() != null) {
filterParts.add("(cpe != null && cpe == :cpe)");
params.put("cpe", cid.getCpe());
} else {
filterParts.add("cpe == null");
}

if (cid.getSwidTagId() != null) {
filterParts.add("(swidTagId != null && swidTagId == :swidTagId)");
params.put("swidTagId", cid.getSwidTagId());
} else {
filterParts.add("swidTagId == null");
}

var coordinatesFilter = "(";
if (cid.getGroup() != null) {
coordinatesFilter += "group == :group";
params.put("group", cid.getGroup());
} else {
coordinatesFilter += "group == null";
}
coordinatesFilter += " && name == :name";
params.put("name", cid.getName());
if (cid.getVersion() != null) {
coordinatesFilter += " && version == :version";
params.put("version", cid.getVersion());
} else {
coordinatesFilter += " && version == null";
}
coordinatesFilter += ")";
filterParts.add(coordinatesFilter);

final var filter = "project == :project && (" + String.join(" && ", filterParts) + ")";
params.put("project", project);

return Pair.of(filter, params);
}

/**
* Intelligently adds dependencies for components that are not already a dependency
* of the specified project and removes the dependency relationship for components
* that are not in the list of specified components.
* @param project the project to bind components to
* @param existingProjectComponents the complete list of existing dependent components
* @param components the complete list of components that should be dependencies of the project
*/
public void reconcileComponents(Project project, List<Component> existingProjectComponents, List<Component> components) {
// Removes components as dependencies to the project for all
// components not included in the list provided
List<Component> markedForDeletion = new ArrayList<>();
for (final Component existingComponent: existingProjectComponents) {
boolean keep = false;
for (final Component component: components) {
if (component.getId() == existingComponent.getId()) {
keep = true;
break;
}
}
if (!keep) {
markedForDeletion.add(existingComponent);
}
}
if (!markedForDeletion.isEmpty()) {
for (Component c: markedForDeletion) {
this.recursivelyDelete(c, false);
}
//this.delete(markedForDeletion);
}
}

public Map<String, Component> getDependencyGraphForComponents(Project project, List<Component> components) {
Map<String, Component> dependencyGraph = new HashMap<>();
if (project.getDirectDependencies() == null || project.getDirectDependencies().isBlank()) {
Expand Down
16 changes: 0 additions & 16 deletions src/main/java/org/dependencytrack/persistence/QueryManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -959,14 +959,6 @@ public List<VulnerableSoftware> getAllVulnerableSoftware(final String cpePart, f
return getVulnerableSoftwareQueryManager().getAllVulnerableSoftware(cpePart, cpeVendor, cpeProduct, purl);
}

public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchSingleIdentityExact(project, cid);
}

public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchFirstIdentityExact(project, cid);
}

public List<Component> matchIdentity(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchIdentity(project, cid);
}
Expand All @@ -975,10 +967,6 @@ public List<Component> matchIdentity(final ComponentIdentity cid) {
return getComponentQueryManager().matchIdentity(cid);
}

public void reconcileComponents(Project project, List<Component> existingProjectComponents, List<Component> components) {
getComponentQueryManager().reconcileComponents(project, existingProjectComponents, components);
}

public List<Component> getAllComponents(Project project) {
return getComponentQueryManager().getAllComponents(project);
}
Expand All @@ -999,10 +987,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon
return getServiceComponentQueryManager().matchServiceIdentity(project, cid);
}

public void reconcileServiceComponents(Project project, List<ServiceComponent> existingProjectServices, List<ServiceComponent> services) {
getServiceComponentQueryManager().reconcileServiceComponents(project, existingProjectServices, services);
}

public ServiceComponent createServiceComponent(ServiceComponent service, boolean commitIndex) {
return getServiceComponentQueryManager().createServiceComponent(service, commitIndex);
}
Expand Down
33 changes: 0 additions & 33 deletions src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import javax.jdo.FetchPlan;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

Expand Down Expand Up @@ -65,38 +64,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon
return singleResult(query.executeWithArray(project, cid.getGroup(), cid.getName(), cid.getVersion()));
}

/**
* Intelligently adds service components that are not already a dependency
* of the specified project and removes the dependency relationship for service components
* that are not in the list of specified components.
* @param project the project to bind components to
* @param existingProjectServices the complete list of existing dependent service components
* @param services the complete list of service components that should be dependencies of the project
*/
public void reconcileServiceComponents(Project project, List<ServiceComponent> existingProjectServices, List<ServiceComponent> services) {
// Removes components as dependencies to the project for all
// components not included in the list provided
List<ServiceComponent> markedForDeletion = new ArrayList<>();
for (final ServiceComponent existingService: existingProjectServices) {
boolean keep = false;
for (final ServiceComponent service: services) {
if (service.getId() == existingService.getId()) {
keep = true;
break;
}
}
if (!keep) {
markedForDeletion.add(existingService);
}
}
if (!markedForDeletion.isEmpty()) {
for (ServiceComponent sc: markedForDeletion) {
this.recursivelyDelete(sc, false);
}
//this.delete(markedForDeletion);
}
}

/**
* Creates a new ServiceComponent.
* @param service the ServiceComponent to persist
Expand Down
Loading

0 comments on commit f886f41

Please sign in to comment.