diff --git a/docs/_posts/2024-xx-xx-v4.11.0.md b/docs/_posts/2024-xx-xx-v4.11.0.md index 1ca4d09bc8..16e011a0cf 100644 --- a/docs/_posts/2024-xx-xx-v4.11.0.md +++ b/docs/_posts/2024-xx-xx-v4.11.0.md @@ -39,6 +39,7 @@ to `true`. Users are highly encouraged to do so. * Align retry configuration and behavior across analyzers - [apiserver/#3494] * Add auto-generated changelog to GitHub releases - [apiserver/#3502] * Bump SPDX license list to v3.23 - [apiserver/#3508] +* Improve observability of Lucene search indexes - [apiserver/#3535] * Show component count in projects list - [frontend/#683] * Add current *fail*, *warn*, and *info* values to bottom of policy violation metrics - [frontend/#707] * Remove unused policy violation widget - [frontend/#710] @@ -162,6 +163,7 @@ Special thanks to everyone who contributed code to implement enhancements and fi [apiserver/#3511]: https://github.com/DependencyTrack/dependency-track/pull/3511 [apiserver/#3512]: https://github.com/DependencyTrack/dependency-track/pull/3512 [apiserver/#3513]: https://github.com/DependencyTrack/dependency-track/pull/3513 +[apiserver/#3535]: https://github.com/DependencyTrack/dependency-track/pull/3535 [frontend/#682]: https://github.com/DependencyTrack/frontend/pull/682 [frontend/#683]: https://github.com/DependencyTrack/frontend/pull/683 diff --git a/src/main/java/org/dependencytrack/search/ComponentIndexer.java b/src/main/java/org/dependencytrack/search/ComponentIndexer.java index 512268f313..d3bab097f4 100644 --- a/src/main/java/org/dependencytrack/search/ComponentIndexer.java +++ b/src/main/java/org/dependencytrack/search/ComponentIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.Component; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.ComponentDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -50,7 +43,7 @@ public final class ComponentIndexer extends IndexManager implements ObjectIndexe private static final Logger LOGGER = Logger.getLogger(ComponentIndexer.class); private static final ComponentIndexer INSTANCE = new ComponentIndexer(); - protected static ComponentIndexer getInstance() { + static ComponentIndexer getInstance() { return INSTANCE; } @@ -72,28 +65,15 @@ public String[] getSearchFields() { * @param component A persisted Component object. */ public void add(final ComponentDocument component) { - final Document doc = new Document(); - addField(doc, IndexConstants.COMPONENT_UUID, component.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.COMPONENT_NAME, component.name(), Field.Store.YES, true); - addField(doc, IndexConstants.COMPONENT_GROUP, component.group(), Field.Store.YES, true); - addField(doc, IndexConstants.COMPONENT_VERSION, component.version(), Field.Store.YES, false); - addField(doc, IndexConstants.COMPONENT_SHA1, component.sha1(), Field.Store.YES, true); - addField(doc, IndexConstants.COMPONENT_DESCRIPTION, component.description(), Field.Store.YES, true); + final Document doc = convertToDocument(component); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding component to index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.COMPONENT_INDEXER) - .content("An error occurred while adding component to index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(ComponentDocument component) { + final Term term = convertToTerm(component); + final Document doc = convertToDocument(component); + updateDocument(term, doc); } /** @@ -102,20 +82,8 @@ public void add(final ComponentDocument component) { * @param component A persisted Component object. */ public void remove(final ComponentDocument component) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.COMPONENT_UUID, component.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a component from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.COMPONENT_INDEXER) - .content("An error occurred while removing a component from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(component); + deleteDocuments(term); } /** @@ -163,4 +131,19 @@ private static List fetchNext(final QueryManager qm, final Lo } } + private Document convertToDocument(final ComponentDocument component) { + final var doc = new Document(); + addField(doc, IndexConstants.COMPONENT_UUID, component.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.COMPONENT_NAME, component.name(), Field.Store.YES, true); + addField(doc, IndexConstants.COMPONENT_GROUP, component.group(), Field.Store.YES, true); + addField(doc, IndexConstants.COMPONENT_VERSION, component.version(), Field.Store.YES, false); + addField(doc, IndexConstants.COMPONENT_SHA1, component.sha1(), Field.Store.YES, true); + addField(doc, IndexConstants.COMPONENT_DESCRIPTION, component.description(), Field.Store.YES, true); + return doc; + } + + private static Term convertToTerm(final ComponentDocument component) { + return new Term(IndexConstants.COMPONENT_UUID, component.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/search/IndexManager.java b/src/main/java/org/dependencytrack/search/IndexManager.java index 98e4444638..ac73b0480b 100644 --- a/src/main/java/org/dependencytrack/search/IndexManager.java +++ b/src/main/java/org/dependencytrack/search/IndexManager.java @@ -20,10 +20,16 @@ import alpine.Config; import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; import alpine.event.framework.Event; import alpine.model.ConfigProperty; import alpine.notification.Notification; import alpine.notification.NotificationLevel; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.BaseUnits; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileDeleteStrategy; import org.apache.commons.io.output.NullPrintStream; @@ -81,12 +87,23 @@ public abstract class IndexManager implements AutoCloseable { private static final Logger LOGGER = Logger.getLogger(IndexManager.class); + + private final Logger logger; + private final IndexType indexType; + private final Tag indexTag; + private final Counter addOperationCounter; + private final Counter updateOperationCounter; + private final Counter deleteOperationCounter; + private final Counter commitOperationCounter; private IndexWriter iwriter; private DirectoryReader searchReader; - private final IndexType indexType; + private Gauge docsRamTotalGauge; + private Gauge ramBytesUsedGauge; + private Gauge numDocsGauge; /** * This methods should be overwritten. + * * @return an array of all fields that can be searched on * @since 3.0.0 */ @@ -96,6 +113,7 @@ public String[] getSearchFields() { /** * Defines the type of supported indexes. + * * @since 3.0.0 */ public enum IndexType { @@ -126,7 +144,7 @@ public static Optional getIndexType(String type) { } } - public static UUID getUuid(Class clazz) { + public static UUID getUuid(Class clazz) { return Arrays.stream(values()) .filter(type -> clazz == type.getClazz()) .map(type -> type.uuid) @@ -138,15 +156,40 @@ public static UUID getUuid(Class clazz) { /** * Constructs a new IndexManager. All classes that extend this class should call * super(indexType) in their constructor. + * * @param indexType the type of index to use * @since 3.0.0 */ protected IndexManager(final IndexType indexType) { + this.logger = Logger.getLogger(getClass()); this.indexType = indexType; + + this.indexTag = Tag.of("index", indexType.name()); + this.addOperationCounter = Counter.builder("search_index_operations") + .description("Total number of index operations") + .tags(Tags.of(indexTag, Tag.of("operation", "add"))) + .baseUnit(BaseUnits.OPERATIONS) + .register(Metrics.getRegistry()); + this.updateOperationCounter = Counter.builder("search_index_operations") + .description("Total number of index operations") + .tags(Tags.of(indexTag, Tag.of("operation", "update"))) + .baseUnit(BaseUnits.OPERATIONS) + .register(Metrics.getRegistry()); + this.deleteOperationCounter = Counter.builder("search_index_operations") + .description("Total number of index operations") + .tags(Tags.of(indexTag, Tag.of("operation", "delete"))) + .baseUnit(BaseUnits.OPERATIONS) + .register(Metrics.getRegistry()); + this.commitOperationCounter = Counter.builder("search_index_operations") + .description("Total number of index operations") + .tags(Tags.of(indexTag, Tag.of("operation", "commit"))) + .baseUnit(BaseUnits.OPERATIONS) + .register(Metrics.getRegistry()); } /** * Returns the index type. + * * @return the index type * @since 3.0.0 */ @@ -156,6 +199,7 @@ public IndexType getIndexType() { /** * Retrieves the index directory based on the type of index used. + * * @return a Directory * @throws IOException when the directory cannot be accessed * @since 3.0.0 @@ -164,7 +208,7 @@ private synchronized Directory getDirectory() throws IOException { final File indexDir = getIndexDirectory(indexType); if (!indexDir.exists()) { if (!indexDir.mkdirs()) { - LOGGER.error("Unable to create index directory: " + indexDir.getCanonicalPath()); + logger.error("Unable to create index directory: " + indexDir.getCanonicalPath()); Notification.dispatch(new Notification() .scope(NotificationScope.SYSTEM) .group(NotificationGroup.FILE_SYSTEM) @@ -179,18 +223,42 @@ private synchronized Directory getDirectory() throws IOException { /** * Opens the index. + * * @throws IOException when the index cannot be opened * @since 3.0.0 */ protected void openIndex() throws IOException { - final Analyzer analyzer = new StandardAnalyzer(); - final IndexWriterConfig config = new IndexWriterConfig(analyzer); + final var analyzer = new StandardAnalyzer(); + + final var config = new IndexWriterConfig(analyzer); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + config.setInfoStream(new LoggingInfoStream(getClass())); + config.setCommitOnClose(true); + iwriter = new IndexWriter(getDirectory(), config); + + docsRamTotalGauge = Gauge.builder("search_index_docs_ram_total", iwriter, IndexWriter::numRamDocs) + .description("Number of documents currently buffered in RAM") + .tags(Tags.of(indexTag)) + .baseUnit(BaseUnits.OBJECTS) + .register(Metrics.getRegistry()); + ramBytesUsedGauge = Gauge.builder("search_index_ram_used", iwriter, IndexWriter::ramBytesUsed) + .description("Memory usage of the index in bytes") + .tags(Tags.of(indexTag)) + .baseUnit(BaseUnits.BYTES) + .register(Metrics.getRegistry()); + numDocsGauge = Gauge.builder("search_index_docs_total", iwriter, w -> w.getDocStats().numDocs) + .description(""" + Number of docs in this index, including docs not yet flushed (still in the RAM buffer), \ + and including deletions""") + .tags(Tags.of(indexTag)) + .baseUnit(BaseUnits.OBJECTS) + .register(Metrics.getRegistry()); } /** * Returns an IndexWriter, by opening the index if necessary. + * * @return an IndexWriter * @throws IOException when the index cannot be opened * @since 3.0.0 @@ -227,6 +295,7 @@ protected synchronized IndexSearcher getIndexSearcher() throws IOException { /** * Returns a QueryParser. + * * @return a QueryParser * @since 3.0.0 */ @@ -238,34 +307,90 @@ protected QueryParser getQueryParser() { return qparser; } + public void addDocument(final Document document) { + try { + getIndexWriter().addDocument(document); + addOperationCounter.increment(); + } catch (CorruptIndexException e) { + handleCorruptIndexException(e); + } catch (IOException e) { + logger.error("An error occurred while adding document to index", e); + Notification.dispatch(new Notification() + .scope(NotificationScope.SYSTEM) + .group(NotificationGroup.INDEXING_SERVICE) + .title(NotificationConstants.Title.COMPONENT_INDEXER) + .content("An error occurred while adding document to index: %s; Check log for details.".formatted(e.getMessage())) + .level(NotificationLevel.ERROR) + ); + } + } + + public void updateDocument(final Term term, final Document document) { + try { + getIndexWriter().updateDocument(term, document); + updateOperationCounter.increment(); + } catch (CorruptIndexException e) { + handleCorruptIndexException(e); + } catch (IOException e) { + logger.error("An error occurred while updating document in index", e); + Notification.dispatch(new Notification() + .scope(NotificationScope.SYSTEM) + .group(NotificationGroup.INDEXING_SERVICE) + .title(getNotificationTitle()) + .content("An error occurred while updating document in index: %s; Check log for details.".formatted(e.getMessage())) + .level(NotificationLevel.ERROR) + ); + } + } + + public void deleteDocuments(final Term term) { + try { + getIndexWriter().deleteDocuments(term); + deleteOperationCounter.increment(); + } catch (CorruptIndexException e) { + handleCorruptIndexException(e); + } catch (IOException e) { + logger.error("An error occurred while deleting documents from index with term %s".formatted(term), e); + Notification.dispatch(new Notification() + .scope(NotificationScope.SYSTEM) + .group(NotificationGroup.INDEXING_SERVICE) + .title(getNotificationTitle()) + .content("An error occurred while removing a component from index: %s; Check log for details.".formatted(e.getMessage())) + .level(NotificationLevel.ERROR) + ); + } + } + /** * Commits changes to the index and closes the IndexWriter. + * * @since 3.0.0 */ public void commit() { try { getIndexWriter().commit(); + commitOperationCounter.increment(); } catch (CorruptIndexException e) { handleCorruptIndexException(e); } catch (IOException e) { - LOGGER.error("Error committing index", e); + logger.error("Error committing index", e); Notification.dispatch(new Notification() .scope(NotificationScope.SYSTEM) .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.CORE_INDEXING_SERVICES) - .content("Error committing index. Check log for details. " + e.getMessage()) + .title(getNotificationTitle()) + .content("Error committing index: %s; Check log for details.".formatted(e.getMessage())) .level(NotificationLevel.ERROR) ); } } protected void handleCorruptIndexException(CorruptIndexException e) { - LOGGER.error("Corrupted Lucene index detected", e); + LOGGER.error("Corrupted index detected", e); Notification.dispatch(new Notification() .scope(NotificationScope.SYSTEM) .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.CORE_INDEXING_SERVICES + "(" + indexType.name().toLowerCase() + ")") - .content("Corrupted Lucene index detected. Check log for details. " + e.getMessage()) + .title(getNotificationTitle()) + .content("Corrupted index detected: %s; Check log for details.".formatted(e.getMessage())) .level(NotificationLevel.ERROR) ); LOGGER.info("Trying to rebuild the corrupted index " + indexType.name()); @@ -274,6 +399,7 @@ protected void handleCorruptIndexException(CorruptIndexException e) { /** * Closes the IndexWriter. + * * @since 3.0.0 */ public void close() { @@ -286,14 +412,25 @@ public void close() { // do nothing... } } + + if (docsRamTotalGauge != null) { + docsRamTotalGauge.close(); + } + if (ramBytesUsedGauge != null) { + ramBytesUsedGauge.close(); + } + if (numDocsGauge != null) { + numDocsGauge.close(); + } } /** * Adds a Field to a Document. - * @param doc the Lucene Document to add a field to - * @param name the name of the field - * @param value the value of the field - * @param store storage options + * + * @param doc the Lucene Document to add a field to + * @param name the name of the field + * @param value the value of the field + * @param store storage options * @param tokenize specifies if the field should be tokenized or not * @since 3.0.0 */ @@ -310,25 +447,11 @@ protected void addField(final Document doc, final String name, String value, fin doc.add(field); } - /** - * Updates a Field in a Document. - * @param doc the Lucene Document to update the field in - * @param name the name of the field - * @param value the value of the field - * @since 3.0.0 - */ - protected void updateField(final Document doc, final String name, String value) { - if (StringUtils.isBlank(value)) { - value = ""; - } - final Field field = (Field) doc.getField(name); - field.setStringValue(value); - } - /** * Retrieves a specific Lucene Document for the specified Object, or null if not found. + * * @param fieldName the name of the field - * @param uuid the UUID to retrieve a Document for + * @param uuid the UUID to retrieve a Document for * @return a Lucene Document * @since 3.0.0 */ @@ -362,6 +485,7 @@ protected Document getDocument(final String fieldName, final String uuid) { /** * Returns the directory where this index is located. + * * @return a File object * @since 3.4.0 */ @@ -373,6 +497,7 @@ private static File getIndexDirectory(final IndexType indexType) { /** * Deletes the index directory. This method should be both overwritten and called via overwriting method. + * * @since 3.4.0 */ public void reindex() { @@ -391,6 +516,7 @@ public void reindex() { /** * Deletes the index directory. + * * @since 3.4.0 */ public static void delete(final IndexType indexType) { @@ -411,8 +537,8 @@ public static void delete(final IndexType indexType) { public static void ensureIndexesExists() { Arrays.stream(IndexManager.IndexType.values()).forEach(indexType -> { if (!isIndexHealthy(indexType)) { - LOGGER.info("(Re)Building index "+indexType.name().toLowerCase()); - LOGGER.debug("Dispatching event to reindex "+indexType.name().toLowerCase()); + LOGGER.info("(Re)Building index " + indexType.name().toLowerCase()); + LOGGER.debug("Dispatching event to reindex " + indexType.name().toLowerCase()); Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, indexType.getClazz())); } }); @@ -422,11 +548,11 @@ public static void ensureIndexesExists() { * Check that the index exists and is not corrupted */ private static boolean isIndexHealthy(final IndexType indexType) { - LOGGER.info("Checking the health of index "+indexType.name()); + LOGGER.info("Checking the health of index " + indexType.name()); File indexDirectoryFile = getIndexDirectory(indexType); - LOGGER.debug("Checking FS directory "+indexDirectoryFile.toPath()); - if(!indexDirectoryFile.exists()) { - LOGGER.warn("The index "+indexType.name()+" does not exist"); + LOGGER.debug("Checking FS directory " + indexDirectoryFile.toPath()); + if (!indexDirectoryFile.exists()) { + LOGGER.warn("The index " + indexType.name() + " does not exist"); return false; } LOGGER.debug("Checking lucene index health"); @@ -441,14 +567,14 @@ private static boolean isIndexHealthy(final IndexType indexType) { luceneIndexDirectory = FSDirectory.open(indexDirectoryFile.toPath()); checkIndex = new CheckIndex(luceneIndexDirectory); checkIndex.setFailFast(true); - if(LOGGER.isDebugEnabled()) { + if (LOGGER.isDebugEnabled()) { checkIndex.setInfoStream(System.out); } else { checkIndex.setInfoStream(new NullPrintStream()); } CheckIndex.Status status = checkIndex.checkIndex(); - if(status.clean) { - LOGGER.info("The index "+indexType.name()+" is healthy"); + if (status.clean) { + LOGGER.info("The index " + indexType.name() + " is healthy"); } else { LOGGER.error("The index " + indexType.name().toLowerCase() + " seems to be corrupted"); } @@ -481,18 +607,18 @@ public static void checkIndexesConsistency() { SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD.getGroupName(), SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD.getPropertyName()); double deltaThreshold = Double.parseDouble(deltaThresholdProperty.getPropertyValue()); double databaseEntityCount = qm.getCount(indexType.getClazz()); - LOGGER.info("Database entity count for type "+indexType.name()+" : "+databaseEntityCount); + LOGGER.info("Database entity count for type " + indexType.name() + " : " + databaseEntityCount); IndexManager indexManager = IndexManagerFactory.getIndexManager(indexType.getClazz()); indexManager.ensureDirectoryReaderOpen(); double indexDocumentCount = indexManager.searchReader.numDocs(); - LOGGER.info("Index document count for type "+indexType.name()+" : "+indexDocumentCount); - double max = Math.max(Math.max(databaseEntityCount, indexDocumentCount),1); - double delta = 100 * (Math.abs(databaseEntityCount-indexDocumentCount) / max); + LOGGER.info("Index document count for type " + indexType.name() + " : " + indexDocumentCount); + double max = Math.max(Math.max(databaseEntityCount, indexDocumentCount), 1); + double delta = 100 * (Math.abs(databaseEntityCount - indexDocumentCount) / max); delta = Math.max(Math.round(delta), 1); - LOGGER.info("Delta ratio for type "+indexType.name()+" : "+delta+"%"); - if(delta > deltaThreshold) { - LOGGER.info("Delta ratio is above the threshold of "+deltaThresholdProperty.getPropertyValue()+"%"); - LOGGER.debug("Dispatching event to reindex "+indexType.name().toLowerCase()); + LOGGER.info("Delta ratio for type " + indexType.name() + " : " + delta + "%"); + if (delta > deltaThreshold) { + LOGGER.info("Delta ratio is above the threshold of " + deltaThresholdProperty.getPropertyValue() + "%"); + LOGGER.debug("Dispatching event to reindex " + indexType.name().toLowerCase()); Event.dispatch(new IndexEvent(IndexEvent.Action.REINDEX, indexType.getClazz())); } } catch (IOException e) { @@ -507,4 +633,23 @@ public static void checkIndexesConsistency() { } }); } + + private String getNotificationTitle() { + if (this instanceof ComponentIndexer) { + return NotificationConstants.Title.COMPONENT_INDEXER; + } else if (this instanceof LicenseIndexer) { + return NotificationConstants.Title.LICENSE_INDEXER; + } else if (this instanceof ProjectIndexer) { + return NotificationConstants.Title.PROJECT_INDEXER; + } else if (this instanceof ServiceComponentIndexer) { + return NotificationConstants.Title.SERVICECOMPONENT_INDEXER; + } else if (this instanceof VulnerabilityIndexer) { + return NotificationConstants.Title.VULNERABILITY_INDEXER; + } else if (this instanceof VulnerableSoftwareIndexer) { + return NotificationConstants.Title.VULNERABLESOFTWARE_INDEXER; + } + + throw new IllegalStateException("Unrecognized indexer class %s".formatted(getClass())); + } + } diff --git a/src/main/java/org/dependencytrack/search/IndexManagerFactory.java b/src/main/java/org/dependencytrack/search/IndexManagerFactory.java index 7bbd7878e1..f620c70fc8 100644 --- a/src/main/java/org/dependencytrack/search/IndexManagerFactory.java +++ b/src/main/java/org/dependencytrack/search/IndexManagerFactory.java @@ -51,6 +51,8 @@ public static ObjectIndexer getIndexManager(final Inde @Override public void add(final DummyDocument object) { } @Override + public void update(final DummyDocument object) { } + @Override public void remove(final DummyDocument object) { } @Override public void commit() { } diff --git a/src/main/java/org/dependencytrack/search/LicenseIndexer.java b/src/main/java/org/dependencytrack/search/LicenseIndexer.java index 761946f5da..8845175603 100644 --- a/src/main/java/org/dependencytrack/search/LicenseIndexer.java +++ b/src/main/java/org/dependencytrack/search/LicenseIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.License; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.LicenseDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.List; @@ -48,7 +41,7 @@ public final class LicenseIndexer extends IndexManager implements ObjectIndexer< private static final Logger LOGGER = Logger.getLogger(LicenseIndexer.class); private static final LicenseIndexer INSTANCE = new LicenseIndexer(); - protected static LicenseIndexer getInstance() { + static LicenseIndexer getInstance() { return INSTANCE; } @@ -70,25 +63,15 @@ public String[] getSearchFields() { * @param license A persisted License object. */ public void add(final LicenseDocument license) { - final Document doc = new Document(); - addField(doc, IndexConstants.LICENSE_UUID, license.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.LICENSE_LICENSEID, license.licenseId(), Field.Store.YES, true); - addField(doc, IndexConstants.LICENSE_NAME, license.name(), Field.Store.YES, true); + final Document doc = convertToDocument(license); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding a license to the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.LICENSE_INDEXER) - .content("An error occurred while adding a license to the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(final LicenseDocument license) { + final Term term = convertToTerm(license); + final Document doc = convertToDocument(license); + updateDocument(term, doc); } /** @@ -97,20 +80,8 @@ public void add(final LicenseDocument license) { * @param license A persisted License object. */ public void remove(final LicenseDocument license) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.LICENSE_UUID, license.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a license from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.LICENSE_INDEXER) - .content("An error occurred while removing a license from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(license); + deleteDocuments(term); } /** @@ -153,4 +124,16 @@ private static List fetchNext(final QueryManager qm, final Long } } + private Document convertToDocument(final LicenseDocument license) { + final var doc = new Document(); + addField(doc, IndexConstants.LICENSE_UUID, license.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.LICENSE_LICENSEID, license.licenseId(), Field.Store.YES, true); + addField(doc, IndexConstants.LICENSE_NAME, license.name(), Field.Store.YES, true); + return doc; + } + + private static Term convertToTerm(final LicenseDocument license) { + return new Term(IndexConstants.LICENSE_UUID, license.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/search/LoggingInfoStream.java b/src/main/java/org/dependencytrack/search/LoggingInfoStream.java new file mode 100644 index 0000000000..e1c5645d95 --- /dev/null +++ b/src/main/java/org/dependencytrack/search/LoggingInfoStream.java @@ -0,0 +1,85 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.search; + +import org.apache.lucene.util.InfoStream; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * Lucene {@link InfoStream} implementation for Dependency-Track's SLF4J-based logging mechanism. + *

+ * Logging of Lucene messages can be enabled on a per-index basis: + * + * 

{@code
+ *     
+ *         
+ *     
+ * }
+ *

+ * It is also possible to enable logging selectively for specific Lucene components: + * + * 

{@code
+ *     
+ *         
+ *     
+ * }
+ *

+ * A few known Lucene components and their abbreviations are: + *

    + *
  • {@link org.apache.lucene.index.DocumentsWriterFlushControl} (DWFC)
    • + *
  • {@link org.apache.lucene.index.DocumentsWriter} (DW)
    • + *
  • {@link org.apache.lucene.index.IndexFileDeleter} (IFD)
    • + *
  • {@link org.apache.lucene.index.IndexWriter} (IW)
    • + *
  • {@link org.apache.lucene.index.MergePolicy} (MP)
    • + *
  • {@link org.apache.lucene.index.MergeScheduler} (MS)
    • + *
+ * + * @since 4.11.0 + */ +@SuppressWarnings("JavadocReference") +class LoggingInfoStream extends InfoStream { + + private final Class parentLoggerClass; + + LoggingInfoStream(final Class parentLoggerClass) { + this.parentLoggerClass = parentLoggerClass; + } + + @Override + public void message(final String component, final String message) { + getLogger(component).debug(message); + } + + @Override + public boolean isEnabled(final String component) { + return getLogger(component).isDebugEnabled(); + } + + @Override + public void close() throws IOException { + } + + private org.slf4j.Logger getLogger(final String component) { + final String loggerName = "%s.lucene.%s".formatted(parentLoggerClass.getName(), component); + return LoggerFactory.getLogger(loggerName); + } + +} diff --git a/src/main/java/org/dependencytrack/search/ObjectIndexer.java b/src/main/java/org/dependencytrack/search/ObjectIndexer.java index 27d3536d43..cf9ca6400e 100644 --- a/src/main/java/org/dependencytrack/search/ObjectIndexer.java +++ b/src/main/java/org/dependencytrack/search/ObjectIndexer.java @@ -41,6 +41,13 @@ public interface ObjectIndexer { */ void add(T object); + /** + * Update object in index. + * @param object the object to update + * @since 4.11.0 + */ + void update(T object); + /** * Remove object from index. * @param object the object to remove diff --git a/src/main/java/org/dependencytrack/search/ProjectIndexer.java b/src/main/java/org/dependencytrack/search/ProjectIndexer.java index f6918af433..4d0d6f7c93 100644 --- a/src/main/java/org/dependencytrack/search/ProjectIndexer.java +++ b/src/main/java/org/dependencytrack/search/ProjectIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.Project; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.ProjectDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -50,7 +43,7 @@ public final class ProjectIndexer extends IndexManager implements ObjectIndexer< private static final Logger LOGGER = Logger.getLogger(ProjectIndexer.class); private static final ProjectIndexer INSTANCE = new ProjectIndexer(); - protected static ProjectIndexer getInstance() { + static ProjectIndexer getInstance() { return INSTANCE; } @@ -72,39 +65,15 @@ public String[] getSearchFields() { * @param project A persisted Project object. */ public void add(final ProjectDocument project) { - final Document doc = new Document(); - addField(doc, IndexConstants.PROJECT_UUID, project.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.PROJECT_NAME, project.name(), Field.Store.YES, true); - addField(doc, IndexConstants.PROJECT_VERSION, project.version(), Field.Store.YES, false); - addField(doc, IndexConstants.PROJECT_DESCRIPTION, project.description(), Field.Store.YES, true); - - /* - // There's going to potentially be confidential information in the project properties. Do not index. - - final StringBuilder sb = new StringBuilder(); - if (project.getProperties() != null) { - for (ProjectProperty property : project.getProperties()) { - sb.append(property.getPropertyValue()).append(" "); - } - } - - addField(doc, IndexConstants.PROJECT_PROPERTIES, sb.toString().trim(), Field.Store.YES, true); - */ + final Document doc = convertToDocument(project); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding a project to the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.PROJECT_INDEXER) - .content("An error occurred while adding a project to the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(final ProjectDocument project) { + final Term term = convertToTerm(project); + final Document doc = convertToDocument(project); + updateDocument(term, doc); } /** @@ -113,20 +82,8 @@ public void add(final ProjectDocument project) { * @param project A persisted Project object. */ public void remove(final ProjectDocument project) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.PROJECT_UUID, project.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a project from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.PROJECT_INDEXER) - .content("An error occurred while removing a project from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(project); + deleteDocuments(term); } /** @@ -174,4 +131,31 @@ private static List fetchNext(final QueryManager qm, final Long } } + private Document convertToDocument(final ProjectDocument project) { + final var doc = new Document(); + addField(doc, IndexConstants.PROJECT_UUID, project.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.PROJECT_NAME, project.name(), Field.Store.YES, true); + addField(doc, IndexConstants.PROJECT_VERSION, project.version(), Field.Store.YES, false); + addField(doc, IndexConstants.PROJECT_DESCRIPTION, project.description(), Field.Store.YES, true); + + /* + // There's going to potentially be confidential information in the project properties. Do not index. + + final StringBuilder sb = new StringBuilder(); + if (project.getProperties() != null) { + for (ProjectProperty property : project.getProperties()) { + sb.append(property.getPropertyValue()).append(" "); + } + } + + addField(doc, IndexConstants.PROJECT_PROPERTIES, sb.toString().trim(), Field.Store.YES, true); + */ + + return doc; + } + + private static Term convertToTerm(final ProjectDocument project) { + return new Term(IndexConstants.PROJECT_UUID, project.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/search/ServiceComponentIndexer.java b/src/main/java/org/dependencytrack/search/ServiceComponentIndexer.java index 3c2e3f8994..6d9b13b1c1 100644 --- a/src/main/java/org/dependencytrack/search/ServiceComponentIndexer.java +++ b/src/main/java/org/dependencytrack/search/ServiceComponentIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.ServiceComponent; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.ServiceComponentDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.List; @@ -48,7 +41,7 @@ public final class ServiceComponentIndexer extends IndexManager implements Objec private static final Logger LOGGER = Logger.getLogger(ServiceComponentIndexer.class); private static final ServiceComponentIndexer INSTANCE = new ServiceComponentIndexer(); - protected static ServiceComponentIndexer getInstance() { + static ServiceComponentIndexer getInstance() { return INSTANCE; } @@ -70,28 +63,15 @@ public String[] getSearchFields() { * @param service A persisted ServiceComponent object. */ public void add(final ServiceComponentDocument service) { - final Document doc = new Document(); - addField(doc, IndexConstants.SERVICECOMPONENT_UUID, service.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.SERVICECOMPONENT_NAME, service.name(), Field.Store.YES, true); - addField(doc, IndexConstants.SERVICECOMPONENT_GROUP, service.group(), Field.Store.YES, true); - addField(doc, IndexConstants.SERVICECOMPONENT_VERSION, service.version(), Field.Store.YES, false); - // TODO: addField(doc, IndexConstants.SERVICECOMPONENT_URL, service.getUrl(), Field.Store.YES, true); - addField(doc, IndexConstants.SERVICECOMPONENT_DESCRIPTION, service.description(), Field.Store.YES, true); + final Document doc = convertToDocument(service); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding service to index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.SERVICECOMPONENT_INDEXER) - .content("An error occurred while adding service to index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(final ServiceComponentDocument service) { + final Term term = convertToTerm(service); + final Document doc = convertToDocument(service); + updateDocument(term, doc); } /** @@ -100,20 +80,8 @@ public void add(final ServiceComponentDocument service) { * @param service A persisted ServiceComponent object. */ public void remove(final ServiceComponentDocument service) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.SERVICECOMPONENT_UUID, service.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a service from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.SERVICECOMPONENT_INDEXER) - .content("An error occurred while removing a service from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(service); + deleteDocuments(term); } /** @@ -156,4 +124,19 @@ private static List fetchNext(final QueryManager qm, f } } + private Document convertToDocument(final ServiceComponentDocument service) { + final var doc = new Document(); + addField(doc, IndexConstants.SERVICECOMPONENT_UUID, service.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.SERVICECOMPONENT_NAME, service.name(), Field.Store.YES, true); + addField(doc, IndexConstants.SERVICECOMPONENT_GROUP, service.group(), Field.Store.YES, true); + addField(doc, IndexConstants.SERVICECOMPONENT_VERSION, service.version(), Field.Store.YES, false); + // TODO: addField(doc, IndexConstants.SERVICECOMPONENT_URL, service.getUrl(), Field.Store.YES, true); + addField(doc, IndexConstants.SERVICECOMPONENT_DESCRIPTION, service.description(), Field.Store.YES, true); + return doc; + } + + private static Term convertToTerm(final ServiceComponentDocument service) { + return new Term(IndexConstants.SERVICECOMPONENT_UUID, service.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/search/VulnerabilityIndexer.java b/src/main/java/org/dependencytrack/search/VulnerabilityIndexer.java index ede9e4c08f..de36c8a16d 100644 --- a/src/main/java/org/dependencytrack/search/VulnerabilityIndexer.java +++ b/src/main/java/org/dependencytrack/search/VulnerabilityIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.Vulnerability; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.VulnerabilityDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.List; @@ -48,7 +41,7 @@ public final class VulnerabilityIndexer extends IndexManager implements ObjectIn private static final Logger LOGGER = Logger.getLogger(VulnerabilityIndexer.class); private static final VulnerabilityIndexer INSTANCE = new VulnerabilityIndexer(); - protected static VulnerabilityIndexer getInstance() { + static VulnerabilityIndexer getInstance() { return INSTANCE; } @@ -70,26 +63,15 @@ public String[] getSearchFields() { * @param vulnerability A persisted Vulnerability object. */ public void add(final VulnerabilityDocument vulnerability) { - final Document doc = new Document(); - addField(doc, IndexConstants.VULNERABILITY_UUID, vulnerability.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.VULNERABILITY_VULNID, vulnerability.vulnId(), Field.Store.YES, true); - addField(doc, IndexConstants.VULNERABILITY_DESCRIPTION, vulnerability.description(), Field.Store.YES, true); - addField(doc, IndexConstants.VULNERABILITY_SOURCE, vulnerability.source(), Field.Store.YES, false); + final Document doc = convertToDocument(vulnerability); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding a vulnerability to the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.VULNERABILITY_INDEXER) - .content("An error occurred while adding a vulnerability to the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(final VulnerabilityDocument vuln) { + final Term term = convertToTerm(vuln); + final Document doc = convertToDocument(vuln); + updateDocument(term, doc); } /** @@ -98,20 +80,8 @@ public void add(final VulnerabilityDocument vulnerability) { * @param vulnerability A persisted Vulnerability object. */ public void remove(final VulnerabilityDocument vulnerability) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.VULNERABILITY_UUID, vulnerability.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a vulnerability from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.VULNERABILITY_INDEXER) - .content("An error occurred while removing a vulnerability from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(vulnerability); + deleteDocuments(term); } /** @@ -154,4 +124,17 @@ private static List fetchNext(final QueryManager qm, fina } } + private Document convertToDocument(final VulnerabilityDocument vuln) { + final var doc = new Document(); + addField(doc, IndexConstants.VULNERABILITY_UUID, vuln.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.VULNERABILITY_VULNID, vuln.vulnId(), Field.Store.YES, true); + addField(doc, IndexConstants.VULNERABILITY_DESCRIPTION, vuln.description(), Field.Store.YES, true); + addField(doc, IndexConstants.VULNERABILITY_SOURCE, vuln.source(), Field.Store.YES, false); + return doc; + } + + private static Term convertToTerm(final VulnerabilityDocument vuln) { + return new Term(IndexConstants.VULNERABILITY_UUID, vuln.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/search/VulnerableSoftwareIndexer.java b/src/main/java/org/dependencytrack/search/VulnerableSoftwareIndexer.java index b569e86ac7..6322ef59f3 100644 --- a/src/main/java/org/dependencytrack/search/VulnerableSoftwareIndexer.java +++ b/src/main/java/org/dependencytrack/search/VulnerableSoftwareIndexer.java @@ -19,21 +19,14 @@ package org.dependencytrack.search; import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; import org.dependencytrack.model.VulnerableSoftware; -import org.dependencytrack.notification.NotificationConstants; -import org.dependencytrack.notification.NotificationGroup; -import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.search.document.VulnerableSoftwareDocument; import javax.jdo.Query; -import java.io.IOException; import java.time.Duration; import java.util.List; @@ -48,7 +41,7 @@ public final class VulnerableSoftwareIndexer extends IndexManager implements Obj private static final Logger LOGGER = Logger.getLogger(VulnerableSoftwareIndexer.class); private static final VulnerableSoftwareIndexer INSTANCE = new VulnerableSoftwareIndexer(); - protected static VulnerableSoftwareIndexer getInstance() { + static VulnerableSoftwareIndexer getInstance() { return INSTANCE; } @@ -70,29 +63,15 @@ public String[] getSearchFields() { * @param vs A persisted VulnerableSoftware object. */ public void add(final VulnerableSoftwareDocument vs) { - final Document doc = new Document(); - addField(doc, IndexConstants.VULNERABLESOFTWARE_UUID, vs.uuid().toString(), Field.Store.YES, false); - addField(doc, IndexConstants.VULNERABLESOFTWARE_CPE_22, vs.cpe22(), Field.Store.YES, false); - addField(doc, IndexConstants.VULNERABLESOFTWARE_CPE_23, vs.cpe23(), Field.Store.YES, false); - addField(doc, IndexConstants.VULNERABLESOFTWARE_VENDOR, vs.vendor(), Field.Store.YES, true); - addField(doc, IndexConstants.VULNERABLESOFTWARE_PRODUCT, vs.product(), Field.Store.YES, true); - addField(doc, IndexConstants.VULNERABLESOFTWARE_VERSION, vs.version(), Field.Store.YES, true); - //todo: index the affected version range fields as well + final Document doc = convertToDocument(vs); + addDocument(doc); + } - try { - getIndexWriter().addDocument(doc); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while adding a VulnerableSoftware to the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.VULNERABLESOFTWARE_INDEXER) - .content("An error occurred while adding a VulnerableSoftware to the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + @Override + public void update(final VulnerableSoftwareDocument vs) { + final Term term = convertToTerm(vs); + final Document doc = convertToDocument(vs); + updateDocument(term, doc); } /** @@ -101,20 +80,8 @@ public void add(final VulnerableSoftwareDocument vs) { * @param vs A persisted VulnerableSoftware object. */ public void remove(final VulnerableSoftwareDocument vs) { - try { - getIndexWriter().deleteDocuments(new Term(IndexConstants.VULNERABLESOFTWARE_UUID, vs.uuid().toString())); - } catch (CorruptIndexException e) { - handleCorruptIndexException(e); - } catch (IOException e) { - LOGGER.error("An error occurred while removing a VulnerableSoftware from the index", e); - Notification.dispatch(new Notification() - .scope(NotificationScope.SYSTEM) - .group(NotificationGroup.INDEXING_SERVICE) - .title(NotificationConstants.Title.VULNERABLESOFTWARE_INDEXER) - .content("An error occurred while removing a VulnerableSoftware from the index. Check log for details. " + e.getMessage()) - .level(NotificationLevel.ERROR) - ); - } + final Term term = convertToTerm(vs); + deleteDocuments(term); } /** @@ -156,4 +123,20 @@ private static List fetchNext(final QueryManager qm, } } + private Document convertToDocument(final VulnerableSoftwareDocument vs) { + final var doc = new Document(); + addField(doc, IndexConstants.VULNERABLESOFTWARE_UUID, vs.uuid().toString(), Field.Store.YES, false); + addField(doc, IndexConstants.VULNERABLESOFTWARE_CPE_22, vs.cpe22(), Field.Store.YES, false); + addField(doc, IndexConstants.VULNERABLESOFTWARE_CPE_23, vs.cpe23(), Field.Store.YES, false); + addField(doc, IndexConstants.VULNERABLESOFTWARE_VENDOR, vs.vendor(), Field.Store.YES, true); + addField(doc, IndexConstants.VULNERABLESOFTWARE_PRODUCT, vs.product(), Field.Store.YES, true); + addField(doc, IndexConstants.VULNERABLESOFTWARE_VERSION, vs.version(), Field.Store.YES, true); + //todo: index the affected version range fields as well + return doc; + } + + private static Term convertToTerm(final VulnerableSoftwareDocument vs) { + return new Term(IndexConstants.VULNERABLESOFTWARE_UUID, vs.uuid().toString()); + } + } diff --git a/src/main/java/org/dependencytrack/tasks/IndexTask.java b/src/main/java/org/dependencytrack/tasks/IndexTask.java index 6f2fa61658..491d380f78 100644 --- a/src/main/java/org/dependencytrack/tasks/IndexTask.java +++ b/src/main/java/org/dependencytrack/tasks/IndexTask.java @@ -18,7 +18,6 @@ */ package org.dependencytrack.tasks; -import alpine.common.logging.Logger; import alpine.common.metrics.Metrics; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; @@ -36,8 +35,6 @@ */ public class IndexTask implements Subscriber { - private static final Logger LOGGER = Logger.getLogger(IndexTask.class); - /** * {@inheritDoc} */ @@ -53,12 +50,11 @@ public void inform(final Event e) { final ObjectIndexer indexManager = IndexManagerFactory.getIndexManager(event); if (IndexEvent.Action.CREATE == event.getAction()) { - indexManager.add((event).getDocument()); + indexManager.add(event.getDocument()); } else if (IndexEvent.Action.UPDATE == event.getAction()) { - indexManager.remove((event).getDocument()); - indexManager.add((event).getDocument()); + indexManager.update(event.getDocument()); } else if (IndexEvent.Action.DELETE == event.getAction()) { - indexManager.remove((event).getDocument()); + indexManager.remove(event.getDocument()); } else if (IndexEvent.Action.COMMIT == event.getAction()) { indexManager.commit(); } else if (IndexEvent.Action.REINDEX == event.getAction()) {