From 042b2cdbc312eb39643671350adc1798f005267e Mon Sep 17 00:00:00 2001 From: Norbert Pomaroli Date: Fri, 15 Oct 2021 14:57:32 +0200 Subject: [PATCH] Add periodic check for elasticsearch indices. Add option to restrict index maintenance actions to specific indices Let index maintenance actions run on the master, if cluster coordinator is used --- LTS-CHANGELOG.adoc | 17 +- .../config/search/ElasticSearchOptions.java | 48 ++++++ .../mesh/cache/impl/EventAwareCacheImpl.java | 24 ++- .../mesh/core/data/search/IndexHandler.java | 11 +- .../mesh/distributed/DistributionUtils.java | 39 +++++ .../mesh/distributed/RequestDelegator.java | 6 + .../parameter/ParameterProviderContext.java | 5 + .../impl/IndexMaintenanceParametersImpl.java | 49 ++++++ .../mesh/search/DevNullSearchProvider.java | 2 +- .../mesh/search/SearchMappingsCache.java | 17 ++ .../gentics/mesh/search/SearchProvider.java | 20 ++- .../mesh/search/TrackingSearchProvider.java | 2 +- .../mesh/cli/BootstrapInitializerImpl.java | 2 +- .../mesh/dagger/module/BindModule.java | 5 + .../mesh/rest/MeshLocalClientImpl.java | 8 +- .../mesh/search/SearchEndpointImpl.java | 3 + .../mesh/search/index/BasicIndexSyncTest.java | 145 +++++++++++++++-- .../mesh/search/index/IndexClearTest.java | 63 ++++++-- .../search/index/IndexSyncCleanupTest.java | 55 ++++++- .../mesh/test/context/MeshTestContext.java | 4 + .../mesh/test/context/helper/EventHelper.java | 58 ++++++- .../test/context/helper/ExpectedEvent.java | 5 +- .../test/context/helper/UnexpectedEvent.java | 52 ++++++ .../distributed/coordinator/Coordinator.java | 11 +- .../coordinator/MasterElector.java | 4 + .../proxy/RequestDelegatorImpl.java | 5 + ...ndexMaintenanceParametersImpl.adoc-include | 15 ++ .../SearchIndexSyncEventModel.adoc-include | 25 +++ .../search/impl/ElasticSearchProvider.java | 152 +++++++++++++++++- .../search/impl/SearchMappingsCacheImpl.java | 55 +++++++ .../mesh/search/index/AdminIndexHandler.java | 7 +- .../index/entry/AbstractIndexHandler.java | 34 +++- .../search/index/group/GroupIndexHandler.java | 6 +- .../MicroschemaContainerIndexHandler.java | 6 +- .../search/index/node/NodeIndexHandler.java | 19 ++- .../index/project/ProjectIndexHandler.java | 6 +- .../search/index/role/RoleIndexHandler.java | 6 +- .../schema/SchemaContainerIndexHandler.java | 6 +- .../search/index/tag/TagIndexHandler.java | 6 +- .../tagfamily/TagFamilyIndexHandler.java | 6 +- .../search/index/user/UserIndexHandler.java | 6 +- .../ElasticsearchProcessVerticle.java | 28 +++- .../eventhandler/CheckIndicesHandler.java | 66 ++++++++ .../eventhandler/MainEventHandler.java | 8 +- .../eventhandler/SyncEventHandler.java | 48 ++++-- .../IndexMaintenanceParametersImpl.java | 10 ++ .../client/impl/MeshRestHttpClientImpl.java | 8 +- .../client/method/SearchClientMethods.java | 6 +- .../com/gentics/mesh/core/rest/MeshEvent.java | 24 ++- .../search/SearchIndexSyncEventModel.java | 40 +++++ .../parameter/IndexMaintenanceParameters.java | 29 ++++ 51 files changed, 1182 insertions(+), 100 deletions(-) create mode 100644 common/src/main/java/com/gentics/mesh/parameter/impl/IndexMaintenanceParametersImpl.java create mode 100644 common/src/main/java/com/gentics/mesh/search/SearchMappingsCache.java create mode 100644 core/src/test/java/com/gentics/mesh/test/context/helper/UnexpectedEvent.java create mode 100644 doc/src/main/docs/generated/tables/IndexMaintenanceParametersImpl.adoc-include create mode 100644 doc/src/main/docs/generated/tables/SearchIndexSyncEventModel.adoc-include create mode 100644 elasticsearch/src/main/java/com/gentics/mesh/search/impl/SearchMappingsCacheImpl.java create mode 100644 elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/CheckIndicesHandler.java create mode 100644 rest-client/src/main/java/com/gentics/mesh/parameter/client/IndexMaintenanceParametersImpl.java create mode 100644 rest-model/src/main/java/com/gentics/mesh/core/rest/event/search/SearchIndexSyncEventModel.java create mode 100644 rest-model/src/main/java/com/gentics/mesh/parameter/IndexMaintenanceParameters.java diff --git a/LTS-CHANGELOG.adoc b/LTS-CHANGELOG.adoc index 45cfc00075..8bf75602eb 100644 --- a/LTS-CHANGELOG.adoc +++ b/LTS-CHANGELOG.adoc @@ -18,11 +18,24 @@ The LTS changelog lists releases which are only accessible via a commercial subs All fixes and changes in LTS releases will be released the next minor release. Changes from LTS 1.4.x will be included in release 1.5.0. [[v1.6.22]] -== 1.6.22 (12.10.2021) +== 1.6.22 (TBD) icon:check[] Server: when a schema was updated with a new field with elastic search properties, these were ignored. This has been fixed now. +icon:check[] Search: If ElasticSearch indices, which were originally created by Mesh were dropped, they would probably be re-created with the default mapping when Mesh +stored a document in the index. This could cause the index to be not completely filled and also the mapping to be incorrect, so that search queries would fail to find +documents. In order to detect and repair such situations, a periodic check has been added to Mesh, which will check existence and correctness of all required ElasticSearch +indices. Missing indices will be created and incorrect indices will be dropped and re-created. Afterwards a full sync for the affected indices will be triggered. The check +period can be configured with the new configuration setting `search.indexCheckInterval`, which defaults to 60_000 milliseconds. Setting the interval to 0 will disable the +periodic check (not recommended). + +icon:plus[] Search: The index maintenance endpoints `/api/v1/search/sync` and `/api/v1/search/clear` have been extended with the query parameter `index`, which can be used +to restrict the synchronized/cleared indices by a regular expression. + +icon:check[] Search: The index maintenance actions triggered via `/api/v1/search/sync` and `/api/v1/search/clear` will now be redirected to the current master instance, +if the cluster coordinator is used. + [[v1.6.21]] -== 1.6.21 (TBD) +== 1.6.21 (12.10.2021) icon:check[] Search: When a full synchronization of the search indices was triggered, language specific indices were unnecessarily removed first. This has been changed, language specific indices will now be treated like all other indices during a full synchronization. diff --git a/api/src/main/java/com/gentics/mesh/etc/config/search/ElasticSearchOptions.java b/api/src/main/java/com/gentics/mesh/etc/config/search/ElasticSearchOptions.java index c95da87221..3f615cecf0 100644 --- a/api/src/main/java/com/gentics/mesh/etc/config/search/ElasticSearchOptions.java +++ b/api/src/main/java/com/gentics/mesh/etc/config/search/ElasticSearchOptions.java @@ -42,6 +42,9 @@ public class ElasticSearchOptions implements Option { public static final boolean DEFAULT_HOSTNAME_VERIFICATION = true; + public static final long DEFAULT_INDEX_CHECK_INTERVAL = 60 * 1000; + public static final long DEFAULT_INDEX_MAPPING_CACHE_TIMEOUT = 60 * 60 * 1000; + public static final String MESH_ELASTICSEARCH_URL_ENV = "MESH_ELASTICSEARCH_URL"; public static final String MESH_ELASTICSEARCH_USERNAME_ENV = "MESH_ELASTICSEARCH_USERNAME"; public static final String MESH_ELASTICSEARCH_PASSWORD_ENV = "MESH_ELASTICSEARCH_PASSWORD"; @@ -66,6 +69,9 @@ public class ElasticSearchOptions implements Option { public static final String MESH_ELASTICSEARCH_HOSTNAME_VERIFICATION_ENV = "MESH_ELASTICSEARCH_HOSTNAME_VERIFICATION"; public static final String MESH_ELASTICSEARCH_INCLUDE_BINARY_FIELDS_ENV = "MESH_ELASTICSEARCH_INCLUDE_BINARY_FIELDS"; + public static final String MESH_ELASTICSEARCH_INDEX_CHECK_INTERVAL_ENV = "MESH_ELASTICSEARCH_INDEX_CHECK_INTERVAL"; + public static final String MESH_ELASTICSEARCH_INDEX_MAPPING_CACHE_TIMEOUT_ENV = "MESH_ELASTICSEARCH_INDEX_MAPPING_CACHE_TIMEOUT"; + @JsonProperty(required = false) @JsonPropertyDescription("Elasticsearch connection url to be used. Set this setting to null will disable the Elasticsearch support.") @EnvironmentVariable(name = MESH_ELASTICSEARCH_URL_ENV, description = "Override the configured elasticsearch server url. The value can be set to null in order to disable the Elasticsearch support.") @@ -189,6 +195,16 @@ public class ElasticSearchOptions implements Option { @EnvironmentVariable(name = MESH_ELASTICSEARCH_SYNC_BATCH_SIZE_ENV, description = "Override the search sync batch size") private int syncBatchSize = DEFAULT_SYNC_BATCH_SIZE; + @JsonProperty(required = false) + @JsonPropertyDescription("Set the interval of index checks in ms. Default: " + DEFAULT_INDEX_CHECK_INTERVAL) + @EnvironmentVariable(name = MESH_ELASTICSEARCH_INDEX_CHECK_INTERVAL_ENV, description = "Override the interval for index checks") + private long indexCheckInterval = DEFAULT_INDEX_CHECK_INTERVAL; + + @JsonProperty(required = false) + @JsonPropertyDescription("Set the timeout for the cache of index mappings in ms. Default: " + DEFAULT_INDEX_MAPPING_CACHE_TIMEOUT) + @EnvironmentVariable(name = MESH_ELASTICSEARCH_INDEX_MAPPING_CACHE_TIMEOUT_ENV, description = "Override the timeout for the cache if index mappings") + private long indexMappingCacheTimeout = DEFAULT_INDEX_MAPPING_CACHE_TIMEOUT; + public ElasticSearchOptions() { } @@ -442,4 +458,36 @@ public int getSyncBatchSize() { public void setSyncBatchSize(int batchSize) { this.syncBatchSize = batchSize; } + + /** + * Index check interval in ms + * @return interval + */ + public long getIndexCheckInterval() { + return indexCheckInterval; + } + + /** + * Set the index check interval in ms + * @param indexCheckInterval interval + */ + public void setIndexCheckInterval(long indexCheckInterval) { + this.indexCheckInterval = indexCheckInterval; + } + + /** + * Timeout for the cache of index mappings in ms + * @return timeout + */ + public long getIndexMappingCacheTimeout() { + return indexMappingCacheTimeout; + } + + /** + * Set the timeout for the cache of index mappings + * @param indexMappingCacheTimeout timeout + */ + public void setIndexMappingCacheTimeout(long indexMappingCacheTimeout) { + this.indexMappingCacheTimeout = indexMappingCacheTimeout; + } } diff --git a/common/src/main/java/com/gentics/mesh/cache/impl/EventAwareCacheImpl.java b/common/src/main/java/com/gentics/mesh/cache/impl/EventAwareCacheImpl.java index 4f8345f9f7..32e1fe1c70 100644 --- a/common/src/main/java/com/gentics/mesh/cache/impl/EventAwareCacheImpl.java +++ b/common/src/main/java/com/gentics/mesh/cache/impl/EventAwareCacheImpl.java @@ -48,15 +48,18 @@ public class EventAwareCacheImpl implements EventAwareCache { private final Counter missCounter; private final Counter hitCounter; - public EventAwareCacheImpl(String name, long maxSize, Duration expireAfter, Vertx vertx, MeshOptions options, MetricsService metricsService, Predicate> filter, - BiConsumer, EventAwareCache> onNext, - MeshEvent... events) { + public EventAwareCacheImpl(String name, long maxSize, Duration expireAfter, Duration expireAfterAccess, Vertx vertx, MeshOptions options, MetricsService metricsService, + Predicate> filter, + BiConsumer, EventAwareCache> onNext, MeshEvent... events) { this.vertx = vertx; this.options = options; Caffeine cacheBuilder = Caffeine.newBuilder().maximumSize(maxSize); if (expireAfter != null) { cacheBuilder = cacheBuilder.expireAfterWrite(expireAfter.getSeconds(), TimeUnit.SECONDS); } + if (expireAfterAccess != null) { + cacheBuilder = cacheBuilder.expireAfterAccess(expireAfterAccess.getSeconds(), TimeUnit.SECONDS); + } this.cache = cacheBuilder.build(); this.filter = filter; this.onNext = onNext; @@ -185,6 +188,7 @@ public static class Builder { private MeshEvent[] events = null; private Vertx vertx; private Duration expireAfter; + private Duration expireAfterAccess; private String name; private MeshOptions options; private MetricsService metricsService; @@ -193,7 +197,7 @@ public EventAwareCache build() { Objects.requireNonNull(events, "No events for the cache have been set"); Objects.requireNonNull(vertx, "No Vert.x instance has been set"); Objects.requireNonNull(name, "No name has been set"); - EventAwareCacheImpl c = new EventAwareCacheImpl<>(name, maxSize, expireAfter, vertx, options, metricsService, filter, onNext, events); + EventAwareCacheImpl c = new EventAwareCacheImpl<>(name, maxSize, expireAfter, expireAfterAccess, vertx, options, metricsService, filter, onNext, events); if (disabled) { c.disable(); } @@ -296,6 +300,18 @@ public Builder expireAfter(long amount, TemporalUnit unit) { return this; } + /** + * Define when the cache should automatically expire after last access + * + * @param amount + * @param unit + * @return Fluent API + */ + public Builder expireAfterAccess(long amount, TemporalUnit unit) { + this.expireAfterAccess = Duration.of(amount, unit); + return this; + } + /** * Sets the name for the cache. This is used for caching metrics. * @param name diff --git a/common/src/main/java/com/gentics/mesh/core/data/search/IndexHandler.java b/common/src/main/java/com/gentics/mesh/core/data/search/IndexHandler.java index 623e174529..60acd08fa6 100644 --- a/common/src/main/java/com/gentics/mesh/core/data/search/IndexHandler.java +++ b/common/src/main/java/com/gentics/mesh/core/data/search/IndexHandler.java @@ -1,8 +1,10 @@ package com.gentics.mesh.core.data.search; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import com.gentics.mesh.context.InternalActionContext; @@ -27,6 +29,7 @@ * @param */ public interface IndexHandler> { + public final static Pattern MATCH_ALL = Pattern.compile(".*"); /** * Initialise the search index by creating the index first and setting the mapping afterwards. @@ -82,9 +85,10 @@ public interface IndexHandler> { /** * Diff the elements within all indices that are handled by the index handler and synchronize the data. * + * @param indexPattern optional index pattern to restrict synchronized indices * @return */ - Flowable syncIndices(); + Flowable syncIndices(Optional indexPattern); /** * Filter the given list and return only indices which match the type of the handler but are no longer in use or unknown. @@ -167,4 +171,9 @@ default GraphPermission getReadPermission(InternalActionContext ac) { */ Observable updatePermissionForBulk(UpdateDocumentEntry entry); + /** + * Check indices handled by this handler for existence and correctness (mapping) + * @return completable + */ + Completable check(); } diff --git a/common/src/main/java/com/gentics/mesh/distributed/DistributionUtils.java b/common/src/main/java/com/gentics/mesh/distributed/DistributionUtils.java index 06f673745a..1527291e91 100644 --- a/common/src/main/java/com/gentics/mesh/distributed/DistributionUtils.java +++ b/common/src/main/java/com/gentics/mesh/distributed/DistributionUtils.java @@ -17,6 +17,11 @@ public class DistributionUtils { private static final Set readOnlyPathPatternSet = createReadOnlyPatternSet(); + /** + * Set of black-listed path patterns (paths matching any of these patterns is supposed to be "modifying" and should be redirected to the master) + */ + private static final Set blackListPathPatternSet = createBlackListPatternSet(); + /** * Check whether the request is a read request. * @@ -25,6 +30,10 @@ public class DistributionUtils { * @return */ public static boolean isReadRequest(HttpMethod method, String path) { + if (isBlackListed(path)) { + return false; + } + switch (method) { case CONNECT: case OPTIONS: @@ -59,6 +68,21 @@ public static boolean isReadOnly(String path) { return false; } + /** + * Check whether the given path matches one of the known black-listed paths. + * @param path path + * @return true if the request to the given path is black-listed (is supposed to be "modifying") + */ + public static boolean isBlackListed(String path) { + for (Pattern pattern : blackListPathPatternSet) { + Matcher m = pattern.matcher(path); + if (m.matches()) { + return true; + } + } + return false; + } + /** * Create the set of read-only patterns * @return pattern set @@ -76,4 +100,19 @@ private static Set createReadOnlyPatternSet() { patterns.add(Pattern.compile("/api/v[0-9]+/utilities/validateMicroschema/?")); return patterns; } + + /** + * Create the set of blacklisted path patterns + * @return pattern set + */ + private static Set createBlackListPatternSet() { + Set patterns = new HashSet<>(); + // clearing the search indices should only be done on the Master + patterns.add(Pattern.compile("/api/v[0-9]+/search/clear")); + // index sync should only be done on the Master + patterns.add(Pattern.compile("/api/v[0-9]+/search/sync")); + // search index operation status should only be fetched from the Master (which is doing the index operations) + patterns.add(Pattern.compile("/api/v[0-9]+/search/status")); + return patterns; + } } diff --git a/common/src/main/java/com/gentics/mesh/distributed/RequestDelegator.java b/common/src/main/java/com/gentics/mesh/distributed/RequestDelegator.java index be264b3b0e..a9958c03b8 100644 --- a/common/src/main/java/com/gentics/mesh/distributed/RequestDelegator.java +++ b/common/src/main/java/com/gentics/mesh/distributed/RequestDelegator.java @@ -23,4 +23,10 @@ public interface RequestDelegator extends Handler { * @param routingContext */ void redirectToMaster(RoutingContext routingContext); + + /** + * Returns true when this instance is the master + * @return true for master + */ + boolean isMaster(); } diff --git a/common/src/main/java/com/gentics/mesh/parameter/ParameterProviderContext.java b/common/src/main/java/com/gentics/mesh/parameter/ParameterProviderContext.java index f78d8e115f..fe4e0760f5 100644 --- a/common/src/main/java/com/gentics/mesh/parameter/ParameterProviderContext.java +++ b/common/src/main/java/com/gentics/mesh/parameter/ParameterProviderContext.java @@ -5,6 +5,7 @@ import com.gentics.mesh.parameter.impl.DeleteParametersImpl; import com.gentics.mesh.parameter.impl.GenericParametersImpl; import com.gentics.mesh.parameter.impl.ImageManipulationParametersImpl; +import com.gentics.mesh.parameter.impl.IndexMaintenanceParametersImpl; import com.gentics.mesh.parameter.impl.NodeParametersImpl; import com.gentics.mesh.parameter.impl.PagingParametersImpl; import com.gentics.mesh.parameter.impl.ProjectPurgeParametersImpl; @@ -71,4 +72,8 @@ default SearchParameters getSearchParameters() { default BackupParameters getBackupParameters() { return new BackupParametersImpl(this); } + + default IndexMaintenanceParameters getIndexMaintenanceParameters() { + return new IndexMaintenanceParametersImpl(this); + } } diff --git a/common/src/main/java/com/gentics/mesh/parameter/impl/IndexMaintenanceParametersImpl.java b/common/src/main/java/com/gentics/mesh/parameter/impl/IndexMaintenanceParametersImpl.java new file mode 100644 index 0000000000..a40a026284 --- /dev/null +++ b/common/src/main/java/com/gentics/mesh/parameter/impl/IndexMaintenanceParametersImpl.java @@ -0,0 +1,49 @@ +package com.gentics.mesh.parameter.impl; + +import java.util.HashMap; +import java.util.Map; + +import org.raml.model.ParamType; +import org.raml.model.parameter.QueryParameter; + +import com.gentics.mesh.handler.ActionContext; +import com.gentics.mesh.parameter.AbstractParameters; +import com.gentics.mesh.parameter.IndexMaintenanceParameters; + +/** + * Parameter implementation for index maintenance parameters + */ +public class IndexMaintenanceParametersImpl extends AbstractParameters implements IndexMaintenanceParameters { + /** + * Create empty instance + */ + public IndexMaintenanceParametersImpl() { + } + + /** + * Create instance with parameters filled from the action context + * @param ac action context + */ + public IndexMaintenanceParametersImpl(ActionContext ac) { + super(ac); + } + + @Override + public String getName() { + return "Index Maintenance Parameters"; + } + + @Override + public Map getRAMLParameters() { + Map parameters = new HashMap<>(); + + // index parameter + QueryParameter indexParameter = new QueryParameter(); + indexParameter.setDescription("Index pattern to handle"); + indexParameter.setExample("node-.*"); + indexParameter.setRequired(false); + indexParameter.setType(ParamType.STRING); + parameters.put(INDEX_PARAMETER_KEY, indexParameter); + return parameters; + } +} diff --git a/common/src/main/java/com/gentics/mesh/search/DevNullSearchProvider.java b/common/src/main/java/com/gentics/mesh/search/DevNullSearchProvider.java index e912fc9edf..c6aa70d3b3 100644 --- a/common/src/main/java/com/gentics/mesh/search/DevNullSearchProvider.java +++ b/common/src/main/java/com/gentics/mesh/search/DevNullSearchProvider.java @@ -113,7 +113,7 @@ public void reset() { } @Override - public Completable clear() { + public Completable clear(String indexPattern) { return Completable.complete(); } diff --git a/common/src/main/java/com/gentics/mesh/search/SearchMappingsCache.java b/common/src/main/java/com/gentics/mesh/search/SearchMappingsCache.java new file mode 100644 index 0000000000..9544ff93cd --- /dev/null +++ b/common/src/main/java/com/gentics/mesh/search/SearchMappingsCache.java @@ -0,0 +1,17 @@ +package com.gentics.mesh.search; + +import com.gentics.mesh.cache.MeshCache; + +import io.vertx.core.json.JsonObject; + +/** + * Interface for the cache of expected search mappings + */ +public interface SearchMappingsCache extends MeshCache { + /** + * Put the value into the cache + * @param key cache key + * @param value cached value + */ + void put(String key, JsonObject value); +} diff --git a/common/src/main/java/com/gentics/mesh/search/SearchProvider.java b/common/src/main/java/com/gentics/mesh/search/SearchProvider.java index 20ee1281d0..eba7609039 100644 --- a/common/src/main/java/com/gentics/mesh/search/SearchProvider.java +++ b/common/src/main/java/com/gentics/mesh/search/SearchProvider.java @@ -150,7 +150,16 @@ public interface SearchProvider { * * @return Completable for the clear action */ - Completable clear(); + default Completable clear() { + return clear(null); + } + + /** + * Delete all indices which are managed by mesh and match the optionally provided index pattern (clear all, if pattern is null) + * @param indexPattern optional index pattern + * @return Completable for the clear action + */ + Completable clear(String indexPattern); /** * Delete the given index and don't fail if the index is not existing. @@ -258,4 +267,13 @@ default JsonObject createIndexSettings(IndexInfo info) { * @return */ boolean isActive(); + + /** + * Check existence and correctness the given index + * @param info index info + * @return completable + */ + default Completable check(IndexInfo info) { + return Completable.complete(); + } } diff --git a/common/src/main/java/com/gentics/mesh/search/TrackingSearchProvider.java b/common/src/main/java/com/gentics/mesh/search/TrackingSearchProvider.java index 243ffe00a8..a286a6cec2 100644 --- a/common/src/main/java/com/gentics/mesh/search/TrackingSearchProvider.java +++ b/common/src/main/java/com/gentics/mesh/search/TrackingSearchProvider.java @@ -191,7 +191,7 @@ public void reset() { } @Override - public Completable clear() { + public Completable clear(String indexPattern) { updateEvents.clear(); deleteEvents.clear(); storeEvents.clear(); diff --git a/core/src/main/java/com/gentics/mesh/cli/BootstrapInitializerImpl.java b/core/src/main/java/com/gentics/mesh/cli/BootstrapInitializerImpl.java index a0747f290d..70191c2a77 100644 --- a/core/src/main/java/com/gentics/mesh/cli/BootstrapInitializerImpl.java +++ b/core/src/main/java/com/gentics/mesh/cli/BootstrapInitializerImpl.java @@ -628,7 +628,7 @@ private void handleLocalData(PostProcessFlags flags, MeshOptions configuration, } if (isSearchEnabled && (flags.isReindex() || flags.isResync())) { - SyncEventHandler.invokeSync(vertx); + SyncEventHandler.invokeSync(vertx, null); } // Handle admin password reset diff --git a/core/src/main/java/com/gentics/mesh/dagger/module/BindModule.java b/core/src/main/java/com/gentics/mesh/dagger/module/BindModule.java index e5699e0c56..0eb23dbab2 100644 --- a/core/src/main/java/com/gentics/mesh/dagger/module/BindModule.java +++ b/core/src/main/java/com/gentics/mesh/dagger/module/BindModule.java @@ -48,6 +48,8 @@ import com.gentics.mesh.plugin.pf4j.PluginEnvironmentImpl; import com.gentics.mesh.plugin.registry.DelegatingPluginRegistry; import com.gentics.mesh.plugin.registry.DelegatingPluginRegistryImpl; +import com.gentics.mesh.search.SearchMappingsCache; +import com.gentics.mesh.search.impl.SearchMappingsCacheImpl; import com.gentics.mesh.search.index.BucketManager; import com.gentics.mesh.search.index.BucketManagerImpl; import com.gentics.mesh.search.index.common.DropIndexHandler; @@ -146,4 +148,7 @@ public abstract class BindModule { @Binds abstract LivenessManager bindLivenessManager(LivenessManagerImpl e); + + @Binds + abstract SearchMappingsCache searchMappingsCache(SearchMappingsCacheImpl e); } diff --git a/core/src/main/java/com/gentics/mesh/rest/MeshLocalClientImpl.java b/core/src/main/java/com/gentics/mesh/rest/MeshLocalClientImpl.java index b400b3a512..09d2b4a5e0 100644 --- a/core/src/main/java/com/gentics/mesh/rest/MeshLocalClientImpl.java +++ b/core/src/main/java/com/gentics/mesh/rest/MeshLocalClientImpl.java @@ -1047,15 +1047,15 @@ public MeshRequest searchMicroschemasRaw(String json) { } @Override - public MeshRequest invokeIndexClear() { - LocalActionContextImpl ac = createContext(GenericMessageResponse.class); + public MeshRequest invokeIndexClear(ParameterProvider... parameters) { + LocalActionContextImpl ac = createContext(GenericMessageResponse.class, parameters); adminIndexHandler.handleClear(ac); return new MeshLocalRequestImpl<>(ac.getFuture()); } @Override - public MeshRequest invokeIndexSync() { - LocalActionContextImpl ac = createContext(GenericMessageResponse.class); + public MeshRequest invokeIndexSync(ParameterProvider... parameters) { + LocalActionContextImpl ac = createContext(GenericMessageResponse.class, parameters); adminIndexHandler.handleSync(ac); return new MeshLocalRequestImpl<>(ac.getFuture()); } diff --git a/core/src/main/java/com/gentics/mesh/search/SearchEndpointImpl.java b/core/src/main/java/com/gentics/mesh/search/SearchEndpointImpl.java index 2a67702ac2..c275eadde6 100644 --- a/core/src/main/java/com/gentics/mesh/search/SearchEndpointImpl.java +++ b/core/src/main/java/com/gentics/mesh/search/SearchEndpointImpl.java @@ -27,6 +27,7 @@ import com.gentics.mesh.core.rest.tag.TagListResponse; import com.gentics.mesh.core.rest.user.UserListResponse; import com.gentics.mesh.graphdb.spi.Database; +import com.gentics.mesh.parameter.impl.IndexMaintenanceParametersImpl; import com.gentics.mesh.parameter.impl.PagingParametersImpl; import com.gentics.mesh.parameter.impl.SearchParametersImpl; import com.gentics.mesh.rest.InternalEndpointRoute; @@ -164,6 +165,7 @@ private void addAdminHandlers() { indexClearEndpoint.path("/clear"); indexClearEndpoint.method(POST); indexClearEndpoint.produces(APPLICATION_JSON); + indexClearEndpoint.addQueryParameters(IndexMaintenanceParametersImpl.class); indexClearEndpoint.description("Drops all indices and recreates them. The index sync is not invoked automatically."); indexClearEndpoint.exampleResponse(OK, miscExamples.createMessageResponse(), "Recreated all indices."); indexClearEndpoint.handler(rc -> { @@ -175,6 +177,7 @@ private void addAdminHandlers() { indexSyncEndpoint.path("/sync"); indexSyncEndpoint.method(POST); indexSyncEndpoint.produces(APPLICATION_JSON); + indexSyncEndpoint.addQueryParameters(IndexMaintenanceParametersImpl.class); indexSyncEndpoint.description( "Invokes the manual synchronisation of the search indices. This operation may take some time to complete and is performed asynchronously. When clustering is enabled it will be executed on any free instance."); indexSyncEndpoint.exampleResponse(OK, miscExamples.createMessageResponse(), "Invoked index synchronisation on all indices."); diff --git a/core/src/test/java/com/gentics/mesh/search/index/BasicIndexSyncTest.java b/core/src/test/java/com/gentics/mesh/search/index/BasicIndexSyncTest.java index 32b3708968..b564b18688 100644 --- a/core/src/test/java/com/gentics/mesh/search/index/BasicIndexSyncTest.java +++ b/core/src/test/java/com/gentics/mesh/search/index/BasicIndexSyncTest.java @@ -6,37 +6,49 @@ import static com.gentics.mesh.test.context.ElasticsearchTestMode.CONTAINER_ES6; import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; import static io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.junit.Assert.assertEquals; -import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; +import com.gentics.elasticsearch.client.ElasticsearchClient; +import com.gentics.elasticsearch.client.HttpErrorException; import com.gentics.mesh.context.impl.BulkActionContextImpl; import com.gentics.mesh.core.data.NodeGraphFieldContainer; import com.gentics.mesh.core.data.Project; import com.gentics.mesh.core.data.node.Node; import com.gentics.mesh.core.data.schema.MicroschemaContainer; import com.gentics.mesh.core.data.schema.SchemaContainer; +import com.gentics.mesh.core.rest.MeshEvent; import com.gentics.mesh.core.rest.common.ContainerType; import com.gentics.mesh.core.rest.common.GenericMessageResponse; import com.gentics.mesh.core.rest.microschema.MicroschemaModel; import com.gentics.mesh.core.rest.microschema.impl.MicroschemaModelImpl; import com.gentics.mesh.core.rest.project.ProjectCreateRequest; +import com.gentics.mesh.core.rest.project.ProjectResponse; import com.gentics.mesh.core.rest.schema.SchemaModel; import com.gentics.mesh.core.rest.schema.impl.SchemaCreateRequest; import com.gentics.mesh.core.rest.schema.impl.SchemaModelImpl; import com.gentics.mesh.core.rest.schema.impl.SchemaResponse; import com.gentics.mesh.core.rest.search.EntityMetrics; -import com.gentics.mesh.core.rest.search.SearchStatusResponse; +import com.gentics.mesh.core.rest.user.UserResponse; import com.gentics.mesh.event.EventQueueBatch; -import com.gentics.mesh.search.verticle.eventhandler.SyncEventHandler; import com.gentics.mesh.test.TestSize; import com.gentics.mesh.test.context.AbstractMeshTest; import com.gentics.mesh.test.context.MeshTestSetting; +import com.gentics.mesh.test.context.helper.ExpectedEvent; +import com.gentics.mesh.test.context.helper.UnexpectedEvent; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.reactivex.Flowable; +import io.vertx.core.json.JsonObject; /** * Test differential sync of elasticsearch. */ @@ -344,6 +356,127 @@ public void testMicroschemaSync() throws Exception { assertMetrics("microschema", 0, 0, 1); } + /** + * Test that the check for index existence and correctness will not change anything if not necessary + * @throws Exception + */ + @Test + public void testIndexSyncCheckNoChange() throws Exception { + int timeoutMs = 10_000; + grantAdmin(); + searchProvider().refreshIndex().blockingAwait(); + + // trigger the check by publishing the event, and expect the "check finished" but not the "sync finished" events to be thrown + try (ExpectedEvent finished = expectEvent(MeshEvent.INDEX_CHECK_FINISHED, timeoutMs); + UnexpectedEvent syncFinished = notExpectEvent(MeshEvent.INDEX_SYNC_FINISHED, timeoutMs)) { + vertx().eventBus().publish(MeshEvent.INDEX_CHECK_REQUEST.address, null); + } + } + + /** + * Test that the check for index mapping will recreate and repopulate a dropped index + * @throws Exception + */ + @Test + public void testAutoIndexRecreation() throws Exception { + int timeoutMs = 10_000; + + // name of the index (without installation prefix) + String meshIndexName = "project"; + String esIndexName = options().getSearchOptions().getPrefix() + meshIndexName; + + ElasticsearchClient client = searchProvider().getClient(); + grantAdmin(); + searchProvider().refreshIndex().blockingAwait(); + + // read all project uuids + Set projectUuids = call(() -> client().findProjects()).getData().stream().map(ProjectResponse::getUuid) + .collect(Collectors.toSet()); + assertThat(projectUuids).isNotEmpty(); + + // read the (correct) mapping for comparing later + JsonObject mappings = getIndexMappings(esIndexName); + + // drop the index + client.deleteIndex(esIndexName).sync(); + + try { + client.readIndex(esIndexName).sync(); + fail("Index " + esIndexName + " should have been deleted"); + } catch (HttpErrorException e) { + // everything else than the expected NOT FOUND is re-thrown + if (e.statusCode != HttpResponseStatus.NOT_FOUND.code()) { + throw e; + } + } + + // trigger the check by publishing the event, and expect the "check finished" and the "sync finished" events to be thrown + try (ExpectedEvent finished = expectEvent(MeshEvent.INDEX_CHECK_FINISHED, timeoutMs); + ExpectedEvent syncFinished = expectEvent(MeshEvent.INDEX_SYNC_FINISHED, timeoutMs)) { + vertx().eventBus().publish(MeshEvent.INDEX_CHECK_REQUEST.address, null); + } + + // read the index and compare the mappings with the original mappings + assertThat(getIndexMappings(esIndexName)).isEqualTo(mappings); + + // read the documents from the index + Flowable.fromIterable(projectUuids).flatMapSingle(uuid -> { + return client.readDocument(esIndexName, uuid).async(); + }).blockingSubscribe(); + } + + /** + * Test that the check for index existence and correctness will drop, recreate and repopulate an incorrect index + * @throws Exception + */ + @Test + public void testAutoIndexFix() throws Exception { + int timeoutMs = 10_000; + + // name of the index (without installation prefix) + String meshIndexName = "user"; + String esIndexName = options().getSearchOptions().getPrefix() + meshIndexName; + + ElasticsearchClient client = searchProvider().getClient(); + grantAdmin(); + searchProvider().refreshIndex().blockingAwait(); + + // read all user uuids + Set userUuids = call(() -> client().findUsers()).getData().stream().map(UserResponse::getUuid) + .collect(Collectors.toSet()); + assertThat(userUuids).isNotEmpty(); + + // read the (correct) mapping for comparing later + JsonObject mappings = getIndexMappings(esIndexName); + + // drop the index for "project" + client.deleteIndex(esIndexName).sync(); + // create with default mappings by storing a dummy document + client.storeDocument(esIndexName, "dummy", new JsonObject("{\"name\": \"dummy\"}")).sync(); + // index mappings should now be different + assertThat(getIndexMappings(esIndexName)).isNotEqualTo(mappings); + + // trigger the check by publishing the event, and expect the "check finished" and the "sync finished" events to be thrown + try (ExpectedEvent finished = expectEvent(MeshEvent.INDEX_CHECK_FINISHED, timeoutMs); + ExpectedEvent syncFinished = expectEvent(MeshEvent.INDEX_SYNC_FINISHED, timeoutMs)) { + vertx().eventBus().publish(MeshEvent.INDEX_CHECK_REQUEST.address, null); + } + + // read the index and compare the mappings with the original mappings + assertThat(getIndexMappings(esIndexName)).isEqualTo(mappings); + + // read the documents from the index + Flowable.fromIterable(userUuids).flatMapSingle(uuid -> { + return client.readDocument(esIndexName, uuid).async(); + }).blockingSubscribe(); + + // trigger the check by publishing the event, and expect the "check finished" but not the "sync finished" events to be thrown + try (ExpectedEvent finished = expectEvent(MeshEvent.INDEX_CHECK_FINISHED, timeoutMs); + UnexpectedEvent syncFinished = notExpectEvent(MeshEvent.INDEX_SYNC_FINISHED, timeoutMs)) { + vertx().eventBus().publish(MeshEvent.INDEX_CHECK_REQUEST.address, null); + } + } + private void assertMetrics(String type, long inserted, long updated, long deleted) { EntityMetrics entityMetrics = call(() -> client().searchStatus()).getMetrics().get(type); assertEquals(inserted, entityMetrics.getInsert().getSynced().longValue()); @@ -354,10 +487,4 @@ private void assertMetrics(String type, long inserted, long updated, long delete assertEquals(0, entityMetrics.getUpdate().getPending().longValue()); assertEquals(0, entityMetrics.getDelete().getPending().longValue()); } - - private void syncIndex() { - waitForEvent(INDEX_SYNC_FINISHED, () -> SyncEventHandler.invokeSync(vertx())); - refreshIndices(); - } - } diff --git a/core/src/test/java/com/gentics/mesh/search/index/IndexClearTest.java b/core/src/test/java/com/gentics/mesh/search/index/IndexClearTest.java index 76fff1323b..9a88d16999 100644 --- a/core/src/test/java/com/gentics/mesh/search/index/IndexClearTest.java +++ b/core/src/test/java/com/gentics/mesh/search/index/IndexClearTest.java @@ -5,24 +5,31 @@ import static com.gentics.mesh.test.ClientHelper.call; import static com.gentics.mesh.test.context.ElasticsearchTestMode.CONTAINER_ES6; import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import org.junit.Before; import org.junit.Test; -import com.gentics.elasticsearch.client.HttpErrorException; +import com.gentics.mesh.core.data.Project; import com.gentics.mesh.core.data.User; import com.gentics.mesh.core.rest.common.GenericMessageResponse; +import com.gentics.mesh.parameter.client.IndexMaintenanceParametersImpl; import com.gentics.mesh.search.verticle.eventhandler.SyncEventHandler; import com.gentics.mesh.test.TestSize; import com.gentics.mesh.test.context.AbstractMeshTest; import com.gentics.mesh.test.context.MeshTestSetting; + @MeshTestSetting(elasticsearch = CONTAINER_ES6, testSize = TestSize.FULL, startServer = true) public class IndexClearTest extends AbstractMeshTest { + @Before + public void setup() throws Exception { + getProvider().clear().blockingAwait(); + syncIndex(); + revokeAdmin(); + } @Test public void testClear() throws Exception { - waitForEvent(INDEX_SYNC_FINISHED, () -> SyncEventHandler.invokeSync(vertx())); + waitForEvent(INDEX_SYNC_FINISHED, () -> SyncEventHandler.invokeSync(vertx(), null)); call(() -> client().invokeIndexClear(), FORBIDDEN, "error_admin_permission_required"); grantAdmin(); @@ -30,13 +37,47 @@ public void testClear() throws Exception { GenericMessageResponse message = call(() -> client().invokeIndexClear()); assertThat(message).matches("search_admin_index_clear"); - try { - getProvider().getDocument(User.composeIndexName(), userUuid()).blockingGet(); - fail("An error should occur"); - } catch (Exception e) { - HttpErrorException error = (HttpErrorException) e.getCause(); - assertEquals(404, error.getStatusCode()); - } + assertDocumentDoesNotExist(User.composeIndexName(), User.composeDocumentId(userUuid())); + + } + + /** + * Test that clearing the index restricted by name only clears the specified index + * @throws Exception + */ + @Test + public void testClearWithName() throws Exception { + runClearTest(false); + } + + /** + * Test that clearing indices can also be done with the full name (including the installation prefix) + * @throws Exception + */ + @Test + public void testClearWithFullName() throws Exception { + runClearTest(true); + } + + /** + * Run the clear test + * @param prefix true to use the prefixed index name, false to use the bare index name + * @throws Exception + */ + protected void runClearTest(boolean prefix) throws Exception { + String index = prefix ? "mesh-user" : "user"; + grantAdmin(); + + // check that the project is found in index + assertDocumentExists(Project.composeIndexName(), Project.composeDocumentId(projectUuid())); + // check that the user is found in index + assertDocumentExists(User.composeIndexName(), User.composeDocumentId(userUuid())); + + call(() -> client().invokeIndexClear(new IndexMaintenanceParametersImpl().setIndex(index))); + // check that the project is still found in index + assertDocumentExists(Project.composeIndexName(), Project.composeDocumentId(projectUuid())); + // check that the user is no longer found in index + assertDocumentDoesNotExist(User.composeIndexName(), User.composeDocumentId(userUuid())); } } diff --git a/core/src/test/java/com/gentics/mesh/search/index/IndexSyncCleanupTest.java b/core/src/test/java/com/gentics/mesh/search/index/IndexSyncCleanupTest.java index 58588acd10..a9ef323abf 100644 --- a/core/src/test/java/com/gentics/mesh/search/index/IndexSyncCleanupTest.java +++ b/core/src/test/java/com/gentics/mesh/search/index/IndexSyncCleanupTest.java @@ -28,9 +28,12 @@ import com.gentics.mesh.core.data.schema.MicroschemaContainer; import com.gentics.mesh.core.data.schema.SchemaContainer; import com.gentics.mesh.core.data.search.index.IndexInfo; +import com.gentics.mesh.core.rest.MeshEvent; +import com.gentics.mesh.parameter.client.IndexMaintenanceParametersImpl; import com.gentics.mesh.test.TestSize; import com.gentics.mesh.test.context.AbstractMeshTest; import com.gentics.mesh.test.context.MeshTestSetting; +import com.gentics.mesh.test.context.helper.ExpectedEvent; import io.vertx.core.json.JsonObject; @@ -38,7 +41,9 @@ public class IndexSyncCleanupTest extends AbstractMeshTest { @Before - public void setup() { + public void setup() throws Exception { + getProvider().clear().blockingAwait(); + syncIndex(); grantAdmin(); } @@ -95,6 +100,49 @@ public void testIndexPurge() throws Exception { } + /** + * Test that synchronizing indices restricted to name only synchronizes the specified index + * @throws Exception + */ + @Test + public void testSyncWithName() throws Exception { + runSyncTest(false); + } + + /** + * Test that synchronizing indices restricted to name only synchronizes the specified index + * @throws Exception + */ + @Test + public void testSyncWithFullName() throws Exception { + runSyncTest(true); + } + + /** + * Run the sync test + * @param prefix true to use the prefixed index name, false to use the bare index name + * @throws Exception + */ + protected void runSyncTest(boolean prefix) throws Exception { + int timeout = 10_000; + String index = prefix ? "mesh-user" : "user"; + + // drop indices for user and project + deleteIndex("mesh-user", "mesh-project"); + // recreate invalid indices for user and project + createThirdPartyIndex("mesh-" + User.composeIndexName()); + createThirdPartyIndex("mesh-" + Project.composeIndexName()); + + try (ExpectedEvent syncFinished = expectEvent(MeshEvent.INDEX_SYNC_FINISHED, timeout)) { + call(() -> client().invokeIndexSync(new IndexMaintenanceParametersImpl().setIndex(index))); + } + + // check that the project is not found in index + assertDocumentDoesNotExist(Project.composeIndexName(), Project.composeDocumentId(projectUuid())); + // check that the user is found in index + assertDocumentExists(User.composeIndexName(), User.composeDocumentId(userUuid())); + } + private void createThirdPartyIndex(String name) throws HttpErrorException { ElasticsearchClient searchClient = searchProvider().getClient(); JsonObject response = searchClient.createIndex(name, new JsonObject()).sync(); @@ -105,6 +153,11 @@ private void createIndex(String name) { searchProvider().createIndex(new IndexInfo(name, new JsonObject(), new JsonObject(), "")).blockingAwait(); } + private void deleteIndex(String...indexNames) throws HttpErrorException { + ElasticsearchClient searchClient = searchProvider().getClient(); + searchClient.deleteIndex(indexNames).sync(); + } + public Set indices() throws HttpErrorException { ElasticsearchClient searchClient = searchProvider().getClient(); JsonObject indicesAfter = searchClient.readIndex("*").sync(); diff --git a/core/src/test/java/com/gentics/mesh/test/context/MeshTestContext.java b/core/src/test/java/com/gentics/mesh/test/context/MeshTestContext.java index ad0d95c4ef..e438a0052e 100644 --- a/core/src/test/java/com/gentics/mesh/test/context/MeshTestContext.java +++ b/core/src/test/java/com/gentics/mesh/test/context/MeshTestContext.java @@ -470,6 +470,10 @@ public MeshOptions init(MeshTestSetting settings) throws Exception { if (settings == null) { throw new RuntimeException("Settings could not be found. Did you forgot to add the @MeshTestSetting annotation to your test?"); } + + // disable periodic index check + meshOptions.getSearchOptions().setIndexCheckInterval(0); + // Clustering options if (settings.clusterMode()) { meshOptions.getClusterOptions().setEnabled(true); diff --git a/core/src/test/java/com/gentics/mesh/test/context/helper/EventHelper.java b/core/src/test/java/com/gentics/mesh/test/context/helper/EventHelper.java index dbeb55bf65..6b1709b6e5 100644 --- a/core/src/test/java/com/gentics/mesh/test/context/helper/EventHelper.java +++ b/core/src/test/java/com/gentics/mesh/test/context/helper/EventHelper.java @@ -2,6 +2,8 @@ import static com.gentics.mesh.core.rest.job.JobStatus.COMPLETED; import static com.gentics.mesh.test.ClientHelper.call; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import java.util.List; import java.util.concurrent.Callable; @@ -11,15 +13,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import javax.jws.soap.SOAPBinding.Use; - +import com.gentics.elasticsearch.client.ElasticsearchClient; +import com.gentics.elasticsearch.client.HttpErrorException; import com.gentics.mesh.cli.BootstrapInitializerImpl; import com.gentics.mesh.core.rest.MeshEvent; import com.gentics.mesh.core.rest.job.JobListResponse; import com.gentics.mesh.core.rest.job.JobResponse; import com.gentics.mesh.core.rest.job.JobStatus; import com.gentics.mesh.parameter.client.PagingParametersImpl; -import com.gentics.mesh.plugin.BackupPlugin; import com.gentics.mesh.search.verticle.ElasticsearchProcessVerticle; import com.gentics.mesh.search.verticle.eventhandler.SyncEventHandler; import com.gentics.mesh.test.context.ClientHandler; @@ -30,6 +31,7 @@ import io.reactivex.Completable; import io.reactivex.functions.Action; import io.vertx.core.eventbus.MessageConsumer; +import io.vertx.core.json.JsonObject; public interface EventHelper extends BaseHelper { @@ -138,6 +140,24 @@ default ExpectedEvent expectEvent(String address, Action code, int timeoutMs) { return new ExpectedEvent(vertx(), address, code, timeoutMs); } + /** + * Create an {@link AutoCloseable} which will register an event handler for the event (when created) and + * will wait for the event to be fired in {@link AutoCloseable#close()}. If the event is fired, it will throw an Exception.
+ * This method should be used like this: + * 
+	 * try (UnexpectedEvent ue = notExpectEvent(MeshEvent.PLUGIN_REGISTERED, 10_000)) {
+	 *   // code, which is expected to not fire the event
+	 *   ...
+	 * }
+	 *
+ * @param event event + * @param timeoutMs timeout in milliseconds. The timeout should not be set too high, because successful test execution will be blocked for the length of the timeout. + * @return AutoClosable instance + */ + default UnexpectedEvent notExpectEvent(MeshEvent event, int timeoutMs) { + return new UnexpectedEvent(vertx(), event.getAddress(), timeoutMs); + } + default void waitForSearchIdleEvent() { getTestContext().waitForSearchIdleEvent(); } @@ -457,4 +477,36 @@ default void awaitEvents() { eventAsserter().await(); } + default void assertDocumentExists(String indexName, String documentId) { + getProvider().getDocument(indexName, documentId).blockingGet(); + } + + default void assertDocumentDoesNotExist(String indexName, String documentId) { + try { + getProvider().getDocument(indexName, documentId).blockingGet(); + fail("Fetching document " + documentId + " from index " + indexName + " is expected to fail"); + } catch (Exception e) { + HttpErrorException error = (HttpErrorException) e.getCause(); + assertEquals(404, error.getStatusCode()); + } + } + + default void syncIndex() throws TimeoutException { + try (ExpectedEvent ee = expectEvent(MeshEvent.INDEX_SYNC_FINISHED, 10_000)) { + SyncEventHandler.invokeSync(vertx(), null); + } + refreshIndices(); + } + + /** + * Get the index mappings for the index with given name. Method will fail, if index does not exist + * @param indexName elasticsearch index name (including prefix) + * @return index mappings + */ + default JsonObject getIndexMappings(String indexName) { + ElasticsearchClient client = searchProvider().getClient(); + return client.readIndex(indexName).async().map(response -> { + return response.getJsonObject(indexName).getJsonObject("mappings"); + }).blockingGet(); + } } diff --git a/core/src/test/java/com/gentics/mesh/test/context/helper/ExpectedEvent.java b/core/src/test/java/com/gentics/mesh/test/context/helper/ExpectedEvent.java index 8ec16e92a4..5a7c5351a7 100644 --- a/core/src/test/java/com/gentics/mesh/test/context/helper/ExpectedEvent.java +++ b/core/src/test/java/com/gentics/mesh/test/context/helper/ExpectedEvent.java @@ -11,7 +11,7 @@ /** * AutoClosable implementation, which will register an event handler upon * creation and will wait (up to the given timeout) in - * {@link ExpectedEvent#clone()} for the event to be fired at least once. + * {@link ExpectedEvent#close()} for the event to be fired at least once. * See {@link EventHelper#expectEvent(com.gentics.mesh.core.rest.MeshEvent, int)} for usage. */ public class ExpectedEvent implements AutoCloseable { @@ -53,7 +53,8 @@ public void close() throws TimeoutException { } } catch (InterruptedException e) { throw new RuntimeException(e); + } finally { + consumer.unregister(); } - consumer.unregister(); } } diff --git a/core/src/test/java/com/gentics/mesh/test/context/helper/UnexpectedEvent.java b/core/src/test/java/com/gentics/mesh/test/context/helper/UnexpectedEvent.java new file mode 100644 index 0000000000..3db08966d9 --- /dev/null +++ b/core/src/test/java/com/gentics/mesh/test/context/helper/UnexpectedEvent.java @@ -0,0 +1,52 @@ +package com.gentics.mesh.test.context.helper; + +import static org.assertj.core.api.Assertions.fail; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import io.vertx.core.Vertx; +import io.vertx.core.eventbus.MessageConsumer; + +/** + * AutoClosable implementation, which will register an event handler upon + * creation and will wait (up to the given timeout) in + * {@link UnexpectedEvent#close()} for the event to be fired. If the event was fired, an exception is thrown. + * See {@link EventHelper#notExpectEvent(com.gentics.mesh.core.rest.MeshEvent, int)} for usage. + */ + +public class UnexpectedEvent implements AutoCloseable { + protected CountDownLatch latch = new CountDownLatch(1); + + protected String address; + + protected int timeoutMs; + + protected MessageConsumer consumer; + + /** + * Create an instance and register the event handler + * @param vertx vertx instance + * @param address event address + * @param timeoutMs timeout in milliseconds + */ + public UnexpectedEvent(Vertx vertx, String address, int timeoutMs) { + this.address = address; + this.timeoutMs = timeoutMs; + consumer = vertx.eventBus().consumer(address); + consumer.handler(msg -> latch.countDown()); + } + + @Override + public void close() { + try { + if (latch.await(timeoutMs, TimeUnit.MILLISECONDS)) { + fail("Event " + address + " was handled at least once within timeout"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } finally { + consumer.unregister(); + } + } +} diff --git a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/Coordinator.java b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/Coordinator.java index 23c0467271..5f8a3930a9 100644 --- a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/Coordinator.java +++ b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/Coordinator.java @@ -22,6 +22,10 @@ public Coordinator(MasterElector elector, MeshOptions options) { this.mode = options.getClusterOptions().getCoordinatorMode(); } + /** + * Get the current master server, may be null + * @return current master, may be null + */ public MasterServer getMasterMember() { return elector.getMasterMember(); } @@ -69,6 +73,11 @@ public boolean isElectable() { * @return */ public boolean isMaster() { - return getMasterMember().isSelf(); + MasterServer masterMember = getMasterMember(); + if (masterMember != null) { + return masterMember.isSelf(); + } else { + return false; + } } } diff --git a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/MasterElector.java b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/MasterElector.java index 5a3b0ad10b..90750ca795 100644 --- a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/MasterElector.java +++ b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/MasterElector.java @@ -305,6 +305,10 @@ public Member localMember() { return hazelcast.get().getCluster().getLocalMember(); } + /** + * Get the server, which is currently the master, may be null + * @return current master, may be null + */ public MasterServer getMasterMember() { if (masterMember == null) { return null; diff --git a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/proxy/RequestDelegatorImpl.java b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/proxy/RequestDelegatorImpl.java index 049c47cbae..305ca9470e 100644 --- a/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/proxy/RequestDelegatorImpl.java +++ b/distributed-coordinator/src/main/java/com/gentics/mesh/distributed/coordinator/proxy/RequestDelegatorImpl.java @@ -169,6 +169,11 @@ public void redirectToMaster(RoutingContext rc) { } } + @Override + public boolean isMaster() { + return coordinator.isMaster(); + } + /** * Log the given messages with loglevel TRACE. * diff --git a/doc/src/main/docs/generated/tables/IndexMaintenanceParametersImpl.adoc-include b/doc/src/main/docs/generated/tables/IndexMaintenanceParametersImpl.adoc-include new file mode 100644 index 0000000000..dc09775009 --- /dev/null +++ b/doc/src/main/docs/generated/tables/IndexMaintenanceParametersImpl.adoc-include @@ -0,0 +1,15 @@ +[options="header",cols="10%,20%,10%,60%"] +|====== + +| Name +| Type +| Mandatory +| Description + + +| index +| string +| false +| Index pattern to handle + +|====== diff --git a/doc/src/main/docs/generated/tables/SearchIndexSyncEventModel.adoc-include b/doc/src/main/docs/generated/tables/SearchIndexSyncEventModel.adoc-include new file mode 100644 index 0000000000..a62258292d --- /dev/null +++ b/doc/src/main/docs/generated/tables/SearchIndexSyncEventModel.adoc-include @@ -0,0 +1,25 @@ +[options="header",cols="10%,10%,10%,70%"] +|====== + +| Property +| Mandatory +| Type +| Description + + +| cause +| true +| object +| Some events will be caused by another action. This object contains information about the cause of the event. + +| indexPattern +| true +| string +| Index pattern + +| origin +| true +| string +| Name of the mesh node from which the event originates. + +|====== diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/impl/ElasticSearchProvider.java b/elasticsearch/src/main/java/com/gentics/mesh/search/impl/ElasticSearchProvider.java index d8169977da..cd3b9d30ca 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/impl/ElasticSearchProvider.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/impl/ElasticSearchProvider.java @@ -11,17 +11,25 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Singleton; +import org.apache.commons.lang3.StringUtils; + import com.gentics.elasticsearch.client.ElasticsearchClient; import com.gentics.elasticsearch.client.HttpErrorException; +import com.gentics.mesh.core.data.search.IndexHandler; import com.gentics.mesh.core.data.search.bulk.BulkEntry; import com.gentics.mesh.core.data.search.index.IndexInfo; import com.gentics.mesh.core.data.search.request.Bulkable; @@ -29,7 +37,9 @@ import com.gentics.mesh.etc.config.search.ComplianceMode; import com.gentics.mesh.etc.config.search.ElasticSearchOptions; import com.gentics.mesh.search.ElasticsearchProcessManager; +import com.gentics.mesh.search.SearchMappingsCache; import com.gentics.mesh.search.SearchProvider; +import com.gentics.mesh.search.verticle.eventhandler.SyncEventHandler; import com.gentics.mesh.util.UUIDUtil; import dagger.Lazy; @@ -70,12 +80,15 @@ public class ElasticSearchProvider implements SearchProvider { private final ComplianceMode complianceMode; + private final Lazy searchMappingsCache; + @Inject - public ElasticSearchProvider(Lazy vertx, MeshOptions options, ElasticsearchClient client) { + public ElasticSearchProvider(Lazy vertx, MeshOptions options, ElasticsearchClient client, Lazy searchMappingsCache) { this.vertx = vertx; this.options = options; this.client = client; this.complianceMode = options.getSearchOptions().getComplianceMode(); + this.searchMappingsCache = searchMappingsCache; } /** @@ -143,7 +156,18 @@ public JsonObject getDefaultIndexSettings() { } @Override - public Completable clear() { + public Completable clear(String indexPattern) { + Pattern pattern = null; + if (indexPattern != null) { + indexPattern = StringUtils.removeStartIgnoreCase(indexPattern, options.getSearchOptions().getPrefix()); + try { + pattern = Pattern.compile(indexPattern); + } catch (PatternSyntaxException e) { + log.warn("Index pattern {} is not valid, synchronizing all indices", e, indexPattern); + } + } + Optional optPattern = Optional.ofNullable(pattern); + String prefix = installationPrefix(); // Read all indices and locate indices which have been created for/by mesh. Completable clearIndices = client.readIndex("_all").async() @@ -163,6 +187,8 @@ public Completable clear() { } } return Observable.fromIterable(indices); + }).filter(index -> { + return optPattern.orElse(IndexHandler.MATCH_ALL).matcher(index).matches(); }).flatMapCompletable(index -> { // Now delete the found indices log.debug("Deleting index {" + index + "}"); @@ -412,9 +438,12 @@ public Completable deleteIndex(boolean failOnMissingIndex, String... indexNames) if (log.isDebugEnabled()) { log.debug("Deleted index {" + indices + "}. Duration " + (System.currentTimeMillis() - start) + "[ms]"); } - }).ignoreElement() - .onErrorResumeNext(ignore404) - .compose(withTimeoutAndLog("Deletion of indices " + indices, true)); + }).ignoreElement(); + + if (!failOnMissingIndex) { + deleteIndex = deleteIndex.onErrorResumeNext(ignore404); + } + deleteIndex = deleteIndex.compose(withTimeoutAndLog("Deletion of indices " + indices, !failOnMissingIndex)); return deleteIndex; } @@ -558,6 +587,119 @@ public boolean isActive() { return client != null; } + @Override + public Completable check(IndexInfo info) { + String indexName = installationPrefix() + info.getIndexName(); + + return client.readIndex(indexName).async().flatMapCompletable(response -> { + JsonObject existing = response.getJsonObject(indexName).getJsonObject("mappings"); + cleanMappings(existing); + + return getExpectedMapping(info).flatMapCompletable(expected -> { + // make a copy to not change the original expected (which is cached and will be used again) + expected = expected.copy(); + cleanMappings(expected); + + // merge the existing mapping into the expected mapping, because ES may have added things (dynamic mapping) when documents were stored + expected.mergeIn(existing, true); + if (expected.equals(existing)) { + log.debug(indexName + ": Mapping is ok"); + return Completable.complete(); + } else { + log.warn(indexName + ": Mapping is NOT OK. Index will be dropped and re-created."); + if (log.isDebugEnabled()) { + log.debug(indexName + ": Expected mapping " + expected + ", but found mapping " + existing); + } + // trigger resync for the index + return client.deleteIndex(indexName).async().ignoreElement().andThen(createIndex(info)).andThen(resync(info.getIndexName())); + } + }); + }).onErrorResumeNext(error -> { + if (isNotFoundError(error)) { + return createIndex(info).andThen(resync(info.getIndexName())); + } else { + log.error("Error while checking index " + indexName, error); + return Completable.complete(); + } + }).compose(withTimeoutAndLog("Check mappings of index {" + indexName + "}", false)); + } + + /** + * Initiate (but do not wait for end of) resync of the given index and complete + * @param indexName index name (without the installation prefix) + * @return completable + */ + protected Completable resync(String indexName) { + return Completable.fromAction(() -> { + SyncEventHandler.invokeSync(vertx.get(), indexName); + }); + } + + /** + * Get the expected mapping for the index. + * If the expected mapping is not found in the cache, it will be determined like this: + *
    + *
  1. A temporary index with the settings and mappings is created
    2. + *
  2. The mappings are read from ES for that temporary index
    3. + *
  3. The temporary index is deleted
    4. + *
+ * The reason for this is that ES will not store the index mapping exactly like the mapping was POSTed when creating the index. + * @param info index info + * @return single mappings as JsonObject + */ + protected Single getExpectedMapping(IndexInfo info) { + String cacheKey = info.getIndexName(); + + JsonObject cachedMappings = searchMappingsCache.get().get(cacheKey); + if (cachedMappings != null) { + return Single.just(cachedMappings); + } + + JsonObject json = createIndexSettings(info); + + String randomName = info.getIndexName() + UUIDUtil.randomUUID(); + String tempIndexName = randomName.toLowerCase(); + + return client.createIndex(tempIndexName, json).async() + .doOnSuccess(response -> { + if (log.isDebugEnabled()) { + log.debug("Created temporary index {" + tempIndexName + "} response: {" + response.toString() + "}"); + } + }) + .doOnError(error -> { + log.error("Error getting index mapping for index " + cacheKey, error); + }) + .flatMap(response -> client.readIndex(tempIndexName).async()) + .flatMap(response -> { + JsonObject mappings = response.getJsonObject(tempIndexName).getJsonObject("mappings"); + searchMappingsCache.get().put(cacheKey, mappings); + return client.deleteIndex(tempIndexName).async().ignoreElement() + .andThen(Single.just(mappings)); + }); + } + + /** + * Clean mappings, so that they can be compared + *
    + *
  • Remove entries completely that contain the attribute "dynamic": "true", because those mappings are likely to be changed by ES when documents are indexed (e.g. the attribute "type": "object" is removed from the mapping)
    • + *
+ * @param mapping mappings to be cleaned (will be modified) + */ + protected void cleanMappings(JsonObject mapping) { + Set fieldNames = new HashSet<>(mapping.fieldNames()); + for (String field : fieldNames) { + Object value = mapping.getValue(field); + if (value instanceof JsonObject) { + JsonObject object = (JsonObject) value; + if (Objects.equals(object.getValue("dynamic"), "true")) { + mapping.remove(field); + } else { + cleanMappings(object); + } + } + } + } + private String getType() { switch (complianceMode) { case ES_6: diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/impl/SearchMappingsCacheImpl.java b/elasticsearch/src/main/java/com/gentics/mesh/search/impl/SearchMappingsCacheImpl.java new file mode 100644 index 0000000000..d55587908c --- /dev/null +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/impl/SearchMappingsCacheImpl.java @@ -0,0 +1,55 @@ +package com.gentics.mesh.search.impl; + +import java.time.temporal.ChronoUnit; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import com.gentics.mesh.cache.AbstractMeshCache; +import com.gentics.mesh.cache.CacheRegistry; +import com.gentics.mesh.cache.EventAwareCache; +import com.gentics.mesh.cache.impl.EventAwareCacheFactory; +import com.gentics.mesh.core.rest.MeshEvent; +import com.gentics.mesh.etc.config.MeshOptions; +import com.gentics.mesh.search.SearchMappingsCache; + +import io.vertx.core.json.JsonObject; + +/** + * Implementation of the {@link SearchMappingsCache} + */ +@Singleton +public class SearchMappingsCacheImpl extends AbstractMeshCache implements SearchMappingsCache { + private static final long CACHE_SIZE = 1000; + + /** + * Create the cache + * @param factory factory + * @param options mesh options + * @return cache instance + */ + private static EventAwareCache createCache(EventAwareCacheFactory factory, MeshOptions options) { + return factory.builder() + .maxSize(CACHE_SIZE) + .events(MeshEvent.STARTUP) + .expireAfterAccess(options.getSearchOptions().getIndexMappingCacheTimeout(), ChronoUnit.MILLIS) + .name("searchmappings") + .build(); + } + + /** + * Create the instance + * @param factory cache factory + * @param registry cache registry + * @param options mesh options + */ + @Inject + public SearchMappingsCacheImpl(EventAwareCacheFactory factory, CacheRegistry registry, MeshOptions options) { + super(createCache(factory, options), registry, CACHE_SIZE); + } + + @Override + public void put(String key, JsonObject value) { + cache.put(key, value); + } +} diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/AdminIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/AdminIndexHandler.java index ecc05401bf..b8be919e63 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/AdminIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/AdminIndexHandler.java @@ -19,6 +19,7 @@ import com.gentics.mesh.core.rest.search.SearchStatusResponse; import com.gentics.mesh.core.verticle.handler.HandlerUtilities; import com.gentics.mesh.graphdb.spi.Database; +import com.gentics.mesh.parameter.IndexMaintenanceParameters; import com.gentics.mesh.search.IndexHandlerRegistry; import com.gentics.mesh.search.SearchProvider; import com.gentics.mesh.search.verticle.eventhandler.SyncEventHandler; @@ -82,7 +83,8 @@ public void handleSync(InternalActionContext ac) { db.asyncTx(() -> Single.just(ac.getUser().isAdmin())) .subscribe(isAdmin -> { if (isAdmin) { - SyncEventHandler.invokeSync(vertx); + IndexMaintenanceParameters param = ac.getIndexMaintenanceParameters(); + SyncEventHandler.invokeSync(vertx, param.getIndex()); ac.send(message(ac, "search_admin_index_sync_invoked"), OK); } else { ac.fail(error(FORBIDDEN, "error_admin_permission_required")); @@ -93,7 +95,8 @@ public void handleSync(InternalActionContext ac) { public void handleClear(InternalActionContext ac) { db.asyncTx(() -> Single.just(ac.getUser().isAdmin())).flatMapCompletable(isAdmin -> { if (isAdmin) { - return searchProvider.clear() + IndexMaintenanceParameters param = ac.getIndexMaintenanceParameters(); + return searchProvider.clear(param.getIndex()) .andThen(Observable.fromIterable(registry.getHandlers()) .flatMapCompletable(handler -> handler.init())); } else { diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/entry/AbstractIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/entry/AbstractIndexHandler.java index 3b87d3fc26..9aa9eea982 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/entry/AbstractIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/entry/AbstractIndexHandler.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.gentics.elasticsearch.client.ElasticsearchClient; @@ -194,14 +195,18 @@ protected boolean isSearchClientAvailable() { return searchProvider != null; } - protected Flowable diffAndSync(String indexName, String projectUuid) { - // Sync each bucket individually - Flowable buckets = bucketManager.getBuckets(getElementClass()); - log.info("Handling index sync on handler {" + getClass().getName() + "}"); - return buckets.flatMap(bucket -> { - log.info("Handling sync of {" + bucket + "}"); - return diffAndSync(indexName, projectUuid, bucket); - }, 1); + protected Flowable diffAndSync(String indexName, String projectUuid, Optional indexPattern) { + if (indexPattern.orElse(MATCH_ALL).matcher(indexName).matches()) { + // Sync each bucket individually + Flowable buckets = bucketManager.getBuckets(getElementClass()); + log.info("Handling index sync on handler {" + getClass().getName() + "}"); + return buckets.flatMap(bucket -> { + log.info("Handling sync of {" + bucket + "}"); + return diffAndSync(indexName, projectUuid, bucket); + }, 1); + } else { + return Flowable.empty(); + } } /** @@ -413,6 +418,19 @@ public EntityMetrics getMetrics() { return meters.createSnapshot(); } + @Override + public Completable check() { + // check the indices + return Observable.defer(() -> Observable.fromIterable(getIndices().values())) + .flatMap(info -> searchProvider.check(info).toObservable() + .doOnSubscribe(ignore -> { + if (log.isDebugEnabled()) { + log.debug("Checking index {" + info + "}"); + } + }), 1) + .ignoreElements(); + } + /** * Filter the given indices. Include all indices which start with index handler type and exclude the provided indexName. * diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/group/GroupIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/group/GroupIndexHandler.java index 945a2086a5..00910f57e9 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/group/GroupIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/group/GroupIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.inject.Inject; @@ -96,8 +98,8 @@ public Stream loadAllElements() { } @Override - public Flowable syncIndices() { - return diffAndSync(Group.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(Group.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/microschema/MicroschemaContainerIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/microschema/MicroschemaContainerIndexHandler.java index f9d42885c0..d7ab425d8c 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/microschema/MicroschemaContainerIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/microschema/MicroschemaContainerIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.inject.Inject; @@ -78,8 +80,8 @@ public MicroschemaMappingProvider getMappingProvider() { } @Override - public Flowable syncIndices() { - return diffAndSync(MicroschemaContainer.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(MicroschemaContainer.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/node/NodeIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/node/NodeIndexHandler.java index 717a73007a..9a4d6d6711 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/node/NodeIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/node/NodeIndexHandler.java @@ -14,8 +14,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -236,13 +238,13 @@ public Set filterUnknownIndices(Set indices) { } @Override - public Flowable syncIndices() { + public Flowable syncIndices(Optional indexPattern) { return Flowable.defer(() -> db.tx(() -> { return boot.meshRoot().getProjectRoot().findAll().stream() .flatMap(project -> project.getBranchRoot().findAll().stream() .flatMap(branch -> branch.findActiveSchemaVersions().stream() .flatMap(version -> Stream.of(DRAFT, PUBLISHED) - .map(type -> diffAndSync(project, branch, version, type))))) + .map(type -> diffAndSync(project, branch, version, type, indexPattern))))) .collect(Collectors.collectingAndThen(Collectors.toList(), Flowable::merge)); })); } @@ -294,21 +296,28 @@ protected void processHits(JsonArray hits, Map versions) { } } - private Flowable diffAndSync(Project project, Branch branch, SchemaContainerVersion version, ContainerType type) { + private Flowable diffAndSync(Project project, Branch branch, SchemaContainerVersion version, ContainerType type, Optional indexPattern) { log.info("Handling index sync on handler {" + getClass().getName() + "}"); // Sync each bucket individually Flowable buckets = bucketManager.getBuckets(NodeGraphFieldContainer.class); return buckets.flatMap(bucket -> { log.info("Handling sync of {" + bucket + "}"); - return diffAndSync(project, branch, version, type, bucket); + return diffAndSync(project, branch, version, type, bucket, indexPattern); }, 1); } private Publisher diffAndSync(Project project, Branch branch, SchemaContainerVersion version, ContainerType type, - Bucket bucket) { + Bucket bucket, Optional indexPattern) { return Flowable.defer(() -> { Map> sourceNodesPerIndex = loadVersionsFromGraph(branch, version, type, bucket); return Flowable.fromIterable(getIndexNames(project, branch, version, type)) + .filter(indexName -> { + boolean match = indexPattern.orElse(MATCH_ALL).matcher(indexName).matches(); + if (!match && log.isDebugEnabled()) { + log.debug("Index {} does not match pattern {} and will be omitted from sync", indexName, indexPattern); + } + return match; + }) .flatMap(indexName -> loadVersionsFromIndex(indexName, bucket).flatMapPublisher(sinkVersions -> { log.debug("Handling index sync on handler {" + getClass().getName() + "} for bucket {" + bucket + "}"); Map sourceNodes = sourceNodesPerIndex.getOrDefault(indexName, Collections.emptyMap()); diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/project/ProjectIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/project/ProjectIndexHandler.java index 3f9a9f9ac1..0a8c020e7b 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/project/ProjectIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/project/ProjectIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -75,8 +77,8 @@ public ProjectMappingProvider getMappingProvider() { } @Override - public Flowable syncIndices() { - return diffAndSync(Project.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(Project.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/role/RoleIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/role/RoleIndexHandler.java index a2c52bab7b..eff29c67d5 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/role/RoleIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/role/RoleIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.inject.Inject; @@ -82,8 +84,8 @@ public Map getIndices() { } @Override - public Flowable syncIndices() { - return diffAndSync(Role.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(Role.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/schema/SchemaContainerIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/schema/SchemaContainerIndexHandler.java index 942e4c73eb..b16490e61b 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/schema/SchemaContainerIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/schema/SchemaContainerIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.inject.Inject; @@ -75,8 +77,8 @@ public MappingProvider getMappingProvider() { } @Override - public Flowable syncIndices() { - return diffAndSync(SchemaContainer.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(SchemaContainer.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/tag/TagIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/tag/TagIndexHandler.java index ad44d0261a..e252c524f1 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/tag/TagIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/tag/TagIndexHandler.java @@ -4,8 +4,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -116,12 +118,12 @@ public IndexInfo getIndex(String projectUuid) { } @Override - public Flowable syncIndices() { + public Flowable syncIndices(Optional indexPattern) { return Flowable.defer(() -> db.tx(() -> { return boot.meshRoot().getProjectRoot().findAll().stream() .map(project -> { String uuid = project.getUuid(); - return diffAndSync(Tag.composeIndexName(uuid), uuid); + return diffAndSync(Tag.composeIndexName(uuid), uuid, indexPattern); }).collect(Collectors.collectingAndThen(Collectors.toList(), Flowable::merge)); })); } diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/tagfamily/TagFamilyIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/tagfamily/TagFamilyIndexHandler.java index 5abbc07c71..6149918170 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/tagfamily/TagFamilyIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/tagfamily/TagFamilyIndexHandler.java @@ -4,8 +4,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -105,13 +107,13 @@ public Map getIndices() { } @Override - public Flowable syncIndices() { + public Flowable syncIndices(Optional indexPattern) { return Flowable.defer(() -> db.tx(() -> { return boot.meshRoot().getProjectRoot().findAll().stream() .map(project -> { String uuid = project.getUuid(); String indexName = TagFamily.composeIndexName(uuid); - return diffAndSync(indexName, uuid); + return diffAndSync(indexName, uuid, indexPattern); }).collect(Collectors.collectingAndThen(Collectors.toList(), Flowable::merge)); })); } diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/index/user/UserIndexHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/index/user/UserIndexHandler.java index b94ddae583..4d0246fb8b 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/index/user/UserIndexHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/index/user/UserIndexHandler.java @@ -2,8 +2,10 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.inject.Inject; @@ -73,8 +75,8 @@ protected MappingProvider getMappingProvider() { } @Override - public Flowable syncIndices() { - return diffAndSync(User.composeIndexName(), null); + public Flowable syncIndices(Optional indexPattern) { + return diffAndSync(User.composeIndexName(), null, indexPattern); } @Override diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/ElasticsearchProcessVerticle.java b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/ElasticsearchProcessVerticle.java index eb24698f67..e94067dea2 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/ElasticsearchProcessVerticle.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/ElasticsearchProcessVerticle.java @@ -1,5 +1,6 @@ package com.gentics.mesh.search.verticle; +import static com.gentics.mesh.core.rest.MeshEvent.INDEX_CHECK_REQUEST; import static com.gentics.mesh.core.rest.MeshEvent.INDEX_SYNC_REQUEST; import static com.gentics.mesh.core.rest.MeshEvent.IS_SEARCH_IDLE; import static com.gentics.mesh.core.rest.MeshEvent.SEARCH_FLUSH_REQUEST; @@ -21,9 +22,12 @@ import com.gentics.mesh.core.data.search.request.SearchRequest; import com.gentics.mesh.core.rest.MeshEvent; import com.gentics.mesh.core.rest.event.MeshEventModel; +import com.gentics.mesh.core.rest.event.search.SearchIndexSyncEventModel; +import com.gentics.mesh.distributed.RequestDelegator; import com.gentics.mesh.etc.config.MeshOptions; import com.gentics.mesh.etc.config.search.ElasticSearchOptions; import com.gentics.mesh.event.EventQueueBatch; +import com.gentics.mesh.json.JsonUtil; import com.gentics.mesh.search.SearchProvider; import com.gentics.mesh.search.impl.ElasticsearchResponseErrorStreamable; import com.gentics.mesh.search.verticle.bulk.BulkOperator; @@ -65,7 +69,9 @@ public class ElasticsearchProcessVerticle extends AbstractVerticle { private final IdleChecker idleChecker; private final SyncEventHandler syncEventHandler; private final ElasticSearchOptions options; + private final RequestDelegator delegator; private final String nodeName; + private final boolean clusteringEnabled; private FlowableProcessor requests = PublishProcessor.create(); @@ -79,13 +85,16 @@ public ElasticsearchProcessVerticle(MainEventHandler mainEventhandler, SearchProvider searchProvider, IdleChecker idleChecker, SyncEventHandler syncEventHandler, - MeshOptions options) { + MeshOptions options, + RequestDelegator delegator) { this.mainEventhandler = mainEventhandler; this.searchProvider = searchProvider; this.idleChecker = idleChecker; this.syncEventHandler = syncEventHandler; this.options = options.getSearchOptions(); + this.delegator = delegator; this.nodeName = options.getNodeName(); + this.clusteringEnabled = options.getClusterOptions().isEnabled(); } @Override @@ -123,6 +132,19 @@ public void start() { vertxHandlers.add(replyingEventHandler(IS_SEARCH_IDLE, Single.fromCallable(idleChecker::isIdle))); vertxHandlers.add(replyingEventHandler(SEARCH_REFRESH_REQUEST, refresh().andThen(Single.just(true)))); + if (options.getIndexCheckInterval() > 0) { + log.trace("Setup periodic index check every {} ms", options.getIndexCheckInterval()); + // periodically send the event to check the indices + vertx.setPeriodic(options.getIndexCheckInterval(), id -> { + // only do this for the current master + if (!clusteringEnabled || delegator.isMaster()) { + vertx.eventBus().publish(INDEX_CHECK_REQUEST.address, null); + } + }); + } else { + log.trace("Periodic index check disabled (interval set to {} ms)", options.getIndexCheckInterval()); + } + log.trace("Done Initializing Elasticsearch process verticle"); } @@ -239,7 +261,7 @@ private void startSync() { log.info("Elasticsearch is available again. Starting sync."); elasticsearchAvailable.onNext(available); }); - vertx.eventBus().publish(INDEX_SYNC_REQUEST.address, null); + vertx.eventBus().publish(INDEX_SYNC_REQUEST.address, new JsonObject(JsonUtil.toJson(new SearchIndexSyncEventModel()))); } /** @@ -319,7 +341,7 @@ private Flowable syncIndices(Throwable error) { boolean indexNotFound = ((ElasticsearchResponseErrorStreamable) error).stream() .anyMatch(err -> "index_not_found_exception".equals(err.getType())); if (indexNotFound && !stopped.get()) { - return syncEventHandler.generateSyncRequests() + return syncEventHandler.generateSyncRequests(null) .doOnNext(request -> { log.trace("SyncRequest+{}", request); idleChecker.addAndGetRequests(request.requestCount()); diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/CheckIndicesHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/CheckIndicesHandler.java new file mode 100644 index 0000000000..8eb73d5db2 --- /dev/null +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/CheckIndicesHandler.java @@ -0,0 +1,66 @@ +package com.gentics.mesh.search.verticle.eventhandler; + +import static com.gentics.mesh.core.rest.MeshEvent.INDEX_CHECK_REQUEST; +import static com.gentics.mesh.core.rest.MeshEvent.INDEX_CHECK_START; +import static com.gentics.mesh.core.rest.MeshEvent.INDEX_CHECK_FINISHED; + +import java.util.Collection; +import java.util.Collections; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import com.gentics.mesh.core.data.search.request.SearchRequest; +import com.gentics.mesh.core.rest.MeshEvent; +import com.gentics.mesh.search.IndexHandlerRegistry; +import com.gentics.mesh.search.verticle.MessageEvent; + +import dagger.Lazy; +import io.reactivex.Flowable; +import io.vertx.core.Vertx; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; + +/** + * Event handler, that will check the currently existing indices (for existence and correctness of the mapping) + */ +@Singleton +public class CheckIndicesHandler implements EventHandler { + private static final Logger log = LoggerFactory.getLogger(CheckIndicesHandler.class); + + private final Lazy registry; + + private final Vertx vertx; + + /** + * Create an instance + * @param registry index handler registry + * @param vertx vertx + */ + @Inject + public CheckIndicesHandler(Lazy registry, Vertx vertx) { + this.registry = registry; + this.vertx = vertx; + } + + @Override + public Collection handledEvents() { + return Collections.singletonList(INDEX_CHECK_REQUEST); + } + + @Override + public Flowable handle(MessageEvent messageEvent) { + return syncIndices().doOnSubscribe(ignore -> { + log.debug("Processing index check job."); + vertx.eventBus().publish(INDEX_CHECK_START.address, null); + }).doFinally(() -> { + log.debug("Index check job finished."); + vertx.eventBus().publish(INDEX_CHECK_FINISHED.address, null); + }); + } + + protected Flowable syncIndices() { + return Flowable.fromIterable(registry.get().getHandlers()) + .flatMap(handler -> handler.check().toFlowable()); + } +} diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/MainEventHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/MainEventHandler.java index 6dd08c00cf..f74b3e645f 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/MainEventHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/MainEventHandler.java @@ -63,6 +63,8 @@ public class MainEventHandler implements EventHandler { private final ProjectCreateEventHandler projectCreateEventHandler; private final ProjectDeleteEventHandler projectDeleteEventHandler; + private final CheckIndicesHandler checkIndicesHandler; + @Inject public MainEventHandler(SyncEventHandler syncEventHandler, EventHandlerFactory eventHandlerFactory, @@ -76,7 +78,7 @@ public MainEventHandler(SyncEventHandler syncEventHandler, SchemaMigrationEventHandler schemaMigrationEventHandler, PermissionChangedEventHandler permissionChangedEventHandler, GroupUserAssignmentHandler userGroupAssignmentHandler, - ProjectUpdateEventHandler projectUpdateEventHandler, ProjectCreateEventHandler projectCreateEventHandler) { + ProjectUpdateEventHandler projectUpdateEventHandler, ProjectCreateEventHandler projectCreateEventHandler, CheckIndicesHandler checkIndicesHandler) { this.syncEventHandler = syncEventHandler; this.eventHandlerFactory = eventHandlerFactory; this.groupEventHandler = groupEventHandler; @@ -94,6 +96,7 @@ public MainEventHandler(SyncEventHandler syncEventHandler, this.userGroupAssignmentHandler = userGroupAssignmentHandler; this.projectUpdateEventHandler = projectUpdateEventHandler; this.projectCreateEventHandler = projectCreateEventHandler; + this.checkIndicesHandler = checkIndicesHandler; handlers = createHandlers(); } @@ -125,7 +128,8 @@ private Map> createHandlers() { branchEventHandler, schemaMigrationEventHandler, permissionChangedEventHandler, - userGroupAssignmentHandler + userGroupAssignmentHandler, + checkIndicesHandler ).collect(toMultiMap(EventHandler::handledEvents)); } diff --git a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/SyncEventHandler.java b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/SyncEventHandler.java index 7035f52668..a45c7a714a 100644 --- a/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/SyncEventHandler.java +++ b/elasticsearch/src/main/java/com/gentics/mesh/search/verticle/eventhandler/SyncEventHandler.java @@ -9,15 +9,23 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import javax.inject.Inject; +import org.apache.commons.lang3.StringUtils; + import com.gentics.mesh.Mesh; import com.gentics.mesh.core.data.search.IndexHandler; import com.gentics.mesh.core.data.search.request.DropIndexRequest; import com.gentics.mesh.core.data.search.request.SearchRequest; import com.gentics.mesh.core.rest.MeshEvent; +import com.gentics.mesh.core.rest.event.search.SearchIndexSyncEventModel; +import com.gentics.mesh.etc.config.MeshOptions; +import com.gentics.mesh.json.JsonUtil; import com.gentics.mesh.search.IndexHandlerRegistry; import com.gentics.mesh.search.SearchProvider; import com.gentics.mesh.search.index.metric.SyncMetersFactory; @@ -28,6 +36,7 @@ import io.reactivex.Flowable; import io.reactivex.Single; import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; @@ -46,16 +55,20 @@ public class SyncEventHandler implements EventHandler { private final SyncMetersFactory syncMetersFactory; + private final MeshOptions options; + /** * Send the index sync event which will trigger the index sync job. + * @param indexPattern optional index pattern */ - public static void invokeSync(Vertx vertx) { - log.info("Sending sync event"); - vertx.eventBus().publish(INDEX_SYNC_REQUEST.address, null); + public static void invokeSync(Vertx vertx, String indexPattern) { + SearchIndexSyncEventModel eventModel = new SearchIndexSyncEventModel().setIndexPattern(indexPattern); + log.info("Sending sync event for index pattern {}", eventModel.getIndexPattern()); + vertx.eventBus().publish(INDEX_SYNC_REQUEST.address, new JsonObject(JsonUtil.toJson(eventModel))); } public static Completable invokeSyncCompletable(Vertx vertx) { - return MeshEvent.doAndWaitForEvent(vertx, INDEX_SYNC_FINISHED, () -> SyncEventHandler.invokeSync(vertx)); + return MeshEvent.doAndWaitForEvent(vertx, INDEX_SYNC_FINISHED, () -> SyncEventHandler.invokeSync(vertx, null)); } public static Completable invokeSyncCompletable(Mesh mesh) { @@ -75,22 +88,27 @@ public static Completable invokeClearCompletable(Vertx vertx) { } @Inject - public SyncEventHandler(Lazy registry, SearchProvider provider, Vertx vertx, SyncMetersFactory syncMetersFactory) { + public SyncEventHandler(Lazy registry, SearchProvider provider, Vertx vertx, SyncMetersFactory syncMetersFactory, MeshOptions options) { this.registry = registry; this.provider = provider; this.vertx = vertx; this.syncMetersFactory = syncMetersFactory; + this.options = options; } @Override public Flowable handle(MessageEvent messageEvent) { - return generateSyncRequests(); + String indexPattern = ".*"; + if (messageEvent.message instanceof SearchIndexSyncEventModel) { + indexPattern = ((SearchIndexSyncEventModel) messageEvent.message).getIndexPattern(); + } + return generateSyncRequests(indexPattern); } - public Flowable generateSyncRequests() { + public Flowable generateSyncRequests(String indexPattern) { return Flowable.concatArray( purgeOldIndices(), - syncIndices(), + syncIndices(indexPattern), publishSyncEndEvent() ).doOnSubscribe(ignore -> { log.info("Processing index sync job."); @@ -116,13 +134,23 @@ public Collection handledEvents() { return Collections.singletonList(INDEX_SYNC_REQUEST); } - private Flowable syncIndices() { + private Flowable syncIndices(String indexPattern) { + Pattern pattern = null; + if (indexPattern != null) { + indexPattern = StringUtils.removeStartIgnoreCase(indexPattern, options.getSearchOptions().getPrefix()); + try { + pattern = Pattern.compile(indexPattern); + } catch (PatternSyntaxException e) { + log.warn("Index pattern {} is not valid, synchronizing all indices", e, indexPattern); + } + } + Optional optPattern = Optional.ofNullable(pattern); return Flowable.fromIterable(registry.get().getHandlers()) .flatMap(handler -> handler.init() .doOnSubscribe(ignore -> log.debug("Init for {}", handler.getClass())) .doOnComplete(() -> log.debug("Init for {} complete", handler.getClass())) - .andThen(handler.syncIndices() + .andThen(handler.syncIndices(optPattern) .doOnSubscribe(ignore -> log.debug("Syncing for {}", handler.getClass())) )); } diff --git a/rest-client/src/main/java/com/gentics/mesh/parameter/client/IndexMaintenanceParametersImpl.java b/rest-client/src/main/java/com/gentics/mesh/parameter/client/IndexMaintenanceParametersImpl.java new file mode 100644 index 0000000000..4ddf3f804e --- /dev/null +++ b/rest-client/src/main/java/com/gentics/mesh/parameter/client/IndexMaintenanceParametersImpl.java @@ -0,0 +1,10 @@ +package com.gentics.mesh.parameter.client; + +import com.gentics.mesh.parameter.IndexMaintenanceParameters; + +/** + * Client Implementation of {@link IndexMaintenanceParameters} + */ +public class IndexMaintenanceParametersImpl extends AbstractParameters implements IndexMaintenanceParameters { + +} diff --git a/rest-client/src/main/java/com/gentics/mesh/rest/client/impl/MeshRestHttpClientImpl.java b/rest-client/src/main/java/com/gentics/mesh/rest/client/impl/MeshRestHttpClientImpl.java index d37e52ba46..b919f6fa0d 100644 --- a/rest-client/src/main/java/com/gentics/mesh/rest/client/impl/MeshRestHttpClientImpl.java +++ b/rest-client/src/main/java/com/gentics/mesh/rest/client/impl/MeshRestHttpClientImpl.java @@ -1000,13 +1000,13 @@ public MeshRequest searchTagFamiliesRaw(String projectName, String j } @Override - public MeshRequest invokeIndexClear() { - return prepareRequest(POST, "/search/clear", GenericMessageResponse.class); + public MeshRequest invokeIndexClear(ParameterProvider... parameters) { + return prepareRequest(POST, "/search/clear" + getQuery(parameters), GenericMessageResponse.class); } @Override - public MeshRequest invokeIndexSync() { - return prepareRequest(POST, "/search/sync", GenericMessageResponse.class); + public MeshRequest invokeIndexSync(ParameterProvider... parameters) { + return prepareRequest(POST, "/search/sync" + getQuery(parameters), GenericMessageResponse.class); } @Override diff --git a/rest-client/src/main/java/com/gentics/mesh/rest/client/method/SearchClientMethods.java b/rest-client/src/main/java/com/gentics/mesh/rest/client/method/SearchClientMethods.java index 5376df7783..5c93932885 100644 --- a/rest-client/src/main/java/com/gentics/mesh/rest/client/method/SearchClientMethods.java +++ b/rest-client/src/main/java/com/gentics/mesh/rest/client/method/SearchClientMethods.java @@ -245,17 +245,19 @@ public interface SearchClientMethods { /** * Clear all search indices by removing and re-creating them. * + * @param parameters * @return */ - MeshRequest invokeIndexClear(); + MeshRequest invokeIndexClear(ParameterProvider... parameters); /** * Trigger the index sync action which will synchronize the index for all elements. This is useful when you want to sync the search index after restoring a * backup. * + * @param parameters * @return */ - MeshRequest invokeIndexSync(); + MeshRequest invokeIndexSync(ParameterProvider... parameters); /** * Return the elasticsearch status. This will also contain information about the progress of running index sync operations. diff --git a/rest-model/src/main/java/com/gentics/mesh/core/rest/MeshEvent.java b/rest-model/src/main/java/com/gentics/mesh/core/rest/MeshEvent.java index ef41d75e6c..00ad4d9a4b 100644 --- a/rest-model/src/main/java/com/gentics/mesh/core/rest/MeshEvent.java +++ b/rest-model/src/main/java/com/gentics/mesh/core/rest/MeshEvent.java @@ -31,6 +31,7 @@ import com.gentics.mesh.core.rest.event.project.ProjectSchemaEventModel; import com.gentics.mesh.core.rest.event.role.PermissionChangedEventModel; import com.gentics.mesh.core.rest.event.s3binary.S3BinaryEventModel; +import com.gentics.mesh.core.rest.event.search.SearchIndexSyncEventModel; import com.gentics.mesh.core.rest.event.tag.TagMeshEventModel; import com.gentics.mesh.core.rest.event.tagfamily.TagFamilyMeshEventModel; @@ -469,7 +470,7 @@ public enum MeshEvent { * Address for the handler which will process index sync requests. */ INDEX_SYNC_REQUEST("mesh.search.index.sync.request", - null, + SearchIndexSyncEventModel.class, "Event address which can be used to trigger the sync process."), /** @@ -509,6 +510,27 @@ public enum MeshEvent { null, "Emitted when the index clear process finishes."), + /** + * Event address which will trigger an index check. + */ + INDEX_CHECK_REQUEST("mesh.search.index.check.request", + null, + "Event address which will trigger an index check."), + + /** + * Emitted when an index check process starts. + */ + INDEX_CHECK_START("mesh.search.index.check.start", + null, + "Emitted when the index check process starts."), + + /** + * Address to which index check results will be published (failed, succeeded) + */ + INDEX_CHECK_FINISHED("mesh.search.index.check.finished", + null, + "Emitted when the index check process finishes."), + /** * Event that is emitted when the search verticle has been working and is now idle. */ diff --git a/rest-model/src/main/java/com/gentics/mesh/core/rest/event/search/SearchIndexSyncEventModel.java b/rest-model/src/main/java/com/gentics/mesh/core/rest/event/search/SearchIndexSyncEventModel.java new file mode 100644 index 0000000000..25f0ba9bfa --- /dev/null +++ b/rest-model/src/main/java/com/gentics/mesh/core/rest/event/search/SearchIndexSyncEventModel.java @@ -0,0 +1,40 @@ +package com.gentics.mesh.core.rest.event.search; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.gentics.mesh.core.rest.event.AbstractMeshEventModel; + +/** + * Model of the event, which is sent to initiate an index sync + */ +public class SearchIndexSyncEventModel extends AbstractMeshEventModel { + @JsonProperty(required = true) + @JsonPropertyDescription("Index pattern") + private String indexPattern = ".*"; + + /** + * Create empty instance + */ + public SearchIndexSyncEventModel() { + } + + /** + * Get the index pattern + * @return index pattern + */ + public String getIndexPattern() { + return indexPattern; + } + + /** + * Set the index pattern + * @param indexPattern index pattern + * @return fluent API + */ + public SearchIndexSyncEventModel setIndexPattern(String indexPattern) { + if (indexPattern != null) { + this.indexPattern = indexPattern; + } + return this; + } +} diff --git a/rest-model/src/main/java/com/gentics/mesh/parameter/IndexMaintenanceParameters.java b/rest-model/src/main/java/com/gentics/mesh/parameter/IndexMaintenanceParameters.java new file mode 100644 index 0000000000..cfcb8fa3b0 --- /dev/null +++ b/rest-model/src/main/java/com/gentics/mesh/parameter/IndexMaintenanceParameters.java @@ -0,0 +1,29 @@ +package com.gentics.mesh.parameter; + +/** + * Interface for index maintenance parameters + */ +public interface IndexMaintenanceParameters extends ParameterProvider { + /** + * Key of the parameter to restrict the indices (via regex) + */ + public static final String INDEX_PARAMETER_KEY = "index"; + + /** + * Regex to restrict the index maintenance action to specific indices (via regex) + * @return index parameter + */ + default String getIndex() { + return getParameter(INDEX_PARAMETER_KEY); + } + + /** + * Set the index parameter + * @param index parameter + * @return fluent API + */ + default IndexMaintenanceParameters setIndex(String index) { + setParameter(INDEX_PARAMETER_KEY, index); + return this; + } +}