diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/Metric.java b/dotCMS/src/main/java/com/dotcms/telemetry/Metric.java new file mode 100644 index 000000000000..915c6ae20d06 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/Metric.java @@ -0,0 +1,83 @@ +package com.dotcms.telemetry; + +/** + * Represents all the information needed to identify a Metric: + * + */ +public class Metric { + + private final String name; + private final String description; + private final MetricCategory category; + private final MetricFeature feature; + + public Metric(final Builder builder) { + this.name = builder.name; + this.description = builder.description; + this.category = builder.category; + this.feature = builder.feature; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public MetricCategory getCategory() { + return category; + } + + public MetricFeature getFeature() { + return feature; + } + + @Override + public String toString() { + return "Metric{" + + "name='" + name + '\'' + + ", description='" + description + '\'' + + ", category=" + category + + ", feature=" + feature + + '}'; + } + + public static class Builder { + String name; + String description; + MetricCategory category; + MetricFeature feature; + + public Builder name(final String name) { + this.name = name; + return this; + } + + public Builder description(final String description) { + this.description = description; + return this; + } + + public Builder category(final MetricCategory category) { + this.category = category; + return this; + } + + public Builder feature(final MetricFeature feature) { + this.feature = feature; + return this; + } + + public Metric build() { + return new Metric(this); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricCalculationError.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricCalculationError.java new file mode 100644 index 000000000000..bb734d785d96 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricCalculationError.java @@ -0,0 +1,35 @@ +package com.dotcms.telemetry; + +import com.fasterxml.jackson.annotation.JsonUnwrapped; + +/** + * Represent an error that occurs while calculating the metric. + */ +public class MetricCalculationError { + + @JsonUnwrapped + private final Metric metric; + private final String error; + + public MetricCalculationError(Metric metric, String error) { + this.metric = metric; + this.error = error; + } + + public Metric getType() { + return metric; + } + + public String getError() { + return error; + } + + @Override + public String toString() { + return "MetricCalculationError{" + + "metric=" + metric + + ", error='" + error + '\'' + + '}'; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricCategory.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricCategory.java new file mode 100644 index 000000000000..f26a43d7712e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricCategory.java @@ -0,0 +1,27 @@ +package com.dotcms.telemetry; + +/** + * Represents the Metric Category where the {@link MetricType} belongs to + */ +public enum MetricCategory { + + DIFFERENTIATING_FEATURES("Differentiating Features"), + PAID_FEATURES("Paid Features"), + PLATFORM_SPECIFIC_CUSTOMIZATION("Platform-Specific Customization"), + SOPHISTICATED_CONTENT_ARCHITECTURE("Sophisticated Content Architecture"), + POSITIVE_USER_EXPERIENCE("Positive User Experience (incl. content velocity)"), + PLATFORM_SPECIFIC_DEVELOPMENT("Platform-Specific Development"), + RECENT_ACTIVITY("Recent Activity"), + HISTORY_LEGACY_DEPRECATED_FEATURES("History (incl. legacy/deprecated features"); + + private final String label; + + MetricCategory(final String label) { + this.label = label; + } + + public String getLabel() { + return label; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricFeature.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricFeature.java new file mode 100644 index 000000000000..77a01b5c0e31 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricFeature.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry; + +/** + * Represents the dotCMS feature that the {@link MetricType} belongs to + */ +public enum MetricFeature { + + CONTENTLETS, + LANGUAGES, + CONTENT_TYPES, + SITES, + URL_MAPS, + WORKFLOW, + USERS, + SITE_SEARCH, + IMAGE_API, + LAYOUT, + CONTENT_TYPE_FIELDS + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricType.java new file mode 100644 index 000000000000..9d1d3d121c77 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricType.java @@ -0,0 +1,59 @@ +package com.dotcms.telemetry; + +import com.dotmarketing.exception.DotDataException; + +import java.util.Optional; + +/** + * This interface represents a Metric that needs to be calculated and included in a MetricSnapshot. + * For each of these a new concrete class implements this interface must be created. The interface + * provides a set of methods to define the metadata for the Metric, such as its name, description, + * category, and feature. It also includes a method to calculate the Metric's value. + * + *

Some of the Metrics to collect are:

+ * + * + * @see MetricCategory + * @see MetricFeature + */ +public interface MetricType { + + String getName(); + + String getDescription(); + + MetricCategory getCategory(); + + MetricFeature getFeature(); + + Optional getValue(); + + default Metric getMetric() { + return new Metric.Builder() + .name(getName()) + .description(getDescription()) + .category(getCategory()) + .feature(getFeature()) + .build(); + } + + default Optional getStat() throws DotDataException { + return getValue().map(o -> new MetricValue(this.getMetric(), o)); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricValue.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricValue.java new file mode 100644 index 000000000000..2e59fbde726b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricValue.java @@ -0,0 +1,62 @@ +package com.dotcms.telemetry; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonUnwrapped; +import org.apache.commons.lang.math.NumberUtils; + +import java.text.DecimalFormat; + +/** + * Represents the value for a {@link MetricType} + */ +public class MetricValue { + + private static final DecimalFormat FORMAT = new DecimalFormat("0.##"); + + @JsonUnwrapped + final Metric metric; + + final Object value; + + public MetricValue(final Metric metric, final Object value) { + this.metric = metric; + this.value = value; + } + + /** + * Check if the value of the Metric is a numeric value. + * + * @return true if the value is a numeric value + */ + @JsonIgnore + public boolean isNumeric() { + return NumberUtils.isNumber(value.toString()); + } + + public Metric getMetric() { + return metric; + } + + /** + * Return the value of the Metric if it is a numeric value then return it as a String formatted + * with two decimals + * + * @return + */ + public Object getValue() { + if (isNumeric()) { + return FORMAT.format(Double.parseDouble(value.toString())); + } else { + return value; + } + } + + @Override + public String toString() { + return "MetricValue{" + + "metric=" + metric + + ", value=" + value + + '}'; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/MetricsSnapshot.java b/dotCMS/src/main/java/com/dotcms/telemetry/MetricsSnapshot.java new file mode 100644 index 000000000000..d0e5ab566690 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/MetricsSnapshot.java @@ -0,0 +1,93 @@ +package com.dotcms.telemetry; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a snapshot of metric statistics, including all calculated metrics, separated into + * numeric and non-numeric categories. It also contains a list of errors, indicating any metrics + * that encountered exceptions during calculation. + */ +public class MetricsSnapshot { + + /** + * Collection of Numeric Metrics + */ + final Collection stats; + + /** + * Collection of No Numeric Metrics + */ + final Collection notNumericStats; + + /** + * Metric that thrown an Exception during the calculation process + */ + final Collection errors; + + public MetricsSnapshot(final Builder builder) { + this.stats = builder.stats; + this.notNumericStats = builder.notNumericStats; + this.errors = builder.errors; + } + + @JsonProperty + public Collection getStats() { + return stats; + } + + @JsonProperty + public Collection getErrors() { + return errors; + } + + @JsonAnyGetter + public Map getNotNumericStats() { + final Map result = new HashMap<>(); + + for (final MetricValue stat : notNumericStats) { + result.put(stat.getMetric().getName(), stat.getValue().toString()); + } + + return result; + } + + @Override + public String toString() { + return "MetricsSnapshot{" + + "stats=" + stats + + ", notNumericStats=" + notNumericStats + + ", errors=" + errors + + '}'; + } + + public static class Builder { + private Collection stats; + private Collection notNumericStats; + private Collection errors; + + public Builder stats(Collection stats) { + this.stats = stats; + return this; + } + + public Builder notNumericStats(Collection notNumericStats) { + this.notNumericStats = notNumericStats; + return this; + } + + public Builder errors(Collection errors) { + this.errors = errors; + return this; + } + + public MetricsSnapshot build() { + return new MetricsSnapshot(this); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricEndpointPayload.java b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricEndpointPayload.java new file mode 100644 index 000000000000..2549f7fb88ed --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricEndpointPayload.java @@ -0,0 +1,125 @@ +package com.dotcms.telemetry.business; + +import com.dotcms.telemetry.MetricsSnapshot; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * Payload to be sent to the Metric Endpoint + */ +public class MetricEndpointPayload { + + static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm" + + ":ss.S'Z'") + .withZone(ZoneId.systemDefault()); + private final String clientName; + private final String clientEnv; + private final int clientVersion; + private final String clientCategory; + private final long schemaVersion; + private final String insertDate; + private final MetricsSnapshot snapshot; + + private MetricEndpointPayload(final Builder builder) { + this.clientName = builder.clientName; + this.clientEnv = builder.clientEnv; + this.clientVersion = builder.clientVersion; + this.clientCategory = builder.clientCategory; + this.schemaVersion = builder.schemaVersion; + this.insertDate = DATE_FORMAT.format(builder.insertDate); + this.snapshot = builder.snapshot; + } + + public String getClientName() { + return clientName; + } + + public String getClientEnv() { + return clientEnv; + } + + public int getClientVersion() { + return clientVersion; + } + + public String getClientCategory() { + return clientCategory; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public String getInsertDate() { + return insertDate; + } + + public MetricsSnapshot getSnapshot() { + return snapshot; + } + + @Override + public String toString() { + return "MetricEndpointPayload{" + + "clientName='" + clientName + '\'' + + ", clientEnv='" + clientEnv + '\'' + + ", clientVersion=" + clientVersion + + ", clientCategory='" + clientCategory + '\'' + + ", schemaVersion=" + schemaVersion + + ", insertDate='" + insertDate + '\'' + + ", snapshot=" + snapshot + + '}'; + } + + public static class Builder { + private String clientName; + private String clientEnv; + private int clientVersion; + private String clientCategory; + private long schemaVersion; + private Instant insertDate; + private MetricsSnapshot snapshot; + + public Builder clientName(String clientName) { + this.clientName = clientName; + return this; + } + + public Builder clientEnv(String clientEnv) { + this.clientEnv = clientEnv; + return this; + } + + public Builder clientVersion(int version) { + this.clientVersion = version; + return this; + } + + public Builder clientCategory(String clientCategory) { + this.clientCategory = clientCategory; + return this; + } + + public Builder schemaVersion(long schemaVersion) { + this.schemaVersion = schemaVersion; + return this; + } + + public Builder insertDate(Instant insertDate) { + this.insertDate = insertDate; + return this; + } + + public Builder snapshot(MetricsSnapshot snapshot) { + this.snapshot = snapshot; + return this; + } + + public MetricEndpointPayload build() { + return new MetricEndpointPayload(this); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricFactory.java b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricFactory.java new file mode 100644 index 000000000000..0c3ef97a621a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricFactory.java @@ -0,0 +1,77 @@ +package com.dotcms.telemetry.business; + +import com.dotcms.business.CloseDBIfOpened; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.db.DbConnectionFactory; +import com.dotmarketing.exception.DotDataException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Utility class tha provide methods to run SQL Query into dotCMS DataBase + */ +public enum MetricFactory { + + INSTANCE; + + @CloseDBIfOpened + public Optional getValue(final String sqlQuery) throws DotDataException { + final DotConnect dotConnect = new DotConnect(); + final List> loadObjectResults = dotConnect.setSQL(sqlQuery) + .loadObjectResults(); + + if (loadObjectResults.isEmpty()) { + return Optional.empty(); + } + + return Optional.ofNullable(loadObjectResults.get(0).get("value")); + } + + /** + * Execute a query that returns a list of String objects. The query should retrieve a field + * called value, as shown in the example below: + * 
+     * {@code
+     * SELECT identifier as value FROM template
+     * }
+     *
+ *

+ * This method will iterate through the results returned by the query and use the values from + * the value field to create a Collection of Strings. + * + * @param sqlQuery the query to be executed. + * + * @return a Collection of Strings with the values returned by the query. + * + * @throws DotDataException if an error occurs while executing the query. + */ + @CloseDBIfOpened + public Optional> getList(final String sqlQuery) throws DotDataException { + final DotConnect dotConnect = new DotConnect(); + final List> loadObjectResults = dotConnect.setSQL(sqlQuery) + .loadObjectResults(); + + if (loadObjectResults.isEmpty()) { + return Optional.empty(); + } + + return Optional.of( + loadObjectResults.stream().map(item -> item.get("value").toString()).collect(Collectors.toList()) + ); + } + + @CloseDBIfOpened + @SuppressWarnings("unchecked") + public int getSchemaDBVersion() throws DotDataException { + final ArrayList> results = new DotConnect() + .setSQL("SELECT max(db_version) AS version FROM db_version") + .loadResults(); + + return Integer.parseInt(results.get(0).get("version").toString()); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricsAPI.java b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricsAPI.java new file mode 100644 index 000000000000..f5d46f953f45 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/business/MetricsAPI.java @@ -0,0 +1,259 @@ +package com.dotcms.telemetry.business; + +import com.dotcms.telemetry.MetricsSnapshot; +import com.dotcms.telemetry.util.JsonUtil; +import com.dotcms.http.CircuitBreakerUrl; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.db.LocalTransaction; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * API for all the Metrics related operations + */ +public enum MetricsAPI { + + INSTANCE; + + final String endPointUrl = Config.getStringProperty("TELEMETRY_PERSISTENCE_ENDPOINT", + "https://dotcms-prod-1.analytics.dotcms.cloud/m"); + + final String customCategory = Config.getStringProperty("TELEMETRY_CLIENT_CATEGORY", + null); + + final String clientNameFromConfig = Config.getStringProperty("TELEMETRY_CLIENT_NAME", + null); + + final int clientEnvVersionFromConfig = Config.getIntProperty("TELEMETRY_CLIENT_VERSION", + -1); + + final String clientEnvFromConfig = Config.getStringProperty("TELEMETRY_CLIENT_ENV", + null); + + final int maxAttemptsToFail = Config.getIntProperty("TELEMETRY_MAX_ATTEMPTS_TO_FAIL", 3); + final int tryAgainDelay = Config.getIntProperty("TELEMETRY_TRY_AGAIN_DELAY", 30); + final int requestTimeout = Config.getIntProperty("TELEMETRY_REQUEST_TIMEOUT", 4000); + + private static int getSchemaDBVersion() throws DotDataException { + try { + return LocalTransaction.wrapReturn(MetricFactory.INSTANCE::getSchemaDBVersion); + } catch (DotDataException e) { + throw e; + } catch (Exception e) { + throw new DotRuntimeException(e); + } + + } + + /** + * Persists the given {@link MetricsSnapshot} + * + * @param metricsSnapshot the snapshot to persist + */ + public void persistMetricsSnapshot(final MetricsSnapshot metricsSnapshot) throws DotDataException { + Logger.debug(this, "Persisting the snapshot"); + + final Client client = getClient(); + + sendMetric(new MetricEndpointPayload.Builder() + .clientName(client.getClientName()) + .clientEnv(client.getEnvironment()) + .clientVersion(client.getVersion()) + .clientCategory(client.getCategory()) + .schemaVersion(getSchemaDBVersion()) + .insertDate(Instant.now()) + .snapshot(metricsSnapshot) + .build() + ); + } + + /** + * Use the {@link MetricFactory#getList(String)} method to execute a Query and return a + * Collection of String + * + * @param sqlQuery the query to be executed + * + * @return a Collection of Strings with the values returned by the query + * + * @see MetricFactory#getList(String) + */ + public List getList(final String sqlQuery) { + try { + return LocalTransaction.wrapReturn(() -> MetricFactory.INSTANCE.getList(sqlQuery)) + .orElse(Collections.emptyList()); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + public Optional getValue(final String sqlQuery) { + try { + return LocalTransaction.wrapReturn(() -> MetricFactory.INSTANCE.getValue(sqlQuery)); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + public Client getClient() throws DotDataException { + + Client client = getClientMetaDataFromHostName(); + + return new Client.Builder() + .clientName(clientNameFromConfig != null ? clientNameFromConfig : + client.getClientName()) + .category(customCategory != null ? customCategory : client.getCategory()) + .version(clientEnvVersionFromConfig != -1 ? clientEnvVersionFromConfig : + client.getVersion()) + .environment(clientEnvFromConfig != null ? clientEnvFromConfig : + client.getEnvironment()) + .build(); + } + + private Client getClientMetaDataFromHostName() throws DotDataException { + final String hostname = APILocator.getServerAPI().getCurrentServer().getName(); + final String[] split = hostname.split("-"); + + final Client.Builder builder = new Client.Builder(); + + if (split.length < 4) { + builder.clientName(hostname) + .environment(hostname) + .version(0); + } else { + final String clientName = String.join("-", Arrays.copyOfRange(split, 1, + split.length - 2)); + + builder.clientName(clientName) + .environment(split[split.length - 2]) + .version(Integer.parseInt(split[split.length - 1])); + + } + + getCategoryFromHostName(hostname).map(ClientCategory::name).ifPresent(builder::category); + + return builder.build(); + } + + private Optional getCategoryFromHostName(final String hostname) { + if (hostname.startsWith("dotcms-corpsites")) { + return Optional.of(ClientCategory.DOTCMS); + } else if (hostname.startsWith("dotcms-")) { + return Optional.of(ClientCategory.CLIENT); + } + + return Optional.empty(); + } + + + private void sendMetric(final MetricEndpointPayload metricEndpointPayload) { + + final CircuitBreakerUrl circuitBreakerUrl = CircuitBreakerUrl.builder() + .setMethod(CircuitBreakerUrl.Method.POST) + .setUrl(endPointUrl) + .setHeaders(Map.of("Content-Type", "application/json")) + .setRawData(JsonUtil.INSTANCE.getAsJson(metricEndpointPayload)) + .setFailAfter(maxAttemptsToFail) + .setTryAgainAfterDelaySeconds(tryAgainDelay) + .setTimeout(requestTimeout) + .build(); + + try { + circuitBreakerUrl.doString(); + final int response = circuitBreakerUrl.response(); + if (response != 201) { + Logger.debug(this, + "ERROR: Unable to save the Metric. HTTP error code: " + response); + } + } catch (Exception e) { + Logger.debug(this, "ERROR: Unable to save the Metric."); + } + } + + private enum ClientCategory { + DOTCMS, + CLIENT + } + + public static class Client { + final String clientName; + final String environment; + final int version; + + final String category; + + public Client(final Builder builder) { + this.clientName = builder.clientName; + this.environment = builder.environment; + this.version = builder.version; + this.category = builder.category; + } + + @Override + public String toString() { + return "Client{" + + "clientName='" + clientName + '\'' + + ", environment='" + environment + '\'' + + ", version=" + version + + ", category='" + category + '\'' + + '}'; + } + + public String getClientName() { + return clientName; + } + + public String getEnvironment() { + return environment; + } + + public int getVersion() { + return version; + } + + public String getCategory() { + return category; + } + + public static class Builder { + String clientName; + String environment; + int version; + + String category; + + Builder clientName(final String clientName) { + this.clientName = clientName; + return this; + } + + Builder environment(final String environment) { + this.environment = environment; + return this; + } + + Builder version(final int version) { + this.version = version; + return this; + } + + Builder category(final String category) { + this.category = category; + return this; + } + + Client build() { + return new Client(this); + } + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/DBMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/DBMetricType.java new file mode 100644 index 000000000000..24f3b45c61c6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/DBMetricType.java @@ -0,0 +1,30 @@ +package com.dotcms.telemetry.collectors; + +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.MetricValue; +import com.dotcms.telemetry.business.MetricsAPI; + +import java.util.Optional; + +/** + * Represents the MetaData of a Metric that we want to collect from DataBase + * + * @see MetricType + */ +public interface DBMetricType extends MetricType { + + String getSqlQuery(); + + @Override + default Optional getValue() { + return MetricsAPI.INSTANCE.getValue(getSqlQuery()); + } + + @Override + default Optional getStat() { + return getValue() + .map(value -> new MetricValue(this.getMetric(), value)); + + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/MetricStatsCollector.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/MetricStatsCollector.java new file mode 100644 index 000000000000..8bdf3347cb70 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/MetricStatsCollector.java @@ -0,0 +1,289 @@ +package com.dotcms.telemetry.collectors; + +import com.dotcms.telemetry.MetricCalculationError; +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.MetricValue; +import com.dotcms.telemetry.MetricsSnapshot; +import com.dotcms.telemetry.collectors.api.ApiMetricAPI; +import com.dotcms.telemetry.collectors.api.ApiMetricTypes; +import com.dotcms.telemetry.collectors.container.TotalFileContainersInLivePageDatabaseMetricType; +import com.dotcms.telemetry.collectors.container.TotalFileContainersInLiveTemplatesDatabaseMetricType; +import com.dotcms.telemetry.collectors.container.TotalFileContainersInWorkingPageDatabaseMetricType; +import com.dotcms.telemetry.collectors.container.TotalStandardContainersInLivePageDatabaseMetricType; +import com.dotcms.telemetry.collectors.container.TotalStandardContainersInLiveTemplatesDatabaseMetricType; +import com.dotcms.telemetry.collectors.container.TotalStandardContainersInWorkingPageDatabaseMetricType; +import com.dotcms.telemetry.collectors.content.LastContentEditedDatabaseMetricType; +import com.dotcms.telemetry.collectors.content.LiveNotDefaultLanguageContentsDatabaseMetricType; +import com.dotcms.telemetry.collectors.content.RecentlyEditedContentDatabaseMetricType; +import com.dotcms.telemetry.collectors.content.TotalContentsDatabaseMetricType; +import com.dotcms.telemetry.collectors.content.WorkingNotDefaultLanguageContentsDatabaseMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfCategoryFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfConstantFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfDateFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfDateTimeFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfHiddenFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfPermissionsFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfRelationshipFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfSiteOrFolderFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfTagFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfTextAreaFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfTextFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfTimeFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfWYSIWYGFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfBinaryFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfBlockEditorFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfCheckboxFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfColumnsFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfFileFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfImageFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfJSONFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfKeyValueFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfLineDividersFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfMultiselectFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfRadioFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfRowsFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfSelectFieldsMetricType; +import com.dotcms.telemetry.collectors.contenttype.CountOfTabFieldsMetricType; +import com.dotcms.telemetry.collectors.language.HasChangeDefaultLanguagesDatabaseMetricType; +import com.dotcms.telemetry.collectors.language.OldStyleLanguagesVarialeMetricType; +import com.dotcms.telemetry.collectors.language.TotalLanguagesDatabaseMetricType; +import com.dotcms.telemetry.collectors.language.TotalLiveLanguagesVariablesDatabaseMetricType; +import com.dotcms.telemetry.collectors.language.TotalUniqueLanguagesDatabaseMetricType; +import com.dotcms.telemetry.collectors.language.TotalWorkingLanguagesVariablesDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.CountOfLiveSitesWithSiteVariablesMetricType; +import com.dotcms.telemetry.collectors.site.CountOfSitesWithIndividualPermissionsMetricType; +import com.dotcms.telemetry.collectors.site.CountOfSitesWithThumbnailsMetricType; +import com.dotcms.telemetry.collectors.site.CountOfWorkingSitesWithSiteVariablesMetricType; +import com.dotcms.telemetry.collectors.site.SitesWithNoDefaultTagStorageDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.SitesWithNoSystemFieldsDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.SitesWithRunDashboardDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.TotalActiveSitesDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.TotalAliasesActiveSitesDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.TotalAliasesAllSitesDatabaseMetricType; +import com.dotcms.telemetry.collectors.site.TotalSitesDatabaseMetricType; +import com.dotcms.telemetry.collectors.sitesearch.CountSiteSearchDocumentMetricType; +import com.dotcms.telemetry.collectors.sitesearch.CountSiteSearchIndicesMetricType; +import com.dotcms.telemetry.collectors.sitesearch.TotalSizeSiteSearchIndicesMetricType; +import com.dotcms.telemetry.collectors.template.TotalAdvancedTemplatesDatabaseMetricType; +import com.dotcms.telemetry.collectors.template.TotalBuilderTemplatesDatabaseMetricType; +import com.dotcms.telemetry.collectors.template.TotalTemplatesDatabaseMetricType; +import com.dotcms.telemetry.collectors.template.TotalTemplatesInLivePagesDatabaseMetricType; +import com.dotcms.telemetry.collectors.template.TotalTemplatesInWorkingPagesDatabaseMetricType; +import com.dotcms.telemetry.collectors.theme.TotalFilesInThemeMetricType; +import com.dotcms.telemetry.collectors.theme.TotalLiveContainerDatabaseMetricType; +import com.dotcms.telemetry.collectors.theme.TotalLiveFilesInThemeMetricType; +import com.dotcms.telemetry.collectors.theme.TotalThemeMetricType; +import com.dotcms.telemetry.collectors.theme.TotalThemeUsedInLiveTemplatesMetricType; +import com.dotcms.telemetry.collectors.theme.TotalThemeUsedInWorkingTemplatesMetricType; +import com.dotcms.telemetry.collectors.theme.TotalWorkingContainerDatabaseMetricType; +import com.dotcms.telemetry.collectors.urlmap.ContentTypesWithUrlMapDatabaseMetricType; +import com.dotcms.telemetry.collectors.urlmap.LiveContentInUrlMapDatabaseMetricType; +import com.dotcms.telemetry.collectors.urlmap.UrlMapPatterWithTwoVariablesDatabaseMetricType; +import com.dotcms.telemetry.collectors.urlmap.WorkingContentInUrlMapDatabaseMetricType; +import com.dotcms.telemetry.collectors.user.ActiveUsersDatabaseMetricType; +import com.dotcms.telemetry.collectors.user.LastLoginDatabaseMetricType; +import com.dotcms.telemetry.collectors.user.LastLoginUserDatabaseMetric; +import com.dotcms.telemetry.collectors.workflow.ActionsDatabaseMetricType; +import com.dotcms.telemetry.collectors.workflow.ContentTypesDatabaseMetricType; +import com.dotcms.telemetry.collectors.workflow.SchemesDatabaseMetricType; +import com.dotcms.telemetry.collectors.workflow.StepsDatabaseMetricType; +import com.dotcms.telemetry.collectors.workflow.SubActionsDatabaseMetricType; +import com.dotcms.telemetry.collectors.workflow.UniqueSubActionsDatabaseMetricType; +import com.dotcms.telemetry.util.MetricCaches; +import com.dotmarketing.db.DbConnectionFactory; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Optional; + +/** + * This class collects and generates all the different Metrics that will be reported to a User or + * any other application/client. + * + * @author Freddy Rodriguez + * @since Jan 8th, 2024 + */ +public final class MetricStatsCollector { + + public static final ApiMetricAPI apiStatAPI = new ApiMetricAPI(); + static final Collection metricStatsCollectors; + + private MetricStatsCollector() { + + } + + static { + metricStatsCollectors = new HashSet<>(); + + metricStatsCollectors.add(new LastContentEditedDatabaseMetricType()); + metricStatsCollectors.add(new RecentlyEditedContentDatabaseMetricType()); + metricStatsCollectors.add(new LastLoginDatabaseMetricType()); + metricStatsCollectors.add(new LastLoginUserDatabaseMetric()); + + metricStatsCollectors.add(new SchemesDatabaseMetricType()); + metricStatsCollectors.add(new ContentTypesDatabaseMetricType()); + metricStatsCollectors.add(new StepsDatabaseMetricType()); + metricStatsCollectors.add(new ActionsDatabaseMetricType()); + metricStatsCollectors.add(new SubActionsDatabaseMetricType()); + metricStatsCollectors.add(new UniqueSubActionsDatabaseMetricType()); + + metricStatsCollectors.add(new TotalContentsDatabaseMetricType()); + metricStatsCollectors.add(new WorkingNotDefaultLanguageContentsDatabaseMetricType()); + metricStatsCollectors.add(new LiveNotDefaultLanguageContentsDatabaseMetricType()); + metricStatsCollectors.add(new OldStyleLanguagesVarialeMetricType()); + + metricStatsCollectors.add(new ActiveUsersDatabaseMetricType()); + metricStatsCollectors.add(new ContentTypesWithUrlMapDatabaseMetricType()); + metricStatsCollectors.add(new LiveContentInUrlMapDatabaseMetricType()); + metricStatsCollectors.add(new UrlMapPatterWithTwoVariablesDatabaseMetricType()); + metricStatsCollectors.add(new WorkingContentInUrlMapDatabaseMetricType()); + + metricStatsCollectors.add(new HasChangeDefaultLanguagesDatabaseMetricType()); + metricStatsCollectors.add(new TotalLanguagesDatabaseMetricType()); + metricStatsCollectors.add(new TotalLiveLanguagesVariablesDatabaseMetricType()); + metricStatsCollectors.add(new TotalUniqueLanguagesDatabaseMetricType()); + metricStatsCollectors.add(new TotalWorkingLanguagesVariablesDatabaseMetricType()); + + metricStatsCollectors.add(new CountOfSitesWithIndividualPermissionsMetricType()); + metricStatsCollectors.add(new CountOfSitesWithThumbnailsMetricType()); + metricStatsCollectors.add(new CountOfWorkingSitesWithSiteVariablesMetricType()); + metricStatsCollectors.add(new CountOfLiveSitesWithSiteVariablesMetricType()); + + metricStatsCollectors.add(new SitesWithNoSystemFieldsDatabaseMetricType()); + metricStatsCollectors.add(new SitesWithNoDefaultTagStorageDatabaseMetricType()); + metricStatsCollectors.add(new SitesWithRunDashboardDatabaseMetricType()); + metricStatsCollectors.add(new TotalAliasesAllSitesDatabaseMetricType()); + metricStatsCollectors.add(new TotalAliasesActiveSitesDatabaseMetricType()); + metricStatsCollectors.add(new TotalSitesDatabaseMetricType()); + metricStatsCollectors.add(new TotalActiveSitesDatabaseMetricType()); + metricStatsCollectors.add(new TotalAdvancedTemplatesDatabaseMetricType()); + + metricStatsCollectors.add(new CountSiteSearchDocumentMetricType()); + metricStatsCollectors.add(new CountSiteSearchIndicesMetricType()); + metricStatsCollectors.add(new TotalSizeSiteSearchIndicesMetricType()); + + metricStatsCollectors.add(new TotalTemplatesDatabaseMetricType()); + metricStatsCollectors.add(new TotalTemplatesInLivePagesDatabaseMetricType()); + metricStatsCollectors.add(new TotalTemplatesInWorkingPagesDatabaseMetricType()); + metricStatsCollectors.add(new TotalBuilderTemplatesDatabaseMetricType()); + + metricStatsCollectors.add(new TotalThemeUsedInLiveTemplatesMetricType()); + metricStatsCollectors.add(new TotalThemeUsedInWorkingTemplatesMetricType()); + + metricStatsCollectors.add(new TotalLiveFilesInThemeMetricType()); + metricStatsCollectors.add(new TotalFilesInThemeMetricType()); + metricStatsCollectors.add(new TotalThemeMetricType()); + + metricStatsCollectors.add(new TotalLiveContainerDatabaseMetricType()); + metricStatsCollectors.add(new TotalWorkingContainerDatabaseMetricType()); + + metricStatsCollectors.add(new TotalStandardContainersInLivePageDatabaseMetricType()); + metricStatsCollectors.add(new TotalFileContainersInLivePageDatabaseMetricType()); + metricStatsCollectors.add(new TotalStandardContainersInWorkingPageDatabaseMetricType()); + metricStatsCollectors.add(new TotalFileContainersInWorkingPageDatabaseMetricType()); + + metricStatsCollectors.add(new TotalFileContainersInLiveTemplatesDatabaseMetricType()); + metricStatsCollectors.add(new TotalStandardContainersInLiveTemplatesDatabaseMetricType()); + + metricStatsCollectors.add(new CountOfCategoryFieldsMetricType()); + metricStatsCollectors.add(new CountOfConstantFieldsMetricType()); + metricStatsCollectors.add(new CountOfDateFieldsMetricType()); + metricStatsCollectors.add(new CountOfDateTimeFieldsMetricType()); + metricStatsCollectors.add(new CountOfBinaryFieldsMetricType()); + metricStatsCollectors.add(new CountOfBlockEditorFieldsMetricType()); + metricStatsCollectors.add(new CountOfCheckboxFieldsMetricType()); + metricStatsCollectors.add(new CountOfColumnsFieldsMetricType()); + metricStatsCollectors.add(new CountOfFileFieldsMetricType()); + metricStatsCollectors.add(new CountOfImageFieldsMetricType()); + metricStatsCollectors.add(new CountOfJSONFieldsMetricType()); + metricStatsCollectors.add(new CountOfKeyValueFieldsMetricType()); + metricStatsCollectors.add(new CountOfLineDividersFieldsMetricType()); + metricStatsCollectors.add(new CountOfMultiselectFieldsMetricType()); + metricStatsCollectors.add(new CountOfRadioFieldsMetricType()); + metricStatsCollectors.add(new CountOfRowsFieldsMetricType()); + metricStatsCollectors.add(new CountOfSelectFieldsMetricType()); + metricStatsCollectors.add(new CountOfTabFieldsMetricType()); + metricStatsCollectors.add(new CountOfHiddenFieldsMetricType()); + metricStatsCollectors.add(new CountOfPermissionsFieldsMetricType()); + metricStatsCollectors.add(new CountOfRelationshipFieldsMetricType()); + metricStatsCollectors.add(new CountOfSiteOrFolderFieldsMetricType()); + metricStatsCollectors.add(new CountOfTagFieldsMetricType()); + metricStatsCollectors.add(new CountOfTextAreaFieldsMetricType()); + metricStatsCollectors.add(new CountOfTextFieldsMetricType()); + metricStatsCollectors.add(new CountOfTimeFieldsMetricType()); + metricStatsCollectors.add(new CountOfWYSIWYGFieldsMetricType()); + + metricStatsCollectors.addAll(ApiMetricTypes.INSTANCE.get()); + } + + public static MetricsSnapshot getStatsAndCleanUp() { + final MetricsSnapshot stats = getStats(); + + apiStatAPI.flushTemporalTable(); + return stats; + } + + /** + * Calculate a MetricSnapshot by iterating through all the MetricType collections. + * + * @return the {@link MetricsSnapshot} with all the calculated metrics. + */ + public static MetricsSnapshot getStats() { + + final Collection stats = new ArrayList<>(); + final Collection noNumberStats = new ArrayList<>(); + Collection errors = new ArrayList<>(); + + try { + openDBConnection(); + + for (final MetricType metricType : metricStatsCollectors) { + try { + getMetricValue(metricType).ifPresent(metricValue -> { + if (metricValue.isNumeric()) { + stats.add(metricValue); + } else { + noNumberStats.add(metricValue); + } + }); + } catch (final Throwable e) { + errors.add(new MetricCalculationError(metricType.getMetric(), e.getMessage())); + Logger.debug(MetricStatsCollector.class, () -> + "Error while calculating Metric " + metricType.getName() + ": " + e.getMessage()); + } + } + } finally { + DbConnectionFactory.closeSilently(); + } + + MetricCaches.flushAll(); + + return new MetricsSnapshot.Builder() + .stats(stats) + .notNumericStats(noNumberStats) + .errors(errors) + .build(); + } + + private static Optional getMetricValue(final MetricType metricType) throws DotDataException { + final Optional metricStatsOptional = metricType.getStat(); + + if (metricStatsOptional.isPresent()) { + final MetricValue metricValue = metricStatsOptional.get(); + + if (metricValue.getValue() instanceof Boolean) { + return Optional.of(new MetricValue(metricValue.getMetric(), + Boolean.getBoolean(metricValue.getValue().toString()) ? 1 : 0)); + } + } + + return metricStatsOptional; + } + + private static void openDBConnection() { + DbConnectionFactory.getConnection(); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricAPI.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricAPI.java new file mode 100644 index 000000000000..a918f79373fc --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricAPI.java @@ -0,0 +1,107 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotmarketing.db.HibernateUtil; +import com.dotmarketing.db.LocalTransaction; +import com.dotmarketing.exception.DotRuntimeException; + +import java.time.Instant; +import java.util.Collection; +import java.util.Map; + +/** + * A utility class to interact with the metric_temporally_table table, providing methods to save or + * flush data. This class encapsulates the logic for saving and flushing data into the + * metric_temporally_table. + *

+ * The metric_temporally_table is a special table designed to store the request to the endpoints we + * wish to track. Later, the data in this table is summarized and stored as part of the + * MetricSnapshot. + */ +public class ApiMetricAPI { + + final RequestHashCalculator requestHashCalculator = new RequestHashCalculator(); + + /** + * Return all the summary from the temporal table + * + * @return Collection of Maps with the summary data. + * + * @see ApiMetricFactory + * @see ApiMetricAPI + */ + public static Collection> getMetricTemporaryTableData() { + try { + return ApiMetricFactory.INSTANCE.getMetricTemporaryTableData(); + } finally { + HibernateUtil.closeSessionSilently(); + } + } + + /** + * Save an Endpoint request to the metric_temporally_table. + *

+ * This is saved on an async way. + * + * @param apiMetricType Metric to be saved + * @param request Request data + */ + public void save(final ApiMetricType apiMetricType, + final ApiMetricWebInterceptor.RereadInputStreamRequest request) { + + final String requestHash = requestHashCalculator.calculate(apiMetricType, request); + final ApiMetricRequest metricAPIHit = new ApiMetricRequest.Builder() + .setMetric(apiMetricType.getMetric()) + .setTime(Instant.now()) + .setHash(requestHash) + .build(); + + ApiMetricFactorySubmitter.INSTANCE.saveAsync(metricAPIHit); + } + + /** + * Before beginning to collect endpoint requests, this function must be called. It saves a + * starting register to the metric_temporally_table, indicating the initiation of data + * collection + */ + public void startCollecting() { + try { + LocalTransaction.wrap(ApiMetricFactory.INSTANCE::saveStartEvent); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void flushTemporalTable() { + try { + LocalTransaction.wrap(() -> { + ApiMetricFactory.INSTANCE.flushTemporaryTable(); + startCollecting(); + }); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + /** + * Create the metrics_temp table + */ + public void createTemporaryTable() { + try { + LocalTransaction.wrap(ApiMetricFactory.INSTANCE::createTemporaryTable); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + /** + * Drop the metrics_temp table + */ + public void dropTemporaryTable() { + try { + LocalTransaction.wrap(ApiMetricFactory.INSTANCE::dropTemporaryTable); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactory.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactory.java new file mode 100644 index 000000000000..781c91de0972 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactory.java @@ -0,0 +1,180 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotcms.business.CloseDBIfOpened; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.vavr.control.Try; +import org.postgresql.util.PGobject; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Collection; +import java.util.Map; + +/** + * A utility class to interact with the metrics_temp table, providing the more basic methods to + * create/drop the table or save/flush data. + *

+ * This class encapsulates the queries for saving/flushing data into the metrics_temp and + * creating/dropping the whole table. + *

+ * The metrics_temp is a special table designed to store the request to the endpoints we wish to + * track. Later, the data in this table is summarized and stored as part of the MetricSnapshot. + */ +public enum ApiMetricFactory { + + INSTANCE; + + private static final String GET_DATA_FROM_TEMPORARY_METRIC_TABLE = + "SELECT " + + "metric_type->>'feature' as feature, " + + "metric_type->>'category' as category, " + + "metric_type->>'name' as name, " + + "COUNT(*) / %1$.2f AS average_per_hour, " + + "COUNT(distinct hash) / %1$.2f AS unique_average_per_hour " + + "FROM metrics_temp " + + "GROUP BY metric_type->>'feature', " + + "metric_type->>'category', " + + "metric_type->>'name' " + + "HAVING metric_type->>'name' IS NOT null"; + private static final String OVERALL_QUERY = "SELECT (EXTRACT(epoch FROM now()) - EXTRACT" + + "(epoch FROM MIN(timestamp)))/3600 " + + "as overall FROM metrics_temp"; + + final ObjectMapper jsonMapper = new ObjectMapper(); + + ApiMetricFactory() { + + } + + /** + * Save request on the metrics_temp + * + * @param apiMetricRequest request + */ + public void save(final ApiMetricRequest apiMetricRequest) { + try { + final String jsonStr = jsonMapper.writeValueAsString(apiMetricRequest.getMetric()); + + final PGobject jsonObject = new PGobject(); + jsonObject.setType("json"); + Try.run(() -> jsonObject.setValue(jsonStr)).getOrElseThrow( + () -> new IllegalArgumentException("Invalid JSON")); + + new DotConnect() + .setSQL("INSERT INTO metrics_temp (timestamp, metric_type, hash) VALUES (?, " + + "?, ?)") + .addParam(OffsetDateTime.now(ZoneOffset.UTC)) + .addParam(jsonObject) + .addParam(apiMetricRequest.getHash()) + .loadResults(); + + } catch (JsonProcessingException | DotDataException e) { + throw new DotRuntimeException(e); + } + } + + + /** + * Save a register with just the current time as timestamp, it is used to mark when we start + * collecting the data. + */ + public void saveStartEvent() { + try { + new DotConnect() + .setSQL("INSERT INTO metrics_temp (timestamp) VALUES (?)") + .addParam(OffsetDateTime.now(ZoneOffset.UTC)) + .loadResults(); + + } catch (DotDataException e) { + throw new DotRuntimeException(e); + } + } + + /** + * Drop all the registers on the table + */ + public void flushTemporaryTable() { + try { + new DotConnect() + .setSQL("DELETE from metrics_temp") + .loadResults(); + + } catch (DotDataException e) { + throw new DotRuntimeException(e); + } + } + + /** + * Create the metrics_temp table + * + * @throws DotDataException if something wrong happened + */ + public void createTemporaryTable() throws DotDataException { + new DotConnect().setSQL("CREATE TABLE metrics_temp (\n" + + " timestamp TIMESTAMPTZ,\n" + + " metric_type JSON,\n" + + " hash VARCHAR(255)\n" + + ")") + .loadResults(); + + saveStartEvent(); + } + + /** + * Drop the metrics_temp table + * + * @throws DotDataException if something wrong happened + */ + public void dropTemporaryTable() throws DotDataException { + new DotConnect().setSQL("DROP TABLE IF EXISTS metrics_temp").loadResults(); + } + + /** + * return the amount of hours between we start collecting the data until we are generating the + * summary + * + * @return the amount of hours + * + * @throws DotDataException An error occurred when accessing the database. + */ + @CloseDBIfOpened + @SuppressWarnings("unchecked") + private double getOverall() throws DotDataException { + final DotConnect dotConnect = new DotConnect(); + + return Double.parseDouble(((Map) dotConnect.setSQL(OVERALL_QUERY) + .loadResults() + .get(0)) + .get("overall") + .toString() + ); + } + + /** + * Return all the summary from the temporal table + * + * @return a collection of maps with the summary data. + * + * @see ApiMetricFactory + * @see ApiMetricAPI + */ + @CloseDBIfOpened + @SuppressWarnings("unchecked") + public Collection> getMetricTemporaryTableData() { + try { + final DotConnect dotConnect = new DotConnect(); + + final double overall = getOverall(); + final String sql = String.format(GET_DATA_FROM_TEMPORARY_METRIC_TABLE, overall); + + return dotConnect.setSQL(sql).loadResults(); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactorySubmitter.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactorySubmitter.java new file mode 100644 index 000000000000..c0f9ee01f553 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricFactorySubmitter.java @@ -0,0 +1,68 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotcms.concurrent.DotConcurrentFactory; +import com.dotcms.concurrent.DotSubmitter; +import com.dotmarketing.db.LocalTransaction; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Config; + +/** + * Utility class for submitting a metric to be stored on the Telemetry Endpoint. All the request are + * send on async way. + */ +public enum ApiMetricFactorySubmitter { + + INSTANCE; + + static final int API_METRIC_SUBMITTER_POOL_SIZE = Config.getIntProperty( + "API_METRIC_SUBMITTER_POOL_SIZE", 1); + static final int API_METRIC_SUBMITTER_MAX_POOL_SIZE = Config.getIntProperty( + "API_METRIC_SUBMITTER_MAX_POOL_SIZE", 40); + + private DotSubmitter submitter; + + ApiMetricFactorySubmitter() { + + } + + /** + * Start the Submitter to be ready to send Metric + */ + public void start() { + final String submitterName = "MetricApiHitStorageSubmitter" + Thread.currentThread().getName(); + + submitter = DotConcurrentFactory.getInstance().getSubmitter(submitterName, + new DotConcurrentFactory.SubmitterConfigBuilder() + .poolSize(API_METRIC_SUBMITTER_POOL_SIZE) + .maxPoolSize(API_METRIC_SUBMITTER_MAX_POOL_SIZE) + .queueCapacity(Integer.MAX_VALUE) + .build() + ); + + } + + /** + * Request the telemetry endpoint asynchronously. + * + * @param metricAPIRequest request + */ + public void saveAsync(final ApiMetricRequest metricAPIRequest) { + submitter.submit(() -> save(metricAPIRequest)); + } + + private void save(final ApiMetricRequest metricAPIRequest) { + try { + LocalTransaction.wrap(() -> ApiMetricFactory.INSTANCE.save(metricAPIRequest)); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + /** + * Stop the submitter. + */ + public void shutdownNow() { + submitter.shutdownNow(); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricRequest.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricRequest.java new file mode 100644 index 000000000000..be6f42f267bd --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricRequest.java @@ -0,0 +1,70 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotcms.telemetry.Metric; + +import java.time.Instant; + +/** + * Represent a request to a Endpoint that we wish to track + * + * @see ApiMetricFactory + */ +public class ApiMetricRequest { + + private final Metric metric; + private final Instant time; + private final String hash; + + public ApiMetricRequest(final Builder builder) { + this.metric = builder.metric; + this.time = builder.time; + this.hash = builder.hash; + } + + public Metric getMetric() { + return metric; + } + + public Instant getTime() { + return time; + } + + public String getHash() { + return hash; + } + + @Override + public String toString() { + return "ApiMetricRequest{" + + "metric=" + metric + + ", time=" + time + + ", hash='" + hash + '\'' + + '}'; + } + + public static class Builder { + private Metric metric; + private Instant time; + private String hash; + + public Builder setMetric(final Metric metric) { + this.metric = metric; + return this; + } + + public Builder setTime(final Instant time) { + this.time = time; + return this; + } + + public Builder setHash(final String hash) { + this.hash = hash; + return this; + } + + public ApiMetricRequest build() { + return new ApiMetricRequest(this); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricType.java new file mode 100644 index 000000000000..34660ffb00ea --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricType.java @@ -0,0 +1,114 @@ +package com.dotcms.telemetry.collectors.api; + + +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.util.MetricCaches; +import com.dotcms.rest.api.v1.HTTPMethod; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.text.DecimalFormat; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; + +/** + * This class is designed to track how many times a specific API endpoint is hit. To create a metric + * that counts the number of times a particular endpoint is called, you should extend this class. + * + *

How It Works

+ *

+ * A process is in place to collect each hit on the endpoint. A listener monitors every hit and + * stores the data in a database table called metrics_temp. When metrics are calculated, the + * necessary data is retrieved from this table. If you want more details on how this process works, + * you can refer to the accompanying diagram. + *

+ * This class, or any class that extends it, is used to gather the data that will be included and + * categorized in the MetricSnapshot. The APIMetricType class overrides the getValue method from + * MetricType and also introduces new methods that can be implemented for any custom APIMetricType + * you want to create. + * + *

Method Details

+ *

+ * - getValue(): This method is overridden to filter data from the metrics_temp table based on the + * implementation of the getName, getFeature, and getCategory methods. It returns the average number + * of hits per hour for a specific endpoint. The endpoint to be monitored is determined by methods + * that each APIMetricType must implement. - getAPIUrl(): Returns a string representing the URL of + * the endpoint to monitor. The URL should be relative. For instance, if dotCMS is running at + * http://localhost:8080, and you want to track hits to the URL /contentAsset/image, you would + * return the string contentAsset/image (excluding the domain name, protocol, and port). - + * getHttpMethod(): Returns an HTTPMethod specifying which HTTP method (GET, POST, etc.) to monitor + * for the endpoint. - shouldCount(): Returns a boolean value to determine whether a specific hit + * should be counted. For example, you can set /contentAsset/image as the URL and GET as the HTTP + * method, and use this method to check for specific parameters in the request. The hit is only + * counted if the parameter is present. + *

+ * Additionally, you need to override the metadata methods getName, getCategory, and getFeature from + * MetricType, as these are not overridden by APIMetricType. + */ +public abstract class ApiMetricType implements MetricType { + + private static final DecimalFormat FORMAT = new DecimalFormat("0.##"); + + /** + * Url of the Endpoint + * + * @return the URL of the endpoint + */ + public abstract String getAPIUrl(); + + /** + * Http method of the Endpoint + * + * @return the HTTP method of the endpoint + */ + public abstract HTTPMethod getHttpMethod(); + + @Override + public final Optional getValue() { + final Optional> metricTypeItem = getMetricTypeItem(); + + return metricTypeItem.flatMap(item -> Optional.ofNullable(item.get("average_per_hour"))) + .or(() -> Optional.of(0)); + } + + /** + * Returns the unique value, indicating how many times the Endpoint was called with the same + * request. To determine if the request is the same, the method checks the following: + *
    + *
  • Query Parameters
    • + *
  • URL Parameters
    • + *
  • Request Body
    • + *
+ *

All of these need to be exactly the same to be considered the same request.

+ * + * @return the unique value + */ + public final Object getUnique() { + final double uniqueAveragePerHour = getMetricTypeItem() + .map(item -> item.get("unique_average_per_hour")) + .map(value -> Double.parseDouble(value.toString())) + .orElse(0d); + return FORMAT.format(uniqueAveragePerHour); + } + + /** + * @return + */ + private Optional> getMetricTypeItem() { + final Collection> result = MetricCaches.TEMPORARY_TABLA_DATA.get(); + + return result.stream() + .filter(item -> item.get("feature").toString().equals(getFeature().name())) + .filter(item -> item.get("category").toString().equals(getCategory().name())) + .filter(item -> item.get("name").toString().equals(getName())) + .limit(1) + .findFirst(); + } + + public boolean shouldCount(final HttpServletRequest request, + final HttpServletResponse response) { + return true; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricTypes.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricTypes.java new file mode 100644 index 000000000000..25e4c4c7f8e6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricTypes.java @@ -0,0 +1,63 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotcms.telemetry.collectors.image.CountOfContentAssetImageBEAPICalls; +import com.dotcms.telemetry.collectors.image.CountOfContentAssetImageFEAPICalls; +import com.dotcms.telemetry.collectors.image.CountOfDAImageBEAPICalls; +import com.dotcms.telemetry.collectors.image.CountOfDAImageFEAPICalls; +import com.dotcms.rest.api.v1.HTTPMethod; + +import javax.servlet.http.HttpServletRequest; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Collection of all the {@link ApiMetricType} + */ +public enum ApiMetricTypes { + + INSTANCE; + + final Collection urlToIntercept; + + ApiMetricTypes() { + urlToIntercept = new HashSet<>(); + urlToIntercept.add(new CountOfContentAssetImageFEAPICalls()); + urlToIntercept.add(new CountOfDAImageFEAPICalls()); + urlToIntercept.add(new CountOfContentAssetImageBEAPICalls()); + urlToIntercept.add(new CountOfDAImageBEAPICalls()); + + } + + public List interceptBy(final HttpServletRequest request) { + + final Optional method = getHttpMethod(request); + + if (method.isPresent()) { + final String uri = request.getRequestURI().replace("/api/", ""); + + return urlToIntercept.stream() + .filter(apiMetricType -> apiMetricType.getHttpMethod().equals(method.get()) && + uri.startsWith(apiMetricType.getAPIUrl())) + .collect(Collectors.toList()); + } + + return Collections.emptyList(); + } + + private Optional getHttpMethod(final HttpServletRequest request) { + try { + return Optional.of(HTTPMethod.valueOf(request.getMethod())); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } + } + + public Collection get() { + return Collections.unmodifiableCollection(urlToIntercept); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricWebInterceptor.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricWebInterceptor.java new file mode 100644 index 000000000000..fca8932ad538 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/ApiMetricWebInterceptor.java @@ -0,0 +1,161 @@ +package com.dotcms.telemetry.collectors.api; + +import com.dotcms.filters.interceptor.Result; +import com.dotcms.filters.interceptor.WebInterceptor; +import com.liferay.util.servlet.ServletInputStreamWrapper; + +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Intercept all the hit to the Endpoint that we wish to track. + */ +public class ApiMetricWebInterceptor implements WebInterceptor { + + public static final ApiMetricAPI apiStatAPI = new ApiMetricAPI(); + + public ApiMetricWebInterceptor() { + ApiMetricFactorySubmitter.INSTANCE.start(); + } + + @Override + public String[] getFilters() { + return ApiMetricTypes.INSTANCE.get().stream() + .map(ApiMetricType::getAPIUrl) + .toArray(String[]::new); + + } + + @Override + public Result intercept(HttpServletRequest req, HttpServletResponse res) throws IOException { + return new Result.Builder().next() + .wrap(new RereadInputStreamRequest(req)) + .build(); + } + + @Override + public boolean afterIntercept(final HttpServletRequest req, final HttpServletResponse res) { + if (res.getStatus() == 200) { + + ApiMetricTypes.INSTANCE.interceptBy(req) + .stream() + .filter(apiMetricType -> apiMetricType.shouldCount(req, res)) + .forEach(apiMetricType -> + apiStatAPI.save(apiMetricType, (RereadInputStreamRequest) req) + ); + } + + return true; + } + + public static class RereadInputStreamRequest extends HttpServletRequestWrapper { + + private RereadAfterCloseInputStream servletInputStream; + + public RereadInputStreamRequest(final HttpServletRequest request) { + super(request); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + if (servletInputStream == null) { + servletInputStream = new RereadAfterCloseInputStream(getRequest().getInputStream()); + } + + return servletInputStream; + } + + public String getReadContent() { + int contentLength = getRequest().getContentLength(); + + if (contentLength > 0) { + return servletInputStream.getReadContent(); + } else { + return ""; + } + } + } + + /** + * This InputStream allows us to read the content multiple times as needed. It enables us to + * read the Request Body several times: once to check if the request is unique, and another time + * to process the request + */ + private static class RereadAfterCloseInputStream extends ServletInputStreamWrapper { + + final ByteArrayOutputStream byteArrayOutputStream; + private final ServletInputStream proxy; + + RereadAfterCloseInputStream(final ServletInputStream proxy) { + super(proxy); + this.proxy = proxy; + + byteArrayOutputStream = new ByteArrayOutputStream(); + } + + @Override + public int read() throws IOException { + int read = this.proxy.read(); + + if (read != -1) { + byteArrayOutputStream.write(read); + } + + return read; + } + + @Override + public int read(byte[] b) throws IOException { + int read = this.proxy.read(b); + + if (read != -1) { + byteArrayOutputStream.write(b); + } + + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int read = this.proxy.read(b, off, len); + + if (read != -1) { + byteArrayOutputStream.write(b); + } + + return read; + } + + @Override + public int readLine(byte[] b, int off, int len) throws IOException { + int read = this.proxy.readLine(b, off, len); + + if (read != -1) { + byteArrayOutputStream.write(b); + } + + return read; + } + + @Override + public void close() throws IOException { + super.close(); + byteArrayOutputStream.close(); + } + + public String getReadContent() { + + if (!this.isFinished()) { + return ""; + } + + return byteArrayOutputStream.toString(); + } + + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/RequestHashCalculator.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/RequestHashCalculator.java new file mode 100644 index 000000000000..76f2b01890db --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/api/RequestHashCalculator.java @@ -0,0 +1,192 @@ +package com.dotcms.telemetry.collectors.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.servlet.http.HttpServletRequest; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Util class to provide methods to calculate a Hash from a {@link HttpServletRequest} + */ +public class RequestHashCalculator { + + final ObjectMapper jsonMapper = new ObjectMapper(); + final MessageDigest digest; + + RequestHashCalculator() { + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Calculate a hash from a {@link HttpServletRequest} using: + *
    + *
  • The Query Parameters
    • + *
  • The body
    • + *
  • The Url Parameters
    • + *
+ *

The steps to calculate this hash are the follows: + *

    + *
  • If the Request has Query parameters the these are sort alphabetically by the + * Parameter + * Name, then 'xxx?a=1&b=2' and 'xxx?b=2&a=1' are the same. If the Request does not has any + * Query parameters then the word 'NONE' is taking instead.
    • + *
  • If the Request has Url parameters the these are sort alphabetically by the + * Parameter Name, + * then 'xxx/a/1/b/2' and 'xxx/b/2/a/1' are the same. If the Request does not has any Url + * parameters then the word 'NONE' is taking instead.
    • + *
  • If the Request has any JSON Body it is sort by alphabetically by the attribute + * name, so: + * 
    +     *          {@code { a: 1, b: 2 }}
+     *
    + * And: + * 
    +     *          {@code { b: 2, a: 1 }}
+     *
    + * are the same. + *
    • + *
  • Finally all of them are concat and a hash is calculated with that String.
    • + *
+ * + * @param apiMetricType + * @param request + * + * @return + */ + public String calculate(final ApiMetricType apiMetricType, + final ApiMetricWebInterceptor.RereadInputStreamRequest request) { + + final String queryParametersAsString = + getQueryParameters(request).map(Map::toString).orElse("NONE"); + final String urlParametersAsString = getUrlParameters(apiMetricType, request) + .map(Map::toString).orElse("NONE"); + final String bodyAsString = getBody(request).map(Map::toString).orElse("NONE"); + + final String requestString = queryParametersAsString + urlParametersAsString + bodyAsString; + + + final byte[] encodedHash = digest.digest(requestString.getBytes(StandardCharsets.UTF_8)); + return new String(encodedHash); + } + + @SuppressWarnings("unchecked") + private Optional> getBody(ApiMetricWebInterceptor.RereadInputStreamRequest request) { + final String requestContent = request.getReadContent(); + try { + if (!requestContent.isEmpty()) { + final Map bodyMap = jsonMapper.readValue(requestContent, + Map.class); + + return Optional.of(sort(bodyMap)); + } + + return Optional.empty(); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + } + + @SuppressWarnings("unchecked") + private TreeMap sort(final Map map) { + TreeMap treeMap = new TreeMap<>(map); + + for (Map.Entry bodyMapEntry : map.entrySet()) { + final Object value = bodyMapEntry.getValue(); + + if (value instanceof Map) { + final TreeMap mapSorted = sort((Map) value); + treeMap.put(bodyMapEntry.getKey(), mapSorted); + } + } + return treeMap; + } + + + private Optional> getUrlParameters(final ApiMetricType apiMetricType, + final HttpServletRequest request) { + String uri = request.getRequestURI(); + String apiUrl = apiMetricType.getAPIUrl(); + + final String urlParameters = uri.substring(uri.indexOf("/es/search") + apiUrl.length()); + + TreeMap sortedParameters = new TreeMap<>(); + + if (urlParameters != null) { + final String[] paramsAndValues = urlParameters.split("/"); + final Pair pair = new Pair(); + + for (String paramOrValue : paramsAndValues) { + pair.add(paramOrValue); + + if (pair.isCompleted()) { + sortedParameters.put(pair.getKey(), pair.getValue()); + pair.reset(); + } + } + } + + return sortedParameters.isEmpty() ? Optional.empty() : Optional.of(sortedParameters); + } + + private Optional> getQueryParameters(final HttpServletRequest request) { + String queryString = request.getQueryString(); + + TreeMap sortedParameters = new TreeMap<>(); + + if (queryString != null) { + String[] params = queryString.split("&"); + + for (String param : params) { + String[] keyValue = param.split("="); + String key = keyValue[0]; + String value = keyValue.length > 1 ? keyValue[1] : ""; + + sortedParameters.put(key, value); + } + } + + return sortedParameters.isEmpty() ? Optional.empty() : Optional.of(sortedParameters); + } + + private static class Pair { + private String key; + private String value; + + void add(final String keyOrValue) { + if (key == null) { + key = keyOrValue; + } else { + value = keyOrValue; + } + } + + boolean isCompleted() { + return key != null && value != null; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + public void reset() { + value = null; + key = null; + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLivePageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLivePageDatabaseMetricType.java new file mode 100644 index 000000000000..3ec66940f38b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLivePageDatabaseMetricType.java @@ -0,0 +1,47 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotcms.telemetry.business.MetricsAPI; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.templates.model.Template; + +import java.util.Collection; + +/** + * Total of containers used in LIVE pages, it extends from + * {@link TotalContainersInTemplateDatabaseMetricType} because it is a counter of the containers + * used in templates is this case we want to take account just the LIVE version of each one, so it + * override the follow behavior: + *
    + *
  • Searching Templates: Override the getTemplatesIds method to return just the Template that + * has LIVE version.
    • + *
  • Retrieve the Template Version: Override the getTemplate method to get the last LIVE + * version of the Template.
    • + *
+ * + * @see TotalContainersInTemplateDatabaseMetricType + */ +public abstract class TotalContainersInLivePageDatabaseMetricType extends TotalContainersInTemplateDatabaseMetricType { + + private static final String LIVE_USED_TEMPLATES_INODES_QUERY = "SELECT " + "distinct " + + "contentlet_as_json -> 'fields' -> 'template' ->> 'value' as value " + "FROM " + + "contentlet INNER JOIN contentlet_version_info ON contentlet.inode = " + + "contentlet_version_info.live_inode " + "WHERE structure_inode IN (SELECT inode FROM " + + "structure where name = 'Page') AND " + "deleted = false"; + + private Collection getLiveUsedTemplatesInodes() { + return MetricsAPI.INSTANCE.getList(LIVE_USED_TEMPLATES_INODES_QUERY); + } + + Collection getTemplatesIds() { + return getLiveUsedTemplatesInodes(); + } + + @Override + final Template getTemplate(String id) throws DotDataException, DotSecurityException { + return APILocator.getTemplateAPI().findLiveTemplate(id, APILocator.systemUser(), false); + } + +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLiveTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLiveTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..7f9e70dfcb44 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInLiveTemplatesDatabaseMetricType.java @@ -0,0 +1,62 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotcms.telemetry.business.MetricsAPI; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.templates.model.Template; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This class count the amount of containers used in LIVE templates, in this case no matter the + * pages so this templates can be used or not on a page, this class extends from + * TotalContainersIntemplateDatabaseMetricType because it is a counter of the containers used in + * templates and it override the follow behavior: + *
    + *
  • Searching Templates: Override the getTemplatesIds method to return only the Template + * IDs for the LIVE templates. This is achieved using a SQL UNION query. The first part of the + * query retrieves standard templates from the template_version_info table, identifying those + * that have LIVE versions. The second part of the query retrieves file templates from the + * contenlet_version_info table, also focusing on those with live versions.
    • + *
  • Retrieve the Template Version: Override the getTemplate method to get the last LIVE + * version of the Template.
    • + *
+ */ +public abstract class TotalContainersInLiveTemplatesDatabaseMetricType extends TotalContainersInTemplateDatabaseMetricType { + + private static final String LIVE_TEMPLATES_INODES_QUERY = "SELECT DISTINCT template" + + ".identifier as value " + + "FROM template_version_info " + + "INNER JOIN template ON template_version_info.identifier = template.identifier " + + "WHERE title NOT LIKE 'anonymous_layout_%' and deleted = false"; + + private static final String LIVE_FILE_TEMPLATES_INODES_QUERY = "SELECT distinct id" + + ".parent_path as value " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path LIKE '/application/templates/%' AND id.asset_name = 'body.vtl' " + + "AND deleted = false AND live_inode is not null"; + + @Override + Collection getTemplatesIds() { + + final List dataBaseTemplateInode = + MetricsAPI.INSTANCE.getList(LIVE_TEMPLATES_INODES_QUERY); + final List dataBaseFileTemplateInode = + MetricsAPI.INSTANCE.getList(LIVE_FILE_TEMPLATES_INODES_QUERY); + + return Stream.concat(dataBaseTemplateInode.stream(), + dataBaseFileTemplateInode.stream()).collect(Collectors.toSet()); + } + + @Override + final Template getTemplate(String id) throws DotDataException, DotSecurityException { + return APILocator.getTemplateAPI() + .findLiveTemplate(id, APILocator.systemUser(), false); + } + +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInTemplateDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInTemplateDatabaseMetricType.java new file mode 100644 index 000000000000..662fef16bcb2 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInTemplateDatabaseMetricType.java @@ -0,0 +1,115 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.MetricType; +import com.dotcms.rendering.velocity.viewtools.DotTemplateTool; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.containers.model.Container; +import com.dotmarketing.portlets.templates.design.bean.ContainerUUID; +import com.dotmarketing.portlets.templates.design.bean.TemplateLayout; +import com.dotmarketing.portlets.templates.design.util.DesignTemplateUtil; +import com.dotmarketing.portlets.templates.model.Template; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Super class to use if you want to generate a Metric to count the containers Used in templates + * + * To create a metric that counts the number of containers used in templates, you need to subclass this class. + * This class overrides the getValue method to perform a search for containers, which can be customized in your implementation. + * Here's what the new implementation of the getValue method does: + * - Searching Templates: This method identifies which templates to consider. You can customize the selection by overriding + * the getTemplateIds method in your subclass. For example, you might decide to include only LIVE, WORKING, + * or Advanced Templates. To apply your own criteria for filtering templates, you need to override the getTemplateIds method. + * - Retrieve the Template Version: You can customize this step to suit your needs, such as selecting the most recent + * LIVE or WORKING version. To modify this behavior, override the getTemplate method. + * - Retrieve Containers: It gathers the containers used by each template version selected on the two previous steps. + * - Filtering Containers: The method filters the containers based on your specific requirements. + * You can also customize this step by overriding the filterContainer method in your subclass. + * + * @see TotalContainersInLivePageDatabaseMetricType + * @see TotalContainersInWorkingPageDatabaseMetricType + */ +public abstract class TotalContainersInTemplateDatabaseMetricType implements MetricType { + + @Override + public final MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public final MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public final Optional getValue() { + final Collection templatesIds = getTemplatesIds(); + final Set containersUsed = new HashSet<>(); + + try { + for (String id : templatesIds) { + final Template template = getTemplate(id); + + if (template == null) { + continue; + } + + final Collection containersId = getContainersInTemplate(template).stream() + .filter(this::filterContainer) + .collect(Collectors.toSet()); + containersUsed.addAll(containersId); + } + + return Optional.of(containersUsed.size()); + } catch (DotDataException | DotSecurityException e) { + throw new DotRuntimeException(e); + } + + + } + + abstract boolean filterContainer(final String containerId); + + public Collection getContainersInTemplate(final Template template) { + final Collection containerUUIDs; + + if(Boolean.TRUE.equals(template.isDrawed())) { + final TemplateLayout layout = template.getDrawedBody() != null ? getLayout(template.getDrawedBody()) : null; + containerUUIDs = layout != null ? APILocator.getTemplateAPI().getContainersUUID(layout) : + Collections.emptySet(); + } else { + containerUUIDs = new HashSet<>( + APILocator.getTemplateAPI().getContainersUUIDFromDrawTemplateBody(template.getBody())); + } + + return containerUUIDs.stream() + .map(ContainerUUID::getIdentifier) + .filter(identifier -> !Container.SYSTEM_CONTAINER.equals(identifier)) + .distinct() + .collect(Collectors.toList()); + } + + private TemplateLayout getLayout(final String drawedBody) { + + try { + return DotTemplateTool.getTemplateLayoutFromJSON(drawedBody); + } catch (IOException var4) { + return DesignTemplateUtil.getDesignParameters(drawedBody, false); + } + } + + abstract Collection getTemplatesIds(); + abstract Template getTemplate(final String id) throws DotDataException, DotSecurityException; +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInWorkingPageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInWorkingPageDatabaseMetricType.java new file mode 100644 index 000000000000..efd4f5c9f708 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalContainersInWorkingPageDatabaseMetricType.java @@ -0,0 +1,89 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotcms.telemetry.business.MetricsAPI; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.templates.model.Template; + +import java.util.Collection; + +/** + * Total of containers used in Working pages. + * + * This class count the amount of containers used in WORKING pages, it means all these pages that don’t have LIVE Version, + * this class extends from {@link TotalContainersInTemplateDatabaseMetricType} because it is a counter + * of the containers used in templates is this case we want to take account just the WORKING version of each one, + * so it override the follow behavior: + * + * - Searching Templates: Override the getTemplatesIds method to return only the Template IDs for + * the templates used on a WORKING page.This is achieved using a SQL UNION query. + * The first part of the query retrieves standard templates from the template_version_info table, + * identifying those that have only a working version. + * The second part of the query retrieves file templates from the contenlet_version_info table, + * also focusing on those with just a working version. + * + * - Retrieve the Template Version: Override the getTemplate method to get the last WORKING version of the Template. + */ +public abstract class TotalContainersInWorkingPageDatabaseMetricType extends TotalContainersInTemplateDatabaseMetricType { + + private static final String WORKING_USED_TEMPLATES_INODES_QUERY = + "SELECT tvi.identifier AS value " + + "FROM template_version_info tvi " + + "INNER JOIN ( " + + "SELECT " + + "DISTINCT contentlet_as_json -> 'fields' -> 'template' ->> 'value' AS template_id, " + + "cvi.working_inode, " + + "cvi.live_inode " + + "FROM contentlet c " + + "INNER JOIN contentlet_version_info cvi ON c.inode = cvi.working_inode " + + "WHERE c.structure_inode IN (SELECT inode FROM structure WHERE name = 'Page') " + + "AND cvi.deleted = false " + + ") page ON page.template_id = tvi.identifier " + + "WHERE tvi.deleted = false " + + "AND ( " + + "(page.live_inode IS NOT NULL AND (tvi.live_inode IS NULL OR tvi.live_inode <> tvi.working_inode)) " + + "OR page.live_inode IS NULL " + + ")"; + + private static final String WORKING_USED_FILE_TEMPLATES_INODES_QUERY = + "SELECT id.id AS value " + + "FROM identifier id " + + "INNER JOIN ( " + + "SELECT contentlet_as_json->'fields'->'hostName'->>'value' hostName, identifier " + + "FROM contentlet " + + ") host ON host.identifier = id.host_inode " + + "INNER JOIN ( " + + "SELECT " + + "DISTINCT contentlet_as_json -> 'fields' -> 'template' ->> 'value' AS template_path, " + + "cvi.working_inode, " + + "cvi.live_inode " + + "FROM contentlet c " + + "INNER JOIN contentlet_version_info cvi ON c.inode = cvi.working_inode " + + "WHERE c.structure_inode IN (SELECT inode FROM structure WHERE name = 'Page') " + + "AND cvi.deleted = false " + + ") page ON page.template_path = CONCAT('//', hostName, id.parent_path) AND asset_name = 'body.vtl' " + + "INNER JOIN contentlet_version_info cvi on id.id = cvi.identifier " + + "WHERE cvi.deleted = false " + + "AND ( " + + "(page.live_inode IS NOT NULL AND (cvi.live_inode IS NULL OR cvi.live_inode <> cvi.working_inode)) " + + "OR page.live_inode IS NULL " + + ")"; + + private Collection getWorkingUsedTemplatesInodes() { + return MetricsAPI.INSTANCE.getList(WORKING_USED_TEMPLATES_INODES_QUERY + " UNION " + + WORKING_USED_FILE_TEMPLATES_INODES_QUERY); + } + + @Override + Collection getTemplatesIds() { + return getWorkingUsedTemplatesInodes(); + } + + @Override + Template getTemplate(String id) throws DotDataException, DotSecurityException { + return APILocator.getTemplateAPI() + .findWorkingTemplate(id, APILocator.systemUser(), false); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLivePageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLivePageDatabaseMetricType.java new file mode 100644 index 000000000000..2b5058cc60bf --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLivePageDatabaseMetricType.java @@ -0,0 +1,25 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of FILE containers used in LIVE pages + */ +public class TotalFileContainersInLivePageDatabaseMetricType extends TotalContainersInLivePageDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_FILE_CONTAINERS_USED_IN_LIVE_PAGES"; + } + + @Override + public String getDescription() { + return "Count of FILE containers used in LIVE pages"; + } + + @Override + boolean filterContainer(final String containerId) { + return FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLiveTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLiveTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..dd30ec4fe6c7 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInLiveTemplatesDatabaseMetricType.java @@ -0,0 +1,24 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of FILE containers used in LIVE templates + */ +public class TotalFileContainersInLiveTemplatesDatabaseMetricType extends TotalContainersInLiveTemplatesDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_FILE_CONTAINERS_USED_IN_LIVE_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Total of FILE containers used in LIVE templates"; + } + @Override + boolean filterContainer(final String containerId) { + return FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInWorkingPageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInWorkingPageDatabaseMetricType.java new file mode 100644 index 000000000000..40d1cabd2095 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalFileContainersInWorkingPageDatabaseMetricType.java @@ -0,0 +1,25 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of FILE containers used in LIVE pages + */ +public class TotalFileContainersInWorkingPageDatabaseMetricType extends TotalContainersInWorkingPageDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_FILE_CONTAINERS_USED_IN_WORKING_PAGES"; + } + + @Override + public String getDescription() { + return "Count of FILE containers used in WORKING pages"; + } + + @Override + boolean filterContainer(final String containerId) { + return FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLivePageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLivePageDatabaseMetricType.java new file mode 100644 index 000000000000..e0d09c899944 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLivePageDatabaseMetricType.java @@ -0,0 +1,25 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of STANDARD containers used in LIVE pages + */ +public class TotalStandardContainersInLivePageDatabaseMetricType extends TotalContainersInLivePageDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_STANDARD_CONTAINERS_USED_IN_LIVE_PAGES"; + } + + @Override + public String getDescription() { + return "Count of STANDARD containers used in LIVE pages"; + } + + @Override + boolean filterContainer(final String containerId) { + return !FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLiveTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLiveTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..a29555de8e2d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInLiveTemplatesDatabaseMetricType.java @@ -0,0 +1,24 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of STANDARD containers used in LIVE templates + */ +public class TotalStandardContainersInLiveTemplatesDatabaseMetricType extends TotalContainersInLiveTemplatesDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_STANDARD_CONTAINERS_USED_IN_LIVE_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Total of STANDARD containers used in LIVE templates"; + } + @Override + boolean filterContainer(final String containerId) { + return !FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInWorkingPageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInWorkingPageDatabaseMetricType.java new file mode 100644 index 000000000000..27a1aaf75e2c --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/container/TotalStandardContainersInWorkingPageDatabaseMetricType.java @@ -0,0 +1,24 @@ +package com.dotcms.telemetry.collectors.container; + +import com.dotmarketing.portlets.containers.business.FileAssetContainerUtil; + +/** + * Total of STANDARD containers used in WORKING pages + */ +public class TotalStandardContainersInWorkingPageDatabaseMetricType extends TotalContainersInWorkingPageDatabaseMetricType { + + @Override + public String getName() { + return "COUNT_STANDARD_CONTAINERS_USED_IN_WORKING_PAGES"; + } + + @Override + public String getDescription() { + return "Count of STANDARD containers used in WORKING pages"; + } + @Override + boolean filterContainer(final String containerId) { + return !FileAssetContainerUtil.getInstance().isFolderAssetContainerId(containerId); + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LastContentEditedDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LastContentEditedDatabaseMetricType.java new file mode 100644 index 000000000000..5d04aa22d970 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LastContentEditedDatabaseMetricType.java @@ -0,0 +1,35 @@ +package com.dotcms.telemetry.collectors.content; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the modification date of the most recently edited Contentlet + */ +public class LastContentEditedDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "LAST_CONTENT_EDITED"; + } + + @Override + public String getDescription() { + return "Mod date of the most recently edited Contentlet"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENTLETS; + } + + @Override + public String getSqlQuery() { + return "SELECT to_char (max(version_ts),'HH12:MI:SS DD Mon YYYY') AS value FROM contentlet_version_info"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LiveNotDefaultLanguageContentsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LiveNotDefaultLanguageContentsDatabaseMetricType.java new file mode 100644 index 000000000000..b1b93177e8ae --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/LiveNotDefaultLanguageContentsDatabaseMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.content; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of contentlets which have at least one live version in a non-default Language + */ +public class LiveNotDefaultLanguageContentsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "LIVE_NOT_DEFAULT_LANGUAGE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Live Content items with non-default Language versions"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENTLETS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT identifier) AS value " + + "FROM contentlet_version_info " + + "WHERE live_inode IS NOT null " + + "AND lang NOT IN (SELECT default_language_id FROM company)"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/RecentlyEditedContentDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/RecentlyEditedContentDatabaseMetricType.java new file mode 100644 index 000000000000..bc439785797f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/RecentlyEditedContentDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.content; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Contentlets that were edited less than a month ago + */ +public class RecentlyEditedContentDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "CONTENTS_RECENTLY_EDITED"; + } + + @Override + public String getDescription() { + return "Count of Contentlets that were Edited less than a month ago"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENTLETS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(working_inode) AS value FROM contentlet_version_info, contentlet " + + "WHERE contentlet.inode = contentlet_version_info.working_inode AND " + + "contentlet.mod_date > now() - interval '1 month'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/TotalContentsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/TotalContentsDatabaseMetricType.java new file mode 100644 index 000000000000..1b0fc6744d30 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/TotalContentsDatabaseMetricType.java @@ -0,0 +1,35 @@ +package com.dotcms.telemetry.collectors.content; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total number of contentlets + */ +public class TotalContentsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT"; + } + + @Override + public String getDescription() { + return "Total number of Contentlets"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENTLETS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(working_inode) AS value FROM contentlet_version_info"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/WorkingNotDefaultLanguageContentsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/WorkingNotDefaultLanguageContentsDatabaseMetricType.java new file mode 100644 index 000000000000..3f568e45e9fe --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/content/WorkingNotDefaultLanguageContentsDatabaseMetricType.java @@ -0,0 +1,44 @@ +package com.dotcms.telemetry.collectors.content; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of Contentlets that has at least one working version on any non-default language and that + * also don't have live version on any non-default language + */ +public class WorkingNotDefaultLanguageContentsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "WORKING_NOT_DEFAULT_LANGUAGE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Working Content items with non-default Language versions"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENTLETS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT identifier) AS value " + + "FROM contentlet_version_info AS cvi1 " + + "WHERE working_inode is not null and live_inode is null " + + "AND lang NOT IN (SELECT default_language_id FROM company) " + + "AND (SELECT COUNT(DISTINCT identifier) " + + "FROM contentlet_version_info AS cvi2 " + + "WHERE live_inode IS NOT null " + + "AND cvi1.identifier = cvi2.identifier " + + "AND lang NOT IN (SELECT default_language_id FROM company) ) = 0"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/ContentTypeFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/ContentTypeFieldsMetricType.java new file mode 100644 index 000000000000..65fdf23b3654 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/ContentTypeFieldsMetricType.java @@ -0,0 +1,64 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.MetricValue; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.db.LocalTransaction; +import com.dotmarketing.exception.DotRuntimeException; + +import javax.ws.rs.NotSupportedException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Collects the total number of contentlets + */ +public abstract class ContentTypeFieldsMetricType implements MetricType { + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.CONTENT_TYPE_FIELDS; + } + + @Override + public Optional getStat() { + try { + return LocalTransaction.wrapReturn(() -> { + final String sqlQuery = "SELECT field_type, count(*)\n" + + "FROM field GROUP BY field_type"; + final DotConnect dotConnect = new DotConnect(); + final List> loadObjectResults = dotConnect.setSQL(sqlQuery) + .loadObjectResults(); + + return getValue(loadObjectResults) + .map(o -> new MetricValue(this.getMetric(), o)) + .or(() -> Optional.of(new MetricValue(this.getMetric(), 0))); + }); + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + + @Override + public Optional getValue() { + throw new NotSupportedException(); + } + + abstract boolean filterCondition(Map map); + + Optional getValue(List> results) { + return Optional.of(results.stream() + .filter(this::filterCondition) + .map(m -> m.get("count")) + .findFirst() + .orElse(0)); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBinaryFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBinaryFieldsMetricType.java new file mode 100644 index 000000000000..24e2227fec80 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBinaryFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfBinaryFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.BinaryField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_BINARY_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of binary fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBlockEditorFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBlockEditorFieldsMetricType.java new file mode 100644 index 000000000000..ddf230571c02 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfBlockEditorFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfBlockEditorFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.StoryBlockField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_BLOCK_EDITOR_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of block editor fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCategoryFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCategoryFieldsMetricType.java new file mode 100644 index 000000000000..e9b0c6bff5f4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCategoryFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfCategoryFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.CategoryField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_CATEGORY_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of category fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCheckboxFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCheckboxFieldsMetricType.java new file mode 100644 index 000000000000..a351d188229d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfCheckboxFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfCheckboxFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.CheckboxField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_CHECKBOX_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of checkbox fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfColumnsFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfColumnsFieldsMetricType.java new file mode 100644 index 000000000000..8b16c4b4d470 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfColumnsFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfColumnsFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.ColumnField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_COLUMN_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of column fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfConstantFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfConstantFieldsMetricType.java new file mode 100644 index 000000000000..d1aff0016ef4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfConstantFieldsMetricType.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfConstantFieldsMetricType extends ContentTypeFieldsMetricType { + @Override + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.ConstantField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_CONSTANT_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of constant fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateFieldsMetricType.java new file mode 100644 index 000000000000..fa781af8ad1d --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateFieldsMetricType.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfDateFieldsMetricType extends ContentTypeFieldsMetricType { + @Override + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.DateField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_DATE_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of date fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateTimeFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateTimeFieldsMetricType.java new file mode 100644 index 000000000000..c9629a8a7e38 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfDateTimeFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfDateTimeFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.DateTimeField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_DATE_TIME_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of date time fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfFileFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfFileFieldsMetricType.java new file mode 100644 index 000000000000..b0bfae4c0a17 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfFileFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfFileFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.FileField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_FILE_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of file fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfHiddenFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfHiddenFieldsMetricType.java new file mode 100644 index 000000000000..071977c25d73 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfHiddenFieldsMetricType.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfHiddenFieldsMetricType extends ContentTypeFieldsMetricType { + @Override + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.HiddenField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_HIDDEN_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of hidden fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfImageFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfImageFieldsMetricType.java new file mode 100644 index 000000000000..1ab981d48d15 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfImageFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfImageFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.ImageField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_IMAGE_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of image fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfJSONFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfJSONFieldsMetricType.java new file mode 100644 index 000000000000..7bd347c9bcf8 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfJSONFieldsMetricType.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfJSONFieldsMetricType extends ContentTypeFieldsMetricType { + @Override + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.JSONField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_JSON_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of JSON fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfKeyValueFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfKeyValueFieldsMetricType.java new file mode 100644 index 000000000000..c0b5b1448506 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfKeyValueFieldsMetricType.java @@ -0,0 +1,20 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfKeyValueFieldsMetricType extends ContentTypeFieldsMetricType { + @Override + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.KeyValueField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_KEY_VALUE_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of key/value fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfLineDividersFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfLineDividersFieldsMetricType.java new file mode 100644 index 000000000000..9769d5ab64a4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfLineDividersFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfLineDividersFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.LineDividerField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_LINE_DIVIDER_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of line divider fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfMultiselectFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfMultiselectFieldsMetricType.java new file mode 100644 index 000000000000..5888ca841a66 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfMultiselectFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfMultiselectFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.MultiSelectField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_MULTI_SELECT_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of multi select fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfPermissionsFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfPermissionsFieldsMetricType.java new file mode 100644 index 000000000000..a3f0b59d8d56 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfPermissionsFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfPermissionsFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.PermissionTabField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_PERMISSIONS_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of permissions fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRadioFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRadioFieldsMetricType.java new file mode 100644 index 000000000000..ceabb66da02b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRadioFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfRadioFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.RadioField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_RADIO_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of radio fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRelationshipFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRelationshipFieldsMetricType.java new file mode 100644 index 000000000000..65a3e95ed283 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRelationshipFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfRelationshipFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.RelationshipField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_RELATIONSHIP_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of relationship fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRowsFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRowsFieldsMetricType.java new file mode 100644 index 000000000000..b3f8e35ec68e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfRowsFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfRowsFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.RowField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_ROW_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of row fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSelectFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSelectFieldsMetricType.java new file mode 100644 index 000000000000..cb0cbec5d04f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSelectFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfSelectFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.SelectField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_SELECT_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of select fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSiteOrFolderFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSiteOrFolderFieldsMetricType.java new file mode 100644 index 000000000000..f3e4f20585aa --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfSiteOrFolderFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfSiteOrFolderFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.HostFolderField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_SITE_OR_FOLDER_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of site or folder fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTabFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTabFieldsMetricType.java new file mode 100644 index 000000000000..b3bb6e76ef6c --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTabFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfTabFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.TabDividerField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_TAB_DIVIDER_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of tab divider fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTagFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTagFieldsMetricType.java new file mode 100644 index 000000000000..06da4b0e9aa9 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTagFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfTagFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.TagField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_TAG_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of tag fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextAreaFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextAreaFieldsMetricType.java new file mode 100644 index 000000000000..d9bf584040c4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextAreaFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfTextAreaFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.TextAreaField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_TEXT_AREA_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of text area fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextFieldsMetricType.java new file mode 100644 index 000000000000..dd96246b2672 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTextFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfTextFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.TextField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_TEXT_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of text fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTimeFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTimeFieldsMetricType.java new file mode 100644 index 000000000000..ec67e4791a8f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfTimeFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfTimeFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.TimeField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_TIME_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of time fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfWYSIWYGFieldsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfWYSIWYGFieldsMetricType.java new file mode 100644 index 000000000000..1319d4f30b0b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/contenttype/CountOfWYSIWYGFieldsMetricType.java @@ -0,0 +1,19 @@ +package com.dotcms.telemetry.collectors.contenttype; + +import java.util.Map; + +public class CountOfWYSIWYGFieldsMetricType extends ContentTypeFieldsMetricType { + boolean filterCondition(Map map) { + return "com.dotcms.contenttype.model.field.WysiwygField".equals(map.get("field_type")); + } + + @Override + public String getName() { + return "COUNT_WYSIWYG_FIELDS"; + } + + @Override + public String getDescription() { + return "Count the number of WYSIWYG fields"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageBEAPICalls.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageBEAPICalls.java new file mode 100644 index 000000000000..7d1203965dc4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageBEAPICalls.java @@ -0,0 +1,58 @@ +package com.dotcms.telemetry.collectors.image; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.api.ApiMetricType; +import com.dotcms.rest.api.v1.HTTPMethod; +import com.dotmarketing.util.PageMode; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Represents a Metric to count how many times the Endpoint '/contentAsset/image/' is called from + * Back End. + */ +public class CountOfContentAssetImageBEAPICalls extends ApiMetricType { + + @Override + public String getName() { + return "COUNT_OF_BE_CONTENTASSET_CALLS"; + } + + @Override + public String getDescription() { + return "Count of 'contentAsset/image' API calls from Back End"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.IMAGE_API; + } + + + @Override + public String getAPIUrl() { + return "/contentAsset/image/"; + } + + @Override + public HTTPMethod getHttpMethod() { + return HTTPMethod.GET; + } + + @Override + @JsonIgnore + public boolean shouldCount(final HttpServletRequest request, + final HttpServletResponse response) { + final PageMode currentPageMode = PageMode.get(request); + return currentPageMode != PageMode.LIVE; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageFEAPICalls.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageFEAPICalls.java new file mode 100644 index 000000000000..98c4198f3052 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfContentAssetImageFEAPICalls.java @@ -0,0 +1,58 @@ +package com.dotcms.telemetry.collectors.image; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.api.ApiMetricType; +import com.dotcms.rest.api.v1.HTTPMethod; +import com.dotmarketing.util.PageMode; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Represents a Metric to count how many times the Endpoint '/contentAsset/image/' is called from + * Front End. + */ +public class CountOfContentAssetImageFEAPICalls extends ApiMetricType { + + @Override + public String getName() { + return "COUNT_OF_FE_CONTENTASSET_CALLS"; + } + + @Override + public String getDescription() { + return "Count of 'contentAsset/image' API calls from Front End"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.IMAGE_API; + } + + + @Override + public String getAPIUrl() { + return "/contentAsset/image/"; + } + + @Override + public HTTPMethod getHttpMethod() { + return HTTPMethod.GET; + } + + @Override + @JsonIgnore + public boolean shouldCount(final HttpServletRequest request, + final HttpServletResponse response) { + final PageMode currentPageMode = PageMode.get(request); + return currentPageMode == PageMode.LIVE; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageBEAPICalls.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageBEAPICalls.java new file mode 100644 index 000000000000..6cf01edb06bd --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageBEAPICalls.java @@ -0,0 +1,57 @@ +package com.dotcms.telemetry.collectors.image; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.api.ApiMetricType; +import com.dotcms.rest.api.v1.HTTPMethod; +import com.dotmarketing.util.PageMode; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Represents a Metric to count how many times the Endpoint '/dA' is called From Back End. + */ +public class CountOfDAImageBEAPICalls extends ApiMetricType { + + @Override + public String getName() { + return "COUNT_OF_BE_DA_CALLS"; + } + + @Override + public String getDescription() { + return "Count of '/dA/' API calls From Back End"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.IMAGE_API; + } + + + @Override + public String getAPIUrl() { + return "/dA/"; + } + + @Override + public HTTPMethod getHttpMethod() { + return HTTPMethod.GET; + } + + @Override + @JsonIgnore + public boolean shouldCount(final HttpServletRequest request, + final HttpServletResponse response) { + final PageMode currentPageMode = PageMode.get(request); + return currentPageMode != PageMode.LIVE; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageFEAPICalls.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageFEAPICalls.java new file mode 100644 index 000000000000..49e8ca78b077 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/image/CountOfDAImageFEAPICalls.java @@ -0,0 +1,57 @@ +package com.dotcms.telemetry.collectors.image; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.api.ApiMetricType; +import com.dotcms.rest.api.v1.HTTPMethod; +import com.dotmarketing.util.PageMode; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Represents a Metric to count how many times the Endpoint '/dA' is called from Front End. + */ +public class CountOfDAImageFEAPICalls extends ApiMetricType { + + @Override + public String getName() { + return "COUNT_OF_FE_DA_CALLS"; + } + + @Override + public String getDescription() { + return "Count of '/dA/' API calls From Front End"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.IMAGE_API; + } + + + @Override + public String getAPIUrl() { + return "/dA/"; + } + + @Override + public HTTPMethod getHttpMethod() { + return HTTPMethod.GET; + } + + @Override + @JsonIgnore + public boolean shouldCount(final HttpServletRequest request, + final HttpServletResponse response) { + final PageMode currentPageMode = PageMode.get(request); + return currentPageMode == PageMode.LIVE; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/HasChangeDefaultLanguagesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/HasChangeDefaultLanguagesDatabaseMetricType.java new file mode 100644 index 000000000000..211c322269c6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/HasChangeDefaultLanguagesDatabaseMetricType.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Checks if the default language was changed from English + */ +public class HasChangeDefaultLanguagesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "IS_DEFAULT_LANGUAGE_NOT_ENGLISH"; + } + + @Override + public String getDescription() { + return "Has default language been changed from English?"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public String getSqlQuery() { + return "SELECT language_code <> 'en' AS value FROM language " + + "WHERE id IN (SELECT default_language_id FROM company)"; + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/OldStyleLanguagesVarialeMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/OldStyleLanguagesVarialeMetricType.java new file mode 100644 index 000000000000..dfda412a742b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/OldStyleLanguagesVarialeMetricType.java @@ -0,0 +1,56 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.MetricType; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.languagesmanager.business.LanguageAPI; +import com.dotmarketing.portlets.languagesmanager.model.Language; +import com.dotmarketing.portlets.languagesmanager.model.LanguageKey; + +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Collects the count of old style Language Variables + */ + +public class OldStyleLanguagesVarialeMetricType implements MetricType { + + @Override + public String getName() { + return "OLD_STYLE_LANGUAGES_VARIABLE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of old-style Language variables"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public Optional getValue() { + + final LanguageAPI languageAPI = APILocator.getLanguageAPI(); + + final Set languagesCodes = languageAPI.getLanguages().stream() + .map(Language::getLanguageCode) + .collect(Collectors.toSet()); + + final Set oldStyleLanguageKey = languagesCodes.stream().flatMap(languagesCode -> languageAPI.getLanguageKeys(languagesCode).stream()) + .map(LanguageKey::getKey) + .collect(Collectors.toSet()); + + return Optional.of(oldStyleLanguageKey.size()); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLanguagesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLanguagesDatabaseMetricType.java new file mode 100644 index 000000000000..b1080c0396ad --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLanguagesDatabaseMetricType.java @@ -0,0 +1,35 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of languages + */ +public class TotalLanguagesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT"; + } + + @Override + public String getDescription() { + return "Count of configured dotCMS Languages"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(*) AS value FROM language"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLiveLanguagesVariablesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLiveLanguagesVariablesDatabaseMetricType.java new file mode 100644 index 000000000000..67aa40715128 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalLiveLanguagesVariablesDatabaseMetricType.java @@ -0,0 +1,39 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.contenttype.model.type.BaseContentType; +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of Language Variables that have a live version. + */ +public class TotalLiveLanguagesVariablesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "LIVE_LANGUAGE_VARIABLE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Live Language Variables"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct contentlet.identifier) AS value FROM contentlet " + + "INNER JOIN structure ON contentlet.structure_inode=structure.inode " + + "INNER JOIN contentlet_version_info ON contentlet.identifier = contentlet_version_info.identifier " + + "WHERE structuretype = " + BaseContentType.KEY_VALUE.getType() + " AND deleted = false AND live_inode IS NOT null"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalUniqueLanguagesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalUniqueLanguagesDatabaseMetricType.java new file mode 100644 index 000000000000..158b677298e1 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalUniqueLanguagesDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of unique configured dotCMS Languages, it means that if two o more Countries use the same + * language then it just count as one + */ +public class TotalUniqueLanguagesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "UNIQUE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of unique configured dotCMS Languages, it means that if two o more Countries use the same " + + "language then it just count as one"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT language_code) AS value FROM language"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalWorkingLanguagesVariablesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalWorkingLanguagesVariablesDatabaseMetricType.java new file mode 100644 index 000000000000..f62a60c3183a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/language/TotalWorkingLanguagesVariablesDatabaseMetricType.java @@ -0,0 +1,44 @@ +package com.dotcms.telemetry.collectors.language; + +import com.dotcms.contenttype.model.type.BaseContentType; +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Language Variable that does not have Live Version. + */ +public class TotalWorkingLanguagesVariablesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "WORKING_LANGUAGE_VARIABLE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Working Language Variables"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LANGUAGES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct c.identifier) AS value " + + "FROM contentlet AS c " + + "INNER JOIN structure ON c.structure_inode=structure.inode " + + "INNER JOIN contentlet_version_info ON c.identifier = contentlet_version_info.identifier " + + "WHERE structuretype = " + BaseContentType.KEY_VALUE.getType() + " AND deleted = false AND live_inode is null " + + "AND working_inode IS NOT null AND (SELECT COUNT(DISTINCT identifier) " + + "FROM contentlet_version_info AS cvi2 " + + "WHERE live_inode IS NOT null " + + "AND c.identifier = cvi2.identifier) = 0"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfLiveSitesWithSiteVariablesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfLiveSitesWithSiteVariablesMetricType.java new file mode 100644 index 000000000000..9a1aaeaeaa71 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfLiveSitesWithSiteVariablesMetricType.java @@ -0,0 +1,16 @@ +package com.dotcms.telemetry.collectors.site; + +/** + * Collects the count of started (live) Sites that have Site Variables. + * + * @author Jose Castro + * @since Oct 4th, 2024 + */ +public class CountOfLiveSitesWithSiteVariablesMetricType extends CountOfSitesWithSiteVariablesMetricType { + + @Override + public PublishStatus getPublishStatus() { + return PublishStatus.LIVE; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithIndividualPermissionsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithIndividualPermissionsMetricType.java new file mode 100644 index 000000000000..f5634e6390d2 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithIndividualPermissionsMetricType.java @@ -0,0 +1,43 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of sites with permissions not inheriting from System Host + */ +public class CountOfSitesWithIndividualPermissionsMetricType implements DBMetricType { + + + @Override + public String getName() { + return "SITES_WITH_INDIVIDUAL_PERMISSIONS"; + } + + @Override + public String getDescription() { + return "Count of sites with permissions not inheriting from System Host"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT c.identifier) AS value\n" + + "FROM contentlet c JOIN structure s\n" + + " ON c.structure_inode = s.inode\n" + + " JOIN contentlet_version_info cvi\n" + + " ON (c.inode = cvi.working_inode OR c.inode = cvi.live_inode)\n" + + "WHERE s.name = 'Host' AND c.identifier <> 'SYSTEM_HOST'\n" + + "AND (SELECT COUNT(*) AS cc FROM permission where inode_id=c.identifier) > 0;"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithSiteVariablesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithSiteVariablesMetricType.java new file mode 100644 index 000000000000..f30fd752c0d3 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithSiteVariablesMetricType.java @@ -0,0 +1,74 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of Sites that have Site Variables. + * + * @author Jose Castro + * @since Oct 4th, 2024 + */ +public abstract class CountOfSitesWithSiteVariablesMetricType implements DBMetricType { + + @Override + public String getName() { + return "WITH_" + getPublishStatus().name() + "_SITE_VARIABLES"; + } + + @Override + public String getDescription() { + return "Count of " + getPublishStatus().getName() + " Sites that have Site Variables"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(identifier) AS value FROM contentlet_version_info WHERE deleted IS FALSE AND " + + getPublishStatus().getCondition() + " AND identifier IN (SELECT DISTINCT " + + "host_id FROM host_variable)"; + } + + /** + * Allows you to specify whether you want to collect Metrics from a started or stopped Site. + * + * @return The {@link PublishStatus} used to generate the metrics. + */ + public abstract PublishStatus getPublishStatus(); + + /** + * Determines the publish status of a given Site: Started (live) or Stopped (working). + */ + public enum PublishStatus { + WORKING("Working", "(working_inode <> live_inode OR live_inode IS NULL)"), + LIVE("Live", "working_inode = live_inode"); + + private final String name; + private final String condition; + + PublishStatus(final String value, final String condition) { + this.name = value; + this.condition = condition; + } + + public String getName() { + return name; + } + + public String getCondition() { + return condition; + } + + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithThumbnailsMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithThumbnailsMetricType.java new file mode 100644 index 000000000000..fb2cba1a0c3c --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfSitesWithThumbnailsMetricType.java @@ -0,0 +1,84 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.MetricType; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; +import io.vavr.control.Try; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Collects the count of sites with thumbnails set + */ +public class CountOfSitesWithThumbnailsMetricType implements MetricType { + + private static final String ALL_SITES_INODES = "SELECT c.inode AS inode \n" + + "FROM contentlet c JOIN structure s\n" + + "ON c.structure_inode = s.inode\n" + + "JOIN contentlet_version_info cvi\n" + + "ON (c.inode = cvi.working_inode)\n" + + "WHERE s.name = 'Host' AND c.identifier <> 'SYSTEM_HOST';\n"; + + + @Override + public String getName() { + return "SITES_WITH_THUMBNAIL_COUNT"; + } + + @Override + public String getDescription() { + return "Count of sites with Thumbnails"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public Optional getValue() { + return Optional.of(getCountOfSitesWithThumbnails()); + } + + private int getCountOfSitesWithThumbnails() { + int hostsWithThumbnailsCount = 0; + + try { + final List allSitesInodes = getAllSitesInodes(); + + for (String siteInode : allSitesInodes) { + final File hostThumbnail = Try.of(() -> APILocator.getContentletAPI().getBinaryFile(siteInode, + Host.HOST_THUMB_KEY, APILocator.systemUser())).getOrNull(); + if (hostThumbnail != null) { + hostsWithThumbnailsCount++; + } + } + + } catch(Exception e) { + Logger.debug(this, "Error counting Sites with thumbnails"); + } + + return hostsWithThumbnailsCount; + } + + private List getAllSitesInodes() throws DotDataException { + final DotConnect db = new DotConnect(); + final List> results = db.setSQL(ALL_SITES_INODES).loadObjectResults(); + return results.stream().map(map -> (String) map.get("inode")).collect(Collectors.toList()); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfWorkingSitesWithSiteVariablesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfWorkingSitesWithSiteVariablesMetricType.java new file mode 100644 index 000000000000..16a6898719a6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/CountOfWorkingSitesWithSiteVariablesMetricType.java @@ -0,0 +1,16 @@ +package com.dotcms.telemetry.collectors.site; + +/** + * Collects the count of stopped (working) Sites that have Site Variables. + * + * @author Jose Castro + * @since Oct 4th, 2024 + */ +public class CountOfWorkingSitesWithSiteVariablesMetricType extends CountOfSitesWithSiteVariablesMetricType { + + @Override + public PublishStatus getPublishStatus() { + return PublishStatus.WORKING; + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoDefaultTagStorageDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoDefaultTagStorageDatabaseMetricType.java new file mode 100644 index 000000000000..bef3d73390d7 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoDefaultTagStorageDatabaseMetricType.java @@ -0,0 +1,49 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of sites With non-default value on the tagStorage attribute + */ +public class SitesWithNoDefaultTagStorageDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "SITES_NON_DEFAULT_TAG_STORAGE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Sites With Not Default value on the tagStorage attribute"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct c.identifier) AS value\n" + + "FROM contentlet c\n" + + " JOIN structure s ON c.structure_inode = s.inode\n" + + " JOIN contentlet_version_info cvi on (c.inode = cvi.working_inode OR c.inode = cvi.live_inode)\n" + + " WHERE s.name = 'Host'\n" + + " AND c.identifier <> 'SYSTEM_HOST'\n" + + " AND c.contentlet_as_json -> 'fields' -> 'tagStorage' ->> 'value' <>\n" + + " -- DEFAULT_HOST identifier\n" + + " (SELECT distinct c.identifier\n" + + " FROM contentlet c\n" + + " JOIN structure s ON c.structure_inode = s.inode\n" + + " JOIN contentlet_version_info cvi\n" + + " on (c.inode = cvi.working_inode OR c.inode = cvi.live_inode)\n" + + " WHERE s.name = 'Host'\n" + + " AND c.contentlet_as_json -> 'fields' -> 'isDefault' ->> 'value' = 'true')"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoSystemFieldsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoSystemFieldsDatabaseMetricType.java new file mode 100644 index 000000000000..0558308ed33e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithNoSystemFieldsDatabaseMetricType.java @@ -0,0 +1,40 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of non-system fields in Host content type + */ +public class SitesWithNoSystemFieldsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "NON_SYSTEM_FIELDS_ON_CONTENT_TYPE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of non-system fields in Host content type"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT count(f.inode) AS value\n" + + "FROM field f, structure s WHERE f.structure_inode = s.inode\n" + + "AND s.name = 'Host'\n" + + "AND f.fixed = false\n" + + "AND field_type <> 'com.dotcms.contenttype.model.field.RowField'\n" + + "AND field_type <> 'com.dotcms.contenttype.model.field.ColumnField'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithRunDashboardDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithRunDashboardDatabaseMetricType.java new file mode 100644 index 000000000000..0906c021cdb1 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/SitesWithRunDashboardDatabaseMetricType.java @@ -0,0 +1,40 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of sites with 'Run Dashboard' enabled + */ +public class SitesWithRunDashboardDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "SITES_RUN_DASHBOARD_TRUE_COUNT"; + } + + @Override + public String getDescription() { + return "Count of sites with 'Run Dashboard' enabled"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + public String getSqlQuery() { + return "SELECT COUNT(c.identifier)\n" + + " AS value FROM contentlet c\n" + + " JOIN structure s ON c.structure_inode = s.inode\n" + + " JOIN contentlet_version_info cvi on (c.inode = cvi.working_inode OR c.inode = cvi.live_inode)\n" + + " WHERE s.name = 'Host'\n" + + " AND c.identifier <> 'SYSTEM_HOST'\n" + + " AND c.contentlet_as_json -> 'fields' -> 'runDashboard' ->> 'value' = 'true'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalActiveSitesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalActiveSitesDatabaseMetricType.java new file mode 100644 index 000000000000..fcfc902dde56 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalActiveSitesDatabaseMetricType.java @@ -0,0 +1,40 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of active sites + */ +public class TotalActiveSitesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_ACTIVE_SITES"; + } + + @Override + public String getDescription() { + return "Total count of active sites"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(id) as value\n" + + "FROM identifier i\n" + + "JOIN contentlet_version_info cvi ON i.id = cvi.identifier\n" + + "WHERE asset_subtype = 'Host'\n" + + " AND id <> 'SYSTEM_HOST'\n" + + "AND cvi.live_inode is not null"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesActiveSitesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesActiveSitesDatabaseMetricType.java new file mode 100644 index 000000000000..840eb0cc404e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesActiveSitesDatabaseMetricType.java @@ -0,0 +1,43 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of aliases on all active sites + */ +public class TotalAliasesActiveSitesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "ALIASES_ACTIVE_SITES_COUNT"; + } + + @Override + public String getDescription() { + return "Count of aliases on all active sites"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT coalesce(SUM(array_length(regexp_split_to_array(trim(result.aliases), '[\\n\\r\\t]'), 1)),0) AS value\n" + + "FROM (SELECT c.contentlet_as_json -> 'fields' -> 'aliases' ->> 'value' AS aliases\n" + + " FROM contentlet c\n" + + " JOIN structure s on c.structure_inode = s.inode JOIN contentlet_version_info\n" + + " cvi on c.inode = cvi.live_inode\n" + + " WHERE s.name = 'Host' AND cvi.live_inode is not null\n" + + " AND c.contentlet_as_json -> 'fields' -> 'hostName' ->> 'value' <> 'System Host')\n" + + " AS result\n" + + "WHERE result.aliases <> ''\n"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesAllSitesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesAllSitesDatabaseMetricType.java new file mode 100644 index 000000000000..66d4cafaf321 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalAliasesAllSitesDatabaseMetricType.java @@ -0,0 +1,43 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of aliases on all sites + */ +public class TotalAliasesAllSitesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "ALIASES_SITES_COUNT"; + } + + @Override + public String getDescription() { + return "Count of aliases on all sites"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT coalesce(SUM(array_length(regexp_split_to_array(trim(result.aliases), '[\\n\\r\\t]'), 1)),0) AS value\n" + + "FROM (SELECT c.contentlet_as_json -> 'fields' -> 'aliases' ->> 'value' AS aliases\n" + + " FROM contentlet c\n" + + " JOIN structure s on c.structure_inode = s.inode JOIN contentlet_version_info\n" + + " cvi on (c.inode = cvi.working_inode OR c.inode = cvi.live_inode)\n" + + " WHERE s.name = 'Host' \n" + + " AND c.contentlet_as_json -> 'fields' -> 'hostName' ->> 'value' <> 'System Host')\n" + + " AS result\n" + + "WHERE result.aliases <> ''\n"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalSitesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalSitesDatabaseMetricType.java new file mode 100644 index 000000000000..8c25dd657616 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/site/TotalSitesDatabaseMetricType.java @@ -0,0 +1,35 @@ +package com.dotcms.telemetry.collectors.site; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of sites + */ +public class TotalSitesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_SITES"; + } + + @Override + public String getDescription() { + return "Total count of sites"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.SITES; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(id) as value FROM identifier WHERE asset_subtype='Host' AND id <> 'SYSTEM_HOST'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchDocumentMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchDocumentMetricType.java new file mode 100644 index 000000000000..7f3d3cabc807 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchDocumentMetricType.java @@ -0,0 +1,32 @@ +package com.dotcms.telemetry.collectors.sitesearch; + +import com.dotcms.content.elasticsearch.business.IndexStats; +import com.dotmarketing.exception.DotDataException; + +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Collects the number of documents in all site search indexes. + */ +public class CountSiteSearchDocumentMetricType extends IndicesSiteSearchMetricType { + + + @Override + public String getName() { + return "INDICES_DOCUMENT_COUNT"; + } + + @Override + public String getDescription() { + return "Number of documents in indexes"; + } + + @Override + public Optional getValue(Collection indices) throws DotDataException { + return Optional.of(indices.stream() + .collect(Collectors.summingLong(IndexStats::getDocumentCount)) + ); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchIndicesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchIndicesMetricType.java new file mode 100644 index 000000000000..83a392a1ce72 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/CountSiteSearchIndicesMetricType.java @@ -0,0 +1,29 @@ +package com.dotcms.telemetry.collectors.sitesearch; + +import com.dotcms.content.elasticsearch.business.IndexStats; +import com.dotmarketing.exception.DotDataException; + +import java.util.Collection; +import java.util.Optional; + +/** + * Collect the count of Site Search indices + */ +public class CountSiteSearchIndicesMetricType extends IndicesSiteSearchMetricType { + + + @Override + public String getName() { + return "INDICES_COUNT"; + } + + @Override + public String getDescription() { + return "Count of indexes"; + } + + @Override + public Optional getValue(Collection indices) throws DotDataException { + return Optional.of(indices.size()); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/IndicesSiteSearchMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/IndicesSiteSearchMetricType.java new file mode 100644 index 000000000000..af9dcc43d626 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/IndicesSiteSearchMetricType.java @@ -0,0 +1,68 @@ +package com.dotcms.telemetry.collectors.sitesearch; + +import com.dotcms.content.elasticsearch.business.IndexStats; +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.MetricValue; +import com.dotcms.telemetry.util.MetricCaches; +import com.dotmarketing.exception.DotDataException; + +import javax.ws.rs.NotSupportedException; +import java.util.Collection; +import java.util.Optional; + +/** + * This class represents any Metric that requires Site Search Index information for its calculation. + * Any Metric that relies on the Site Search index must extend from this class. + * + *

Overridden Methods:

+ * + * - getStat: This method is called when collecting the Metric values. It relies on a getValue method, + * which is overridden by each subclass of MetricType. In this class, getStat uses its own version of getValue, + * necessitating the override. + * + * - getValue: This method is overridden to throw a NotSupportedException. As mentioned earlier, + * this class does not use this method to calculate values; instead, it uses a different getValue method. + * + * - getValue(final Collection indices): This is the actual getValue method used by the getStat method. + * The getStat method first retrieves the indices information and then calls this getValue method to calculate the Metric's value. + * + *

In Summary:

+ * The getStat method works as follows: + * - Retrieves Site Search Indices Information: It uses the Elasticsearch Client Utility class to send a request to the + * Elasticsearch server and retrieve the indices information. + * - Caches the Site Search Indices Information: This information is stored in a cache. If multiple classes need this + * information, the request to the Elasticsearch server is made only once. Each MetricType then uses the cached information. + * The cache is cleared when all Metrics are calculated. + * - Calls the getValue(final Collection indices) Method: This method calculates the Metric value. + * It must be overridden by each concrete class that extends from this class. + * + * @see MetricType + */ +public abstract class IndicesSiteSearchMetricType implements MetricType { + + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + public MetricFeature getFeature() { + return MetricFeature.SITE_SEARCH; + } + + @Override + public Optional getStat() throws DotDataException { + return getValue(MetricCaches.SITE_SEARCH_INDICES.get()) + .map(o -> new MetricValue(this.getMetric(), o)) + .or(() -> Optional.of(new MetricValue(this.getMetric(), 0))); + + } + + @Override + public Optional getValue() { + throw new NotSupportedException(); + } + + abstract Optional getValue(final Collection indices) throws DotDataException; + + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/TotalSizeSiteSearchIndicesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/TotalSizeSiteSearchIndicesMetricType.java new file mode 100644 index 000000000000..af93df6bfd02 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/sitesearch/TotalSizeSiteSearchIndicesMetricType.java @@ -0,0 +1,33 @@ +package com.dotcms.telemetry.collectors.sitesearch; + +import com.dotcms.content.elasticsearch.business.IndexStats; +import com.dotmarketing.exception.DotDataException; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Collect the total size in Mb of all the Site Search indices. + */ +public class TotalSizeSiteSearchIndicesMetricType extends IndicesSiteSearchMetricType { + + + @Override + public String getName() { + return "TOTAL_INDICES_SIZE"; + } + + @Override + public String getDescription() { + return "Total size of indexes"; + } + + @Override + public Optional getValue(Collection indices) throws DotDataException { + return Optional.of(new ByteSizeValue( + indices.stream().collect(Collectors.summingLong(IndexStats::getSizeRaw))).toString() + ); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalAdvancedTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalAdvancedTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..b7dc05dbf2fc --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalAdvancedTemplatesDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.template; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of advanced templates + */ +public class TotalAdvancedTemplatesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_ADVANCED_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Total count of advanced templates"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT template.identifier) as value " + + "FROM template INNER JOIN template_version_info on template.identifier = template_version_info.identifier " + + "WHERE drawed = false and deleted = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalBuilderTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalBuilderTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..ae0bd90a6130 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalBuilderTemplatesDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.template; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of builder templates, it means excluding File, Advanced, and Layout templates + */ +public class TotalBuilderTemplatesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_TEMPLATE_BUILDER_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Total count of Builder templates, excluding File, Advanced, and Layout templates"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT template.identifier) as value " + + "FROM template INNER JOIN template_version_info on template.identifier = template_version_info.identifier " + + "WHERE drawed = true AND deleted = false AND title NOT LIKE 'anonymous_layout_%'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesDatabaseMetricType.java new file mode 100644 index 000000000000..d98fa0d343ef --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesDatabaseMetricType.java @@ -0,0 +1,47 @@ +package com.dotcms.telemetry.collectors.template; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of templates + */ +public class TotalTemplatesDatabaseMetricType implements DBMetricType { + + @Override + public String getName() { + return "COUNT_OF_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Total count of templates"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT template.count + file_template.count as value " + + "FROM (SELECT COUNT(DISTINCT template.identifier)" + + "FROM template_version_info " + + "INNER JOIN template ON template_version_info.identifier = template.identifier " + + "WHERE title NOT LIKE 'anonymous_layout_%' and deleted = false) template, " + + "(SELECT COUNT(DISTINCT cvi.identifier) " + + "FROM contentlet_version_info cvi, identifier id " + + "WHERE id.parent_path like '/application/templates%' and " + + "id.asset_name = 'properties.vtl' and " + + "cvi.identifier = id.id and " + + "cvi.deleted = false) as file_template"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInLivePagesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInLivePagesDatabaseMetricType.java new file mode 100644 index 000000000000..5e1712e76a36 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInLivePagesDatabaseMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.template; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total all templates used in LIVE pages + */ +public class TotalTemplatesInLivePagesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_TEMPLATES_USED_IN_LIVE_PAGES"; + } + + @Override + public String getDescription() { + return "Count of all templates used in LIVE pages"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT count(distinct contentlet.contentlet_as_json->'fields'->'template'->'value') as value " + + "FROM contentlet INNER JOIN contentlet_version_info ON contentlet_version_info.identifier = contentlet.identifier " + + "WHERE contentlet_version_info.live_inode is not null AND " + + "deleted = false AND structure_inode in (SELECT inode FROM structure WHERE name = 'Page')"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInWorkingPagesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInWorkingPagesDatabaseMetricType.java new file mode 100644 index 000000000000..968a967a1944 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/template/TotalTemplatesInWorkingPagesDatabaseMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.template; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total all templates used in Working pages + */ +public class TotalTemplatesInWorkingPagesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_OF_TEMPLATES_USED_IN_WORKING_PAGES"; + } + + @Override + public String getDescription() { + return "Count of all templates used in WORKING pages"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT count(distinct contentlet.contentlet_as_json->'fields'->'template'->'value') as value " + + "FROM contentlet INNER JOIN contentlet_version_info ON contentlet_version_info.identifier = contentlet.identifier " + + "WHERE contentlet_version_info.live_inode is null AND " + + "deleted = false AND structure_inode in (SELECT inode FROM structure WHERE name = 'Page')"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalFilesInThemeMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalFilesInThemeMetricType.java new file mode 100644 index 000000000000..381958e7fff5 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalFilesInThemeMetricType.java @@ -0,0 +1,41 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; +import com.dotmarketing.util.Logger; + +/** + * Collects the total of Number of LIVE/WORKING files in themes + */ +public class TotalFilesInThemeMetricType implements DBMetricType { + @Override + public String getName() { + return "TOTAL_FILES_IN_THEMES"; + } + + @Override + public String getDescription() { + return "Count of Number of WORKING and LIVE files in themes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct CONCAT(id.parent_path, asset_name)) as value " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE deleted = false AND " + + "id.parent_path LIKE ANY (SELECT CONCAT(id.parent_path, '%') " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path LIKE '/application/themes/%' AND id.asset_name = 'template.vtl')"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveContainerDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveContainerDatabaseMetricType.java new file mode 100644 index 000000000000..6db9ab7248fb --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveContainerDatabaseMetricType.java @@ -0,0 +1,42 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of Live containers + */ +public class TotalLiveContainerDatabaseMetricType implements DBMetricType { + + @Override + public String getName() { + return "COUNT_OF_LIVE_CONTAINERS"; + } + + @Override + public String getDescription() { + return "Total count of LIVE containers"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT container.count + file_container.count as value " + + "FROM (SELECT COUNT(DISTINCT cvi.identifier) FROM container_version_info cvi WHERE deleted = false AND live_inode is not null) container, " + + "(SELECT COUNT(DISTINCT cvi.identifier) " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path like '/application/containers%' and " + + "id.asset_name = 'container.vtl' and cvi.deleted = false AND live_inode is not null) as file_container"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveFilesInThemeMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveFilesInThemeMetricType.java new file mode 100644 index 000000000000..d553dcd8605a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalLiveFilesInThemeMetricType.java @@ -0,0 +1,41 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total of Number of LIVE files in themes + */ +public class TotalLiveFilesInThemeMetricType implements DBMetricType { + @Override + public String getName() { + return "TOTAL_LIVE_FILES_IN_THEMES"; + } + + @Override + public String getDescription() { + return "Count of Number of LIVE files in themes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct CONCAT(id.parent_path, asset_name)) as value " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE cvi.live_inode IS NOT NULL AND deleted = false AND " + + "id.parent_path LIKE ANY (SELECT CONCAT(id.parent_path, '%') " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path LIKE '/application/themes/%' AND id.asset_name = 'template.vtl')"; + + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeMetricType.java new file mode 100644 index 000000000000..2fa6ca1885f3 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total of themes + */ +public class TotalThemeMetricType implements DBMetricType { + @Override + public String getName() { + return "TOTAL_THEMES"; + } + + @Override + public String getDescription() { + return "Count of themes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(id.parent_path) as value " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path LIKE '/application/themes/%' AND id.asset_name = 'template.vtl' " + + "AND deleted = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInLiveTemplatesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInLiveTemplatesMetricType.java new file mode 100644 index 000000000000..273ce64f495a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInLiveTemplatesMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of themes used by LIVE templates + */ +public class TotalThemeUsedInLiveTemplatesMetricType implements DBMetricType { + @Override + public String getName() { + return "TOTAL_USED_THEMES_IN_LIVE_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Count of themes used by templates"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT count(distinct folder.identifier) value FROM folder " + + "WHERE inode in ( " + + "SELECT DISTINCT theme FROM template INNER JOIN template_version_info ON template_version_info.live_inode = template.inode " + + "WHERE title NOT LIKE 'anonymous_layout_%' AND deleted = false)"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInWorkingTemplatesMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInWorkingTemplatesMetricType.java new file mode 100644 index 000000000000..250e19fb0cca --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalThemeUsedInWorkingTemplatesMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of themes used by WORKING templates + */ +public class TotalThemeUsedInWorkingTemplatesMetricType implements DBMetricType { + @Override + public String getName() { + return "TOTAL_USED_THEMES_IN_WORKING_TEMPLATES"; + } + + @Override + public String getDescription() { + return "Count of themes used by templates"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT count(distinct folder.identifier) value FROM folder " + + "WHERE inode in ( " + + "SELECT DISTINCT theme FROM template INNER JOIN template_version_info ON template_version_info.working_inode = template.inode " + + "WHERE title NOT LIKE 'anonymous_layout_%' AND deleted = false and template_version_info.live_inode is null)"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalWorkingContainerDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalWorkingContainerDatabaseMetricType.java new file mode 100644 index 000000000000..c54d5a0cfc28 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/theme/TotalWorkingContainerDatabaseMetricType.java @@ -0,0 +1,42 @@ +package com.dotcms.telemetry.collectors.theme; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the total count of Working containers + */ +public class TotalWorkingContainerDatabaseMetricType implements DBMetricType { + + @Override + public String getName() { + return "COUNT_OF_WORKING_CONTAINERS"; + } + + @Override + public String getDescription() { + return "Total count of WORKING containers"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.LAYOUT; + } + + @Override + public String getSqlQuery() { + return "SELECT container.count + file_container.count as value " + + "FROM (SELECT COUNT(DISTINCT cvi.identifier) FROM container_version_info cvi WHERE deleted = false AND live_inode is null) container, " + + "(SELECT COUNT(DISTINCT cvi.identifier) " + + "FROM contentlet_version_info cvi INNER JOIN identifier id ON cvi.identifier = id.id " + + "WHERE id.parent_path like '/application/containers%' and " + + "id.asset_name = 'container.vtl' and cvi.deleted = false AND live_inode is null) as file_container"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/ContentTypesWithUrlMapDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/ContentTypesWithUrlMapDatabaseMetricType.java new file mode 100644 index 000000000000..e89203819e57 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/ContentTypesWithUrlMapDatabaseMetricType.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.urlmap; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collects the count of content types with a non-null detail page. + */ +public class ContentTypesWithUrlMapDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "CONTENT_TYPES_WITH_URL_MAP"; + } + + @Override + public String getDescription() { + return "Count of content types with URL maps"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.URL_MAPS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(*) AS value " + + "FROM structure WHERE url_map_pattern IS NOT null AND page_detail IS NOT null"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/LiveContentInUrlMapDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/LiveContentInUrlMapDatabaseMetricType.java new file mode 100644 index 000000000000..865889a32503 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/LiveContentInUrlMapDatabaseMetricType.java @@ -0,0 +1,40 @@ +package com.dotcms.telemetry.collectors.urlmap; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of all the Contentlets in content types with URL maps that have LIVE Version + */ +public class LiveContentInUrlMapDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "LIVE_CONTENTLETS_IN_CONTENT_TYPES_WITH_URL_MAP"; + } + + @Override + public String getDescription() { + return "Count of Live Contentlets in content types with URL maps"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.URL_MAPS; + } + + @Override + public String getSqlQuery() { + return "SELECT count(DISTINCT contentlet.identifier) AS value FROM contentlet " + + "INNER JOIN contentlet_version_info ON contentlet.identifier = contentlet_version_info.identifier " + + "INNER JOIN structure ON contentlet.structure_inode = structure.inode " + + "WHERE url_map_pattern IS NOT null AND page_detail IS NOT null AND deleted = false " + + "AND live_inode IS NOT null"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/UrlMapPatterWithTwoVariablesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/UrlMapPatterWithTwoVariablesDatabaseMetricType.java new file mode 100644 index 000000000000..eb64c713050b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/UrlMapPatterWithTwoVariablesDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.urlmap; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Content Types that are using a detail page with 2 or more variables. + */ +public class UrlMapPatterWithTwoVariablesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "COUNT_URL_MAP_PATTER_WITH_MORE_THAT_ONE_VARIABLE"; + } + + @Override + public String getDescription() { + return "Count of URL Map Patterns with more than one variable"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.URL_MAPS; + } + + @Override + public String getSqlQuery() { + return "SELECT count(*) AS value FROM structure " + + "WHERE page_detail IS NOT null AND REGEXP_COUNT(url_map_pattern, '\\{[^}]*\\}') >= 2"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/WorkingContentInUrlMapDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/WorkingContentInUrlMapDatabaseMetricType.java new file mode 100644 index 000000000000..943412b6a374 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/urlmap/WorkingContentInUrlMapDatabaseMetricType.java @@ -0,0 +1,40 @@ +package com.dotcms.telemetry.collectors.urlmap; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of all the Contentlets in content types with URL maps that does not have LIVE Version + */ +public class WorkingContentInUrlMapDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "WORKING_CONTENTLETS_IN_CONTENT_TYPES_WITH_URL_MAP"; + } + + @Override + public String getDescription() { + return "Count of Working Contentlets in content types with URL maps"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.URL_MAPS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(DISTINCT contentlet.identifier) as value FROM contentlet\n" + + "INNER JOIN contentlet_version_info ON contentlet.identifier = contentlet_version_info.identifier\n" + + "INNER JOIN structure ON contentlet.structure_inode = structure.inode\n" + + "WHERE url_map_pattern IS NOT null AND page_detail is not null AND deleted = false " + + "AND live_inode IS null AND working_inode is not null"; + } +} + diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/ActiveUsersDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/ActiveUsersDatabaseMetricType.java new file mode 100644 index 000000000000..88baca8d65f5 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/ActiveUsersDatabaseMetricType.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.user; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; + +/** + * Collect the count of Active User + */ +public class ActiveUsersDatabaseMetricType implements UsersDatabaseMetricType { + + @Override + public String getName() { + return "ACTIVE_USERS_COUNT"; + } + + @Override + public String getDescription() { + return "Count of Active Users"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.USERS; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(userid) as value FROM user_ " + + "WHERE " + USER_EXCLUDE + " AND lastlogindate > now() - interval '1 month'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginDatabaseMetricType.java new file mode 100644 index 000000000000..2b47bcf6e164 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginDatabaseMetricType.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.user; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; + +/** + * Collects the date of the last login. + */ +public class LastLoginDatabaseMetricType implements UsersDatabaseMetricType { + + @Override + public String getName() { + return "LAST_LOGIN"; + } + + @Override + public String getDescription() { + return "Date of the last Login"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.USERS; + } + + @Override + public String getSqlQuery() { + return "SELECT to_char (lastlogindate,'HH12:MI:SS DD Mon YYYY') AS value " + + "FROM user_ where " + USER_EXCLUDE + " ORDER BY lastlogindate DESC LIMIT 1"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginUserDatabaseMetric.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginUserDatabaseMetric.java new file mode 100644 index 000000000000..16ff1c31a01a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/LastLoginUserDatabaseMetric.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.user; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; + +/** + * Email address of the last logged-in user + */ +public class LastLoginUserDatabaseMetric implements UsersDatabaseMetricType { + + @Override + public String getName() { + return "LAST_LOGIN_USER"; + } + + @Override + public String getDescription() { + return "Email address of the Last login User"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.USERS; + } + + @Override + public String getSqlQuery() { + return "SELECT emailaddress AS value " + + "FROM user_ where " + USER_EXCLUDE + " ORDER BY lastlogindate DESC LIMIT 1"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/UsersDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/UsersDatabaseMetricType.java new file mode 100644 index 000000000000..ff06bed49cac --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/user/UsersDatabaseMetricType.java @@ -0,0 +1,15 @@ +package com.dotcms.telemetry.collectors.user; + +import com.dotcms.telemetry.MetricType; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Represents the MetaData of a User Metric that we want to collect from DataBase + * + * @see MetricType + */ +public interface UsersDatabaseMetricType extends DBMetricType { + + String USER_EXCLUDE="companyid<>'default' and userid<> 'system' and userid <> 'anonymous'"; + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ActionsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ActionsDatabaseMetricType.java new file mode 100644 index 000000000000..b63afc301cdb --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ActionsDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Workflow Actions + */ +public class ActionsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "ACTIONS_COUNT"; + } + + @Override + public String getDescription() { + return "Count of workflow actions in all schemes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(*) AS value FROM workflow_action " + + "INNER JOIN workflow_scheme ON workflow_scheme.id=workflow_action.scheme_id " + + "WHERE archived = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ContentTypesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ContentTypesDatabaseMetricType.java new file mode 100644 index 000000000000..58205f056eb0 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/ContentTypesDatabaseMetricType.java @@ -0,0 +1,38 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Content Types that are NOT using 'System Workflow' + */ +public class ContentTypesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "CONTENT_TYPES_ASSIGNED"; + } + + @Override + public String getDescription() { + return "Count content types assigned schemes other than System Workflow"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct structure_id) AS value FROM workflow_scheme_x_structure " + + "INNER JOIN workflow_scheme ON workflow_scheme.id=workflow_scheme_x_structure.scheme_id " + + "WHERE name != 'System Workflow'"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SchemesDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SchemesDatabaseMetricType.java new file mode 100644 index 000000000000..3052ce19ad91 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SchemesDatabaseMetricType.java @@ -0,0 +1,36 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + + +/** + * Collect the count of Workflow Schemes + */ +public class SchemesDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "SCHEMES_COUNT"; + } + + @Override + public String getDescription() { + return "Count of workflow schemes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + @Override + public String getSqlQuery() { + return "SELECT count(*) as value FROM workflow_scheme WHERE archived=false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/StepsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/StepsDatabaseMetricType.java new file mode 100644 index 000000000000..1497a3003083 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/StepsDatabaseMetricType.java @@ -0,0 +1,37 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + +/** + * Collect the count of Workflow Steps + */ +public class StepsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "STEPS_COUNT"; + } + + @Override + public String getDescription() { + return "Count of steps in all schemes"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(*) AS value FROM workflow_step " + + "INNER JOIN workflow_scheme ON workflow_scheme.id=workflow_step.scheme_id " + + "WHERE archived = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SubActionsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SubActionsDatabaseMetricType.java new file mode 100644 index 000000000000..d28a25720d9a --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/SubActionsDatabaseMetricType.java @@ -0,0 +1,41 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + + +/** + * Collect the count of Workflow SubActions, no matter if the same Sub Action is use for more than one Action + * in this case it count several times. + */ +public class SubActionsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "SUBACTIONS_COUNT"; + } + + @Override + public String getDescription() { + return "Count of workflow subactions in all Workflow actions"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(*) AS value " + + "FROM workflow_action_class " + + "INNER JOIN workflow_action ON workflow_action.id=workflow_action_class.action_id " + + "INNER JOIN workflow_scheme ON workflow_scheme.id=workflow_action.scheme_id " + + "WHERE archived = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/UniqueSubActionsDatabaseMetricType.java b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/UniqueSubActionsDatabaseMetricType.java new file mode 100644 index 000000000000..8d36a1dcf15b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/collectors/workflow/UniqueSubActionsDatabaseMetricType.java @@ -0,0 +1,41 @@ +package com.dotcms.telemetry.collectors.workflow; + +import com.dotcms.telemetry.MetricCategory; +import com.dotcms.telemetry.MetricFeature; +import com.dotcms.telemetry.collectors.DBMetricType; + + +/** + * Collect the count of Unique Workflow SubActions, it means if the same Sub Action is use by several + * Workflow Action then it count as 1 + */ +public class UniqueSubActionsDatabaseMetricType implements DBMetricType { + @Override + public String getName() { + return "UNIQUE_SUBACTIONS_COUNT"; + } + + @Override + public String getDescription() { + return "Count of unique workflow subactions in all Workflow actions"; + } + + @Override + public MetricCategory getCategory() { + return MetricCategory.DIFFERENTIATING_FEATURES; + } + + @Override + public MetricFeature getFeature() { + return MetricFeature.WORKFLOW; + } + + @Override + public String getSqlQuery() { + return "SELECT COUNT(distinct workflow_action_class.name) AS value " + + "FROM workflow_action_class " + + "INNER JOIN workflow_action ON workflow_action.id=workflow_action_class.action_id " + + "INNER JOIN workflow_scheme ON workflow_scheme.id=workflow_action.scheme_id " + + "WHERE archived = false"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/job/MetricsStatsJob.java b/dotCMS/src/main/java/com/dotcms/telemetry/job/MetricsStatsJob.java new file mode 100644 index 000000000000..ee726513c68c --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/job/MetricsStatsJob.java @@ -0,0 +1,48 @@ +package com.dotcms.telemetry.job; + +import com.dotcms.concurrent.lock.ClusterLockManager; +import com.dotcms.exception.ExceptionUtil; +import com.dotcms.telemetry.MetricsSnapshot; +import com.dotcms.telemetry.business.MetricsAPI; +import com.dotcms.telemetry.collectors.MetricStatsCollector; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import io.vavr.Lazy; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.StatefulJob; + +/** + * Stateful job used to collect and persist a snapshot of the Metrics stats. Frequency set to once a + * day. + */ +public class MetricsStatsJob implements StatefulJob { + + public static final String JOB_NAME = "MetricsStatsJob"; + public static final String JOB_GROUP = "MetricsStatsJobGroup"; + public static final String ENABLED_PROP = "TELEMETRY_SAVE_SCHEDULE_JOB_ENABLED"; + public static final String CRON_EXPR_PROP = "TELEMETRY_SAVE_SCHEDULE"; + private static final String CRON_EXPRESSION_DEFAULT = "0 0 22 * * ?"; + + public static final Lazy ENABLED = + Lazy.of(() -> Config.getBooleanProperty(ENABLED_PROP, true)); + public static final Lazy CRON_EXPRESSION = + Lazy.of(() -> Config.getStringProperty(CRON_EXPR_PROP, CRON_EXPRESSION_DEFAULT)); + + @Override + public void execute(final JobExecutionContext jobExecutionContext) throws JobExecutionException { + final MetricsSnapshot metricsSnapshot; + try { + metricsSnapshot = MetricStatsCollector.getStatsAndCleanUp(); + MetricsAPI.INSTANCE.persistMetricsSnapshot(metricsSnapshot); + } catch (final Throwable e) { + Logger.debug(this, String.format("Error occurred during job execution: %s", + ExceptionUtil.getErrorMessage(e)), e); + } + } + + public void run(ClusterLockManager lockManager) { + + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/osgi/Activator.java b/dotCMS/src/main/java/com/dotcms/telemetry/osgi/Activator.java new file mode 100644 index 000000000000..397501ce346e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/osgi/Activator.java @@ -0,0 +1,111 @@ +package com.dotcms.telemetry.osgi; + +import com.dotcms.concurrent.DotConcurrentFactory; +import com.dotcms.concurrent.lock.ClusterLockManager; +import com.dotcms.telemetry.rest.TelemetryResource; +import com.dotcms.telemetry.collectors.api.ApiMetricAPI; +import com.dotcms.telemetry.collectors.api.ApiMetricFactorySubmitter; +import com.dotcms.telemetry.collectors.api.ApiMetricWebInterceptor; +import com.dotcms.telemetry.job.MetricsStatsJob; +import com.dotcms.filters.interceptor.FilterWebInterceptorProvider; +import com.dotcms.filters.interceptor.WebInterceptor; +import com.dotcms.filters.interceptor.WebInterceptorDelegate; +import com.dotcms.rest.config.RestServiceUtil; +import com.dotmarketing.filters.InterceptorFilter; +import com.dotmarketing.osgi.GenericBundleActivator; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import io.vavr.Lazy; +import org.apache.logging.log4j.core.util.CronExpression; +import org.osgi.framework.BundleContext; + +import java.text.ParseException; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class Activator extends GenericBundleActivator { + public static String version; + + private final WebInterceptorDelegate delegate = FilterWebInterceptorProvider + .getInstance(Config.CONTEXT) + .getDelegate(InterceptorFilter.class); + + private final WebInterceptor apiCallWebInterceptor = new ApiMetricWebInterceptor(); + private final MetricsStatsJob metricsStatsJob = new MetricsStatsJob(); + + private static final String METRICS_JOB_LOCK_KEY = "metrics_job_lock"; + private ScheduledFuture scheduledFuture; + private final Lazy enableTelemetry = Lazy.of(() -> + Config.getBooleanProperty("FEATURE_FLAG_TELEMETRY", false)); + + private final Lazy enableAPIMetrics = Lazy.of(() -> + Config.getBooleanProperty("TELEMETRY_API_METRICS_ENABLED", false)); + + public static final ApiMetricAPI apiStatAPI = new ApiMetricAPI(); + + @Override + public void start(final BundleContext context) { + + PluginVersionUtil.init(context); + + if(Boolean.TRUE.equals(enableTelemetry.get())) { + Logger.debug(Activator.class.getName(), "Starting the Telemetry plugin"); + + RestServiceUtil.addResource(TelemetryResource.class); + + try { + apiStatAPI.dropTemporaryTable(); + apiStatAPI.createTemporaryTable(); + + if(Boolean.TRUE.equals(enableAPIMetrics.get())) { + Logger.debug(Activator.class.getName(), "API metrics enabled"); + delegate.addFirst(apiCallWebInterceptor); + ApiMetricFactorySubmitter.INSTANCE.start(); + } + Logger.debug(Activator.class.getName(), "Scheduling Telemetry Job"); + scheduleMetricsJob(); + Logger.debug(Activator.class.getName(), "The Telemetry plugin was started"); + } catch (Throwable t) { + Logger.debug(this, "Error starting the Telemetry plugin.", t); + } + } + } + + private void scheduleMetricsJob() throws ParseException { + final ClusterLockManager lockManager = DotConcurrentFactory.getInstance() + .getClusterLockManager(METRICS_JOB_LOCK_KEY); + + CronExpression cron = new CronExpression(Config + .getStringProperty("TELEMETRY_SAVE_SCHEDULE", "0 0 22 * * ?")) ; + + final Instant now = Instant.now(); + final Instant previousRun = cron.getPrevFireTime(Date.from(now)).toInstant(); + final Instant nextRun = cron.getNextValidTimeAfter(Date.from(previousRun)).toInstant(); + final Duration delay = Duration.between(now, nextRun); + final Duration runEvery = Duration.between(previousRun, nextRun); + + scheduledFuture = DotConcurrentFactory.getScheduledThreadPoolExecutor().scheduleAtFixedRate( + () -> metricsStatsJob.run(lockManager) + , delay.get(ChronoUnit.SECONDS), + runEvery.get(ChronoUnit.SECONDS), + TimeUnit.SECONDS); + } + + @Override + public void stop(BundleContext context) throws Exception { + if(Boolean.TRUE.equals(enableTelemetry.get())) { + RestServiceUtil.removeResource(TelemetryResource.class); + scheduledFuture.cancel(false); + apiStatAPI.dropTemporaryTable(); + + if(Boolean.TRUE.equals(enableAPIMetrics.get())) { + delegate.remove(apiCallWebInterceptor.getName(), true); + ApiMetricFactorySubmitter.INSTANCE.shutdownNow(); + } + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/osgi/PluginVersionUtil.java b/dotCMS/src/main/java/com/dotcms/telemetry/osgi/PluginVersionUtil.java new file mode 100644 index 000000000000..15173b777101 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/osgi/PluginVersionUtil.java @@ -0,0 +1,18 @@ +package com.dotcms.telemetry.osgi; + +import org.osgi.framework.BundleContext; + +public class PluginVersionUtil { + + private static String version; + + private PluginVersionUtil(){} + + public static void init(BundleContext context) { + version = context.getBundle().getHeaders().get("Bundle-Version"); + } + + public static String getVersion(){ + return version; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/rest/ResponseEntityMetricsSnapshotView.java b/dotCMS/src/main/java/com/dotcms/telemetry/rest/ResponseEntityMetricsSnapshotView.java new file mode 100644 index 000000000000..6a1c77b9ac44 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/rest/ResponseEntityMetricsSnapshotView.java @@ -0,0 +1,14 @@ +package com.dotcms.telemetry.rest; + +import com.dotcms.telemetry.MetricsSnapshot; +import com.dotcms.rest.ResponseEntityView; + +/** + * {@link ResponseEntityView} of {@link MetricsSnapshot} + */ +public class ResponseEntityMetricsSnapshotView extends ResponseEntityView { + + public ResponseEntityMetricsSnapshotView(final MetricsSnapshot entity) { + super(entity); + } +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/rest/TelemetryResource.java b/dotCMS/src/main/java/com/dotcms/telemetry/rest/TelemetryResource.java new file mode 100644 index 000000000000..cb50f24c37fa --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/rest/TelemetryResource.java @@ -0,0 +1,51 @@ +package com.dotcms.telemetry.rest; + +import com.dotcms.telemetry.collectors.MetricStatsCollector; +import com.dotcms.rest.WebResource; +import com.dotcms.rest.annotation.NoCache; +import com.dotmarketing.business.Role; +import com.dotmarketing.util.Logger; +import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/v1/telemetry") +public class TelemetryResource { + + @Path("/stats") + @GET + @JSONP + @NoCache + @Produces({MediaType.APPLICATION_JSON}) + @Operation(summary = "Retrieves dotCMS usage data", + responses = { + @ApiResponse( + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = + ResponseEntityMetricsSnapshotView.class)))}) + public final Response getData(@Context final HttpServletRequest request, + @Context final HttpServletResponse response) { + new WebResource.InitBuilder(new WebResource()) + .requestAndResponse(request, response) + .requiredBackendUser(true) + .rejectWhenNoUser(true) + .requiredRoles(Role.CMS_ADMINISTRATOR_ROLE) + .init(); + + return Response.ok(new ResponseEntityMetricsSnapshotView(MetricStatsCollector.getStats())) + .build(); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/util/IndicesSiteSearchUtil.java b/dotCMS/src/main/java/com/dotcms/telemetry/util/IndicesSiteSearchUtil.java new file mode 100644 index 000000000000..b297ad179ba4 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/util/IndicesSiteSearchUtil.java @@ -0,0 +1,30 @@ +package com.dotcms.telemetry.util; + +import com.dotcms.content.elasticsearch.business.IndexStats; +import com.dotcms.content.elasticsearch.business.IndexType; +import com.dotmarketing.business.APILocator; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Provide util methods to get data from the SiteSearch Indices + */ +public enum IndicesSiteSearchUtil { + + INSTANCE; + + /** + * Return all the Site Search Index Information. + * + * @return + */ + public Collection getESIndices() { + return APILocator.getESIndexAPI().getIndicesStats().entrySet().stream() + .map(Map.Entry::getValue) + .filter(index -> IndexType.SITE_SEARCH.is(index.getIndexName())) + .collect(Collectors.toList()); + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/util/JsonUtil.java b/dotCMS/src/main/java/com/dotcms/telemetry/util/JsonUtil.java new file mode 100644 index 000000000000..8583bb691b56 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/util/JsonUtil.java @@ -0,0 +1,24 @@ +package com.dotcms.telemetry.util; + +import com.dotcms.telemetry.business.MetricEndpointPayload; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.json.JsonMapper; + +/** + * Util class to handle Json format + */ +public enum JsonUtil { + + INSTANCE; + + static final JsonMapper jsonMapper = new JsonMapper(); + + public String getAsJson(final MetricEndpointPayload metricEndpointPayload) { + try { + return jsonMapper.writeValueAsString(metricEndpointPayload); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCache.java b/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCache.java new file mode 100644 index 000000000000..f52f7d25975f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCache.java @@ -0,0 +1,33 @@ +package com.dotcms.telemetry.util; + + +import java.util.function.Supplier; + +/** + * This is a cache designed to store any results needed by multiple instances of + * {@link com.dotcms.telemetry.MetricType}. You can utilize a supplier to obtain the data you wish + * to store. The cache is flushed every 10 minutes. + * + * @param + */ +public class MetricCache { + + private final Supplier supplier; + private T currentValue = null; + + public MetricCache(final Supplier supplier) { + this.supplier = supplier; + } + + public T get() { + if (currentValue == null) { + currentValue = this.supplier.get(); + } + + return currentValue; + } + + public void flush() { + currentValue = null; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCaches.java b/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCaches.java new file mode 100644 index 000000000000..4044048424e9 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/telemetry/util/MetricCaches.java @@ -0,0 +1,33 @@ +package com.dotcms.telemetry.util; + +import com.dotcms.telemetry.collectors.api.ApiMetricAPI; + +import java.util.Arrays; + +/** + * Collection of all the Cache regions used on the Telemetry feature + */ +public enum MetricCaches { + + SITE_SEARCH_INDICES(new MetricCache<>(IndicesSiteSearchUtil.INSTANCE::getESIndices)), + TEMPORARY_TABLA_DATA(new MetricCache<>(ApiMetricAPI::getMetricTemporaryTableData)); + + private final MetricCache metricCache; + + MetricCaches(final MetricCache metricCache) { + this.metricCache = metricCache; + } + + public static void flushAll() { + Arrays.stream(MetricCaches.values()).parallel().forEach(metricCaches -> metricCaches.cache().flush()); + } + + public T get() { + return (T) metricCache.get(); + } + + public MetricCache cache() { + return metricCache; + } + +}