diff --git a/src/current/_includes/v23.2/ui/ui-metrics-navigation.md b/src/current/_includes/v23.2/ui/ui-metrics-navigation.md index a1c346aeb21..a3ca330bb9e 100644 --- a/src/current/_includes/v23.2/ui/ui-metrics-navigation.md +++ b/src/current/_includes/v23.2/ui/ui-metrics-navigation.md @@ -2,4 +2,4 @@ Use the **Graph** menu to display metrics for your entire cluster or for a speci To the right of the Graph and Dashboard menus, a time interval selector allows you to filter the view for a predefined or custom time interval. Use the navigation buttons to move to the previous, next, or current time interval. When you select a time interval, the same interval is selected in the [SQL Activity]({% link {{ page.version.version }}/ui-overview.md %}#sql-activity) pages. However, if you select 10 or 30 minutes, the interval defaults to 1 hour in SQL Activity pages. -When viewing graphs, a tooltip will appear at your mouse cursor providing further insight into the data under the mouse cursor. Click anywhere within the graph to pin the tooltip in place, decoupling the tooltip from your mouse movements. Click anywhere within the graph to cause the tooltip to follow your mouse once more. +When viewing graphs, two perpendicular lines will appear at your mouse cursor providing further insight into the data. The metric values are displayed in the legend under the graph. Click anywhere within the graph to pin the values in place, decoupling the values from your mouse movements. Click anywhere within the graph to cause the values to change with your mouse movements once more. \ No newline at end of file diff --git a/src/current/images/v23.2/ui-backfill-pending-ranges.png b/src/current/images/v23.2/ui-backfill-pending-ranges.png new file mode 100644 index 00000000000..2f22382a8a4 Binary files /dev/null and b/src/current/images/v23.2/ui-backfill-pending-ranges.png differ diff --git a/src/current/images/v23.2/ui-changefeed-restarts.png b/src/current/images/v23.2/ui-changefeed-restarts.png new file mode 100644 index 00000000000..69b6c620f04 Binary files /dev/null and b/src/current/images/v23.2/ui-changefeed-restarts.png differ diff --git a/src/current/images/v23.2/ui-changefeed-status.png b/src/current/images/v23.2/ui-changefeed-status.png new file mode 100644 index 00000000000..b054c41b7bf Binary files /dev/null and b/src/current/images/v23.2/ui-changefeed-status.png differ diff --git a/src/current/images/v23.2/ui-commit-latency.png b/src/current/images/v23.2/ui-commit-latency.png new file mode 100644 index 00000000000..6d68bcd7b6b Binary files /dev/null and b/src/current/images/v23.2/ui-commit-latency.png differ diff --git a/src/current/images/v23.2/ui-emitted-bytes.png b/src/current/images/v23.2/ui-emitted-bytes.png new file mode 100644 index 00000000000..3ef282a0533 Binary files /dev/null and b/src/current/images/v23.2/ui-emitted-bytes.png differ diff --git a/src/current/images/v23.2/ui-max-checkpoint-latency.png b/src/current/images/v23.2/ui-max-checkpoint-latency.png new file mode 100644 index 00000000000..8ea2a8b01a3 Binary files /dev/null and b/src/current/images/v23.2/ui-max-checkpoint-latency.png differ diff --git a/src/current/images/v23.2/ui-oldest-protected-timestamp.png b/src/current/images/v23.2/ui-oldest-protected-timestamp.png new file mode 100644 index 00000000000..3d82dff7938 Binary files /dev/null and b/src/current/images/v23.2/ui-oldest-protected-timestamp.png differ diff --git a/src/current/images/v23.2/ui-rangefeed-catchup-scans-duration.png b/src/current/images/v23.2/ui-rangefeed-catchup-scans-duration.png new file mode 100644 index 00000000000..b0aec58a93e Binary files /dev/null and b/src/current/images/v23.2/ui-rangefeed-catchup-scans-duration.png differ diff --git a/src/current/images/v23.2/ui-ranges-in-catchup-mode.png b/src/current/images/v23.2/ui-ranges-in-catchup-mode.png new file mode 100644 index 00000000000..ad64c23b4bd Binary files /dev/null and b/src/current/images/v23.2/ui-ranges-in-catchup-mode.png differ diff --git a/src/current/images/v23.2/ui-schema-registry-registrations.png b/src/current/images/v23.2/ui-schema-registry-registrations.png new file mode 100644 index 00000000000..7c257a02e04 Binary files /dev/null and b/src/current/images/v23.2/ui-schema-registry-registrations.png differ diff --git a/src/current/images/v23.2/ui-sink-counts.png b/src/current/images/v23.2/ui-sink-counts.png new file mode 100644 index 00000000000..0171f7d7643 Binary files /dev/null and b/src/current/images/v23.2/ui-sink-counts.png differ diff --git a/src/current/images/v23.2/ui_changefeed_restarts.png b/src/current/images/v23.2/ui_changefeed_restarts.png deleted file mode 100644 index 7989e71fcb6..00000000000 Binary files a/src/current/images/v23.2/ui_changefeed_restarts.png and /dev/null differ diff --git a/src/current/images/v23.2/ui_sink_byte_traffic.png b/src/current/images/v23.2/ui_sink_byte_traffic.png deleted file mode 100644 index 4bb61c4e83d..00000000000 Binary files a/src/current/images/v23.2/ui_sink_byte_traffic.png and /dev/null differ diff --git a/src/current/images/v23.2/ui_sink_counts.png b/src/current/images/v23.2/ui_sink_counts.png deleted file mode 100644 index dc8a6690cbf..00000000000 Binary files a/src/current/images/v23.2/ui_sink_counts.png and /dev/null differ diff --git a/src/current/v23.2/monitor-and-debug-changefeeds.md b/src/current/v23.2/monitor-and-debug-changefeeds.md index 6c4aa3212ae..1fdfb10393c 100644 --- a/src/current/v23.2/monitor-and-debug-changefeeds.md +++ b/src/current/v23.2/monitor-and-debug-changefeeds.md @@ -5,6 +5,10 @@ toc: true docs_area: stream_data --- +{{site.data.alerts.callout_info}} +Monitoring is only available for [{{ site.data.products.enterprise }} changefeeds]({% link {{ page.version.version }}/change-data-capture-overview.md %}#stream-row-level-changes-with-changefeeds). +{{site.data.alerts.end}} + Changefeeds work as jobs in CockroachDB, which allows for [monitoring](#monitor-a-changefeed) and [debugging](#debug-a-changefeed) through the [DB Console]({% link {{ page.version.version }}/ui-overview.md %}) [**Jobs**]({% link {{ page.version.version }}/ui-jobs-page.md %}) page and [`SHOW JOBS`]({% link {{ page.version.version }}/show-jobs.md %}) SQL statements using the job ID. @@ -24,10 +28,6 @@ We recommend monitoring changefeeds with [Prometheus]({% link {{ page.version.ve ## Monitor a changefeed -{{site.data.alerts.callout_info}} -Monitoring is only available for {{ site.data.products.enterprise }} changefeeds. -{{site.data.alerts.end}} - Changefeed progress is exposed as a high-water timestamp that advances as the changefeed progresses. This is a guarantee that all changes before or at the timestamp have been emitted. You can monitor a changefeed: - On the [**Changefeeds** dashboard]({% link {{ page.version.version }}/ui-cdc-dashboard.md %}) of the DB Console. diff --git a/src/current/v23.2/ui-cdc-dashboard.md b/src/current/v23.2/ui-cdc-dashboard.md index 2ad56c19f33..b89f923714c 100644 --- a/src/current/v23.2/ui-cdc-dashboard.md +++ b/src/current/v23.2/ui-cdc-dashboard.md @@ -9,65 +9,133 @@ The **Changefeeds** dashboard in the DB Console lets you monitor the [changefeed To view this dashboard, [access the DB Console]({% link {{ page.version.version }}/ui-overview.md %}#db-console-access), click **Metrics** on the left-hand navigation bar, and then select **Dashboard** > **Changefeeds**. +{{site.data.alerts.callout_success}} +For more general recommendations on metrics and tooling options for change data capture monitoring, refer to the [Monitor and Debug Changefeeds]({% link {{ page.version.version }}/monitor-and-debug-changefeeds.md %}) page. +{{site.data.alerts.end}} + ## Dashboard navigation {% include {{ page.version.version }}/ui/ui-metrics-navigation.md %} The **Changefeeds** dashboard displays the following time series graphs: -## Max Changefeed Latency +## Changefeed Status -This graph shows the maximum latency for resolved timestamps of any running changefeed. +This graph displays the status of all running changefeeds. -DB Console Max Changefeed Latency graph +DB Console Changefeed Status graph showing running, paused, and failed changefeeds. -{{site.data.alerts.callout_info}} -The maximum latency for resolved timestamps is distinct from and slower than the commit-to-emit latency for individual change messages. For more information about resolved timestamps, see [Ordering guarantees]({% link {{ page.version.version }}/changefeed-messages.md %}#ordering-guarantees). -{{site.data.alerts.end}} +Metric | Description +--------|---- +**Running** | The number of changefeeds running. +**Paused** | The number of [paused]({% link {{ page.version.version }}/pause-job.md %}) changefeeds. +**Failed** | The number of changefeeds that have failed. + +Refer to [`RESUME JOB`]({% link {{ page.version.version }}/resume-job.md %}) and [`PAUSE JOB`]({% link {{ page.version.version }}/pause-job.md %}) to manage a changefeed. + +In the case of a failed changefeed, you may want to use the [`cursor`]({% link {{ page.version.version }}/create-changefeed.md %}#cursor-option) option to restart the changefeed. Refer to the [Start a new changefeed where the previous failed]({% link {{ page.version.version }}/create-changefeed.md %}#start-a-new-changefeed-where-another-ended) example. + +## Commit Latency + +This graph displays the 99th, 90th, and 50th percentile of commit latency for running changefeeds. This is the difference between an event's MVCC timestamp and the time it was acknowledged as received by the [downstream sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). + +DB Console Commit Latency graph showing the 99th, 90th, and 50th percentile of commit latency. + +If the sink batches events, then the difference between the oldest event in the batch and acknowledgement is recorded. Latency during backfill is excluded. -## Sink Byte Traffic +## Emitted Bytes -This graph shows the number of bytes emitted by CockroachDB into the sink for changefeeds. +This graph shows the number of bytes emitted by CockroachDB into the changefeed's [downstream sink]({% link {{ page.version.version }}/changefeed-sinks.md %}). -DB Console Sink Byte Traffic graph +DB Console Emitted Bytes Graph showing the time and emitted bites Metric | Description --------|---- -**Emitted Bytes** | The number of bytes emitted by CockroachDB into the sink for changefeeds. +**Emitted Bytes** | The number of bytes emitted by CockroachDB into the configured changefeed sinks. ## Sink Counts -This graph shows: +This graph displays data relating to the number of messages and flushes at the changefeed sink. - The number of messages that CockroachDB sent to the sink. - The number of flushes that the sink performed for changefeeds. -DB Console Sink Counts graph +DB Console Sink Counts graph Metric | Description --------|---- -**Messages** | The number of messages that CockroachDB sent to the sink for changefeeds. -**Flushes** | The the number of flushes that the sink performed for changefeeds. +**Messages** | The number of messages that CockroachDB sent to the downstream sink. +**Flushes** | The total number of flushes to the sink. -## Sink Timings +## Max Checkpoint Latency -This graph shows: +This graph displays the most any changefeed's persisted [checkpoint]({% link {{ page.version.version }}/how-does-an-enterprise-changefeed-work.md %}) is behind the present time. Larger values indicate issues with successfully ingesting or emitting changes. If errors cause a changefeed to restart, or the changefeed is [paused]({% link {{ page.version.version }}/pause-job.md %}) and unpaused, emitted data up to the last checkpoint may be re-emitted. -- The time in milliseconds per second required by CockroachDB to send messages to the sink. -- The time CockroachDB spent waiting for the sink to flush the messages for changefeeds. +DB Console Max Checkpoint Latency graph -DB Console Sink Timings graph +{{site.data.alerts.callout_info}} +The maximum checkpoint latency is distinct from, and slower than, the commit latency for individual change messages. For more information about resolved timestamps, refer to the [Changefeed Messages]({% link {{ page.version.version }}/changefeed-messages.md %}#resolved-messages) page. +{{site.data.alerts.end}} + +## Changefeed Restarts + +This graph displays the number of times changefeeds restarted due to [retryable errors]({% link {{ page.version.version }}/monitor-and-debug-changefeeds.md %}#changefeed-retry-errors). + +DB Console Changefeed Restarts graph Metric | Description --------|---- -**Message Emit Time** | The time in milliseconds per second required by CockroachDB to send messages to the sink for changefeeds. -**Flush Time** | The time in milliseconds per second that CockroachDB spent waiting for the sink to flush the messages for changefeeds. +**Retryable Errors** | The number of retryable errors that changefeeds are encountering. That is, an error the changefeed will automatically retry. -## Changefeed Restarts +## Oldest Protected Timestamp + +This graph displays the oldest [protected timestamp]({% link {{ page.version.version }}/architecture/storage-layer.md %}#protected-timestamps) of any running changefeed on the cluster. + +DB Console Oldest Protected Timestamp graph + +Metric | Description +--------|---- +**Protected Timestamp Age** | The age of the oldest protected timestamp (of any changefeed) that is protecting data from being [garbage collected]({% link {{ page.version.version }}/architecture/storage-layer.md %}#garbage-collection). + +## Backfill Pending Ranges + +This graph displays the number of ranges being backfilled that are yet to enter the changefeed pipeline. An [initial scan]({% link {{ page.version.version }}/create-changefeed.md %}#initial-scan) or [schema change]({% link {{ page.version.version }}/online-schema-changes.md %}) can cause a backfill. + +DB Console Backfill Pending Ranges graph + +Metric | Description +--------|---- +**Backfill Pending Ranges** | The number of ranges in the backfill process that have not yet entered the changefeed stream. + +## Schema Registry Registrations + +This graph displays the rate of schema registration requests made by CockroachDB nodes to a configured schema registry endpoint. For example, a [Kafka sink]({% link {{ page.version.version }}/changefeed-sinks.md %}#kafka) pointing to a [Confluent Schema Registry]({% link {{ page.version.version }}/stream-a-changefeed-to-a-confluent-cloud-kafka-cluster.md %}). -This graph displays the number of times changefeeds restarted due to retryable errors. +DB Console Schema Registry Registrations graph + +Metric | Description +--------|---- +**Schema Registry Registrations** | The number of registrations to a downstream schema registry. -DB Console Changefeed Restarts graph +## Ranges in catchup mode + +This graph displays the total number of ranges with an active [rangefeed]({% link {{ page.version.version }}/create-and-configure-changefeeds.md %}#enable-rangefeeds) that is performing a catchup scan. + +DB Console Ranges in Catchup Mode graph + +Metric | Description +--------|---- +**Ranges** | The number of ranges that are performing a catchup scan. + +## Rangefeed Catchup Scans Duration + +This graph displays the duration of catchup scans that changefeeds are performing. + +DB Console Rangefeed Catchup Scans Duration graph + +Metric | Description +--------|---- +**(Node Hostname)** | The duration of the catchup scan displayed per node. {% include {{ page.version.version }}/ui/ui-summary-events.md %}