From b9a4c3553e81ac016d6061723f4d5f5e4ef20d2b Mon Sep 17 00:00:00 2001 From: Waldirio M Pinheiro Date: Wed, 12 May 2021 14:08:23 -0700 Subject: [PATCH] Qpid and Pulp live check (#70) * New script to check qpid live --- README.md | 39 ++++++++++++++++++++++++++++++++++++ check-pulp-msgs.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100755 check-pulp-msgs.sh diff --git a/README.md b/README.md index f61a904..9419c13 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Tools for use in supporting the operation of Satellite 6 - [tuning-profiles](#tuning-profiles) - [postgres-activity-report](#postgres-activity-report) - [production-log-load-stats](#production-log-load-stats) +- [check-pulp-msg](#check-pulp-msg) ## [check-perf-tuning](check-perf-tuning) @@ -97,3 +98,41 @@ Analyze `production.log` for load+performance statistics about types of requests ## [tuning profiles](tuning-profiles) Some tuning template settings (custom-hiera.yml) for Satellite 6 with [32](tuning-profiles/custom-hiera-medium-32G.yaml), [64](tuning-profiles/custom-hiera-large-64G.yaml), [128](tuning-profiles/custom-hiera-ex-large-128G.yaml) or [256GB](tuning-profiles/custom-hiera-2ex-large-256G.yaml) of RAM. If you have less than 32GB RAM the default settings for Satellite 6 are appropriate. + + +## [check-pulp-msg](check-pulp-msg) + +This script will check pulp's `resource_manager` and `reserved_resource_worker` qpid queues and print some information to stdout; if the environment variable `CHECK_PULP_MSG_LOG_OUTPUT=Y` is set, the output will also be written to a log at `/var/log/pulp_queue.log` + +The output will show the count and total number of bytes for messages currently in the queue, as well as total messages coming into and going out of the queue since the queue came online. The output was also display the number of connections and bindings for each queue. + +In order to use, just download the script `check-pulp-msgs.sh` and execute it. +``` +# ./check-pulp-msgs.sh +``` +You can also check the help +``` +# ./check-pulp-msgs.sh --help +You can just run './check-pulp-msgs.sh' to view on your screen or 'LOG_OUTPUT=Y ./check-pulp-msgs.sh' to view on screen + log in the file '/var/log/pulp_queue.log' +``` +or +``` +# ./check-pulp-msgs.sh -h +You can just run './check-pulp-msgs.sh' to view on your screen or 'LOG_OUTPUT=Y ./check-pulp-msgs.sh' to view on screen + log in the file '/var/log/pulp_queue.log' +``` + +So, that said, if you would like to see only on your screen you can just execute the command `./check-pulp-msgs.sh`. However, if you would like to see on your screen and also log the information for troubleshooting purposes, you can try `LOG_OUTPUT=Y ./check-pulp-msgs.sh` + +Below an example of the output +``` +Sat May 1 15:17:05 EDT 2021 + queue dur autoDel excl msg msgIn msgOut bytes bytesIn bytesOut cons bind + ====================================================================================================================================================== + reserved_resource_worker-0@satellite.example.com.celery.pidbox Y 0 0 0 0 0 0 1 2 + reserved_resource_worker-0@satellite.example.com.dq2 Y 0 228 228 0 320k 320k 1 2 + reserved_resource_worker-1@satellite.example.com.celery.pidbox Y 0 0 0 0 0 0 1 2 + reserved_resource_worker-1@satellite.example.com.dq2 Y 0 208 208 0 5.80m 5.80m 1 2 + resource_manager Y 0 218 218 0 5.95m 5.95m 1 2 + resource_manager@satellite.example.com.celery.pidbox Y 0 0 0 0 0 0 1 2 + resource_manager@satellite.example.com.dq2 Y 0 0 0 0 0 0 1 2 +``` diff --git a/check-pulp-msgs.sh b/check-pulp-msgs.sh new file mode 100755 index 0000000..a19520f --- /dev/null +++ b/check-pulp-msgs.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# +# Developer ..........: Waldirio M Pinheiro +# Created at .........: 05/01/2021 +# Tested Sat Version .: 6.6, 6.7, 6.8, 6.9 +# Purpose ............: Check the qpid messages coming from/to Pulp. +# There is a BZ that is causing an weird behavior +# and this script help to identify it. Also there +# is a KCS, both below +# +# https://access.redhat.com/solutions/5911931 +# https://bugzilla.redhat.com/show_bug.cgi?id=1945534 +# + +SRV_FQDN=$(hostname -f) +CERT="/etc/pki/pulp/qpid/client.crt" +LOG="/var/log/pulp_queue.log" + +qpid_stat() +{ + qpid-stat --ssl-certificate=$CERT -b amqps://localhost:5671 -q | grep -E "(resource_manager.*$SRV_FQDN.*|^ queu|^ =|resource_worker.*$SRV_FQDN.*|resource_manager )" +} + +write_log() +{ + if [[ $CHECK_PULP_MSG_LOG_OUTPUT == "Y" ]] ; then + tee -a $LOG + else + cat + fi +} + +check() +{ +while : +do + date | write_log + qpid_stat | write_log + sleep 3 +done +} + +# Main +if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then + echo "You can just run '$0' to view on your screen or 'CHECK_PULP_MSG_LOG_OUTPUT=Y $0' to view on screen + log in the file '$LOG'" + exit +fi + +check