Updates to process_mon.sh and ubuntu_updates_check.sh

cloudbackup_mon.sh

@@ -117,9 +117,8 @@ function token_response() {
 }
 
 token_response
-token=$(echo "$json" | python -c "${python_import} print data['access']['token']['id']")
-url=$(echo "$json" | python -c "${python_import} print(''.join([y['publicURL'] for y in [ x['endpoints'] for x in data['access']['serviceCatalog'] if x['name'] == 'cloudBackup' ][0] if y['region'].lower() == '$region' ]))")
-
+token=$(echo "$json" | python3 -c "${python_import} print(data['access']['token']['id'])")
+url=$(echo "$json" | python3 -c "${python_import} print(''.join([y['publicURL'] for y in [ x['endpoints'] for x in data['access']['serviceCatalog'] if x['name'] == 'cloudBackup' ][0] if y['region'].lower() == '$region' ]))")
 
 # Get latest backup ID
 filter="${this_backup_conf_id:+if x['BackupConfigurationId'] == $this_backup_conf_id}"
@@ -134,8 +133,8 @@ function backup_config_response() {
   fi
 }
 backup_config_response
-backup_config_ids=($(echo "$backup_config" | python -c "${python_import} print (' '.join([str(x['BackupConfigurationId']) for x in data $filter]))"))
-backup_id=($(echo "$backup_config" | python -c "${python_import} print (' '.join([str(x['LastRunBackupReportId']) for x in data $filter]))"))
+backup_config_ids=($(echo "$backup_config" | python3 -c "${python_import} print (' '.join([str(x['BackupConfigurationId']) for x in data $filter]))"))
+backup_id=($(echo "$backup_config" | python3 -c "${python_import} print (' '.join([str(x['LastRunBackupReportId']) for x in data $filter]))"))
 
 report_retry_count=-1
 function report_response() {
@@ -152,15 +151,15 @@ if [ ${#backup_id[@]} -eq 1 -a "[$backup_id]" != "[None]" ]; then
   # Run report to see if backup was successful:
   report_response
 
-  conf_id=$(echo "$report" | python -c "${python_import} print data['BackupConfigurationId']")
+  conf_id=$(echo "$report" | python3 -c "${python_import} print(data['BackupConfigurationId'])")
 
   # Parse report
-  name=$(echo "$report" | python -c "${python_import} print data['BackupConfigurationName']")
-  diagnostics=$(echo "$report" | python -c "${python_import} print data['Diagnostics']")
-  numerrors=$(echo "$report" | python -c "${python_import} print data['NumErrors']")
-  reason=$(echo "$report" | python -c "${python_import} print data['Reason']")
-  state=$(echo "$report" | python -c "${python_import} print data['State']")
-  start_time=$(echo "$report" | python -c "${python_import} import re; print int(int(re.search(\"\\d+\", data['StartTime']).group(0))/1000)")
+  name=$(echo "$report" | python3 -c "${python_import} print(data['BackupConfigurationName'])")
+  diagnostics=$(echo "$report" | python3 -c "${python_import} print(data['Diagnostics'])")
+  numerrors=$(echo "$report" | python3 -c "${python_import} print(data['NumErrors'])")
+  reason=$(echo "$report" | python3 -c "${python_import} print(data['Reason'])")
+  state=$(echo "$report" | python3 -c "${python_import} print(data['State'])")
+  start_time=$(echo "$report" | python3 -c "${python_import} import re; print(int(int(re.search(\"\\d+\", data['StartTime']).group(0))/1000))")
   now=$(date '+%s')
   age=$(( $now - $start_time ))
 

file_info.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Rackspace Cloud Monitoring plugin to provide file/directory information.
 
@@ -28,27 +28,27 @@
 
 def main():
     if len(sys.argv) != 2:
-        print "Requires a full path to the target passed as an argument"
+        print("Requires a full path to the target passed as an argument")
         sys.exit(0)
 
     path = sys.argv[1]
     if not os.path.exists(path):
-        print "status err target does not exist"
+        print("status err target does not exist")
         sys.exit(0)
 
     try:
         details = os.stat(path)
         age = int(time.time() - details.st_ctime)
         size = details.st_size
-        mode = oct(details.st_mode & 0777)
+        mode = oct(details.st_mode & 0o0777)
 
-        print "status ok target exists"
-        print "metric age int", age
-        print "metric bytes int", size
-        print "metric mode string", mode
-    except Exception, e:
-        print "status err Exception discovered: {}".format(str(e))
+        print("status ok target exists")
+        print("metric age int", age)
+        print("metric bytes int", size)
+        print("metric mode string", mode)
+    except Exception as e:
+        print("status err Exception discovered: {}".format(str(e)))
 
 
 if __name__ == '__main__':
-    main()
+    main()

file_info.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Rackspace Cloud Monitoring Plug-In
+# Check the mtime of a file and how long it has been since it has been modified
+#
+# (c) 2015 Justin Gallardo <[email protected]>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage:
+# Place plug-in in /usr/lib/rackspace-monitoring-agent/plugins
+#
+# It accepts one argument, which should be the file you wish to check the mtime of.
+#
+# If the file is a symlink, it will dereference it.
+#
+# Returns 4 metrics:
+#  - mtime: The time(unix epoch) the file was last modified
+#  - age: The number of seconds that have elapsed since the file was modified
+#  - size: The number of bytes of the file
+#  - type: The type of file
+#
+# The following is an example 'criteria' for a Rackspace Monitoring Alarm:
+#
+# if (metric['age'] > 3600) {
+#  return new AlarmStatus(CRITICAL, 'The file has not been modified in more than 1 hour. Last modified #{age} seconds ago');
+# }
+# return new AlarmStatus(OK, 'The file was last modified #{age} seconds ago.');
+#
+file=$1
+
+if [ ! -e $file ]; then
+  echo "status critical \"$file\" does not exist"
+  exit 1
+fi
+
+if [ ! -r $file ]; then
+  echo "status critical \"$file\" is not readable"
+  exit 1
+fi
+
+mtime=$(stat --dereference -c%Y $file)
+now=$(date '+%s')
+age=$(( $now - $mtime ))
+size=$(stat --dereference -c%s $file)
+file_type=$(file -bL $file)
+
+echo "status ok file statted"
+echo "metric mtime uint64 $mtime"
+echo "metric age uint64 $age seconds"
+echo "metric size uint64 $size"
+echo "metric type string $file_type"
+exit 0
+

holland_mysqldump.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 #  Copyright 2013 Rackspace
 #

port_check.sh

@@ -38,7 +38,7 @@ if [ $PROTOCOL = "udp" ]; then
     OPTIONS="-u"
 fi
 
-nc ${OPTIONS} ${IP} ${PORT} < /dev/null > /dev/null 2>&1
+nc -zw 1 ${OPTIONS} ${IP} ${PORT} < /dev/null > /dev/null 2>&1
 
 if [ $? -ne 0 ]; then
     echo "status Nothing listening on port ${IP}:${PORT} (${PROTOCOL})"

process_mon.sh

@@ -46,13 +46,17 @@
 # return new AlarmStatus(CRITICAL, 'Process not running.');
 # }
 #
+# if (metric['process_age'] > 86400) {
+#   return new AlarmStatus(WARNING, 'Process has been running for over 24 hours.');
+# }
+#
 # return new AlarmStatus(OK, 'Process running normally.');
 
 function help {
 
 cat <<HELP
 
-SYNOPSIS:  ./process_mon.sh [process]... 
+SYNOPSIS:  ./process_mon.sh [process]...
 USAGE EXAMPLE: ./process_mon.sh lsync
 
 HELP
@@ -64,4 +68,14 @@ if [ -z "$1" ]; then
 fi
 
 process_check=`ps ax | grep -v grep | grep -v process_mon | grep -c "$1"`
+process_pid=`ps ax | grep -v grep | grep -v process_mon | grep "$1" | head -n 1 | awk '{print $1}'`
+if (( $process_check > 0 )); then
+  process_age=`ps -o etimes= -p "$process_pid"`
+  process_age=${process_age## }
+fi
+
 echo "metric process_mon int $process_check"
+echo "metric process_pid int ${process_pid:-0}"
+echo "metric process_age int ${process_age:-0}"
+
+exit 0

ssl_ca_expiration_check.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# This check was written in response to the poorly communicated expiration
+# of the 2016 PEM file required to make SSL-encrypted connections to Rackspace
+# cloud databases, including MySQL and Redis (from Object Rocket).  This blew
+# up on us in February 2021 when the 2016 PEM file was replaced with the 2021
+# PEM file.
+#
+# The idea here is that we want to be able to get alerted about 4-6 weeks before
+# the CA cert expires again, which will be in 5+ years and we won't remember
+# it without an alert.  Then we can proactively reach out to Rackspace, maybe
+# accept the new cert for 2026 before they switch over and not experience
+# downtime.
+#
+# See:
+# https://docs.objectrocket.com/redis_stunnel.html
+# http://ssl.rackspaceclouddb.com/rackspace-ca-2021.pem
+#
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 </path/to/ca/certificate.pem>"
+  exit 100
+fi
+
+CA_FILE=$1
+NOW=$(TZ=UTC date '+%s')
+
+CMD="openssl x509 -noout -in $CA_FILE -dates"
+NOT_BEFORE=$($CMD | grep notBefore | sed 's/^not.*\=//')
+NOT_AFTER=$($CMD | grep notAfter | sed 's/^not.*\=//')
+
+NOT_BEFORE_AT=$(TZ=UTC date '+%s' --date "$NOT_BEFORE")
+NOT_AFTER_AT=$(TZ=UTC date '+%s' --date "$NOT_AFTER")
+NOT_BEFORE_LOCAL=$(date --date "$NOT_BEFORE")
+NOT_AFTER_LOCAL=$(date --date "$NOT_AFTER")
+NOT_BEFORE_IN=$(( $NOT_BEFORE_AT - $NOW ))
+NOT_AFTER_IN=$(( $NOT_AFTER_AT - $NOW ))
+
+echo "metric not_before string $NOT_BEFORE"
+echo "metric not_before_local string $NOT_BEFORE_LOCAL"
+echo "metric not_before_at uint64 $NOT_BEFORE_AT"
+echo "metric not_before_in int32 $NOT_BEFORE_IN"
+echo "metric now uint64 $NOW"
+echo "metric not_after string $NOT_AFTER"
+echo "metric not_after_local string $NOT_AFTER_LOCAL"
+echo "metric not_after_at uint64 $NOT_AFTER_AT"
+echo "metric not_after_in int32 $NOT_AFTER_IN"
+exit 0

ssl_verify.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Verifies that a given remote host / port has a valid SSL certificate.
+#
+# Usage:
+#   ssl_verify.sh HOST PORT
+#
+# This is really written to make sure that using SSL for cloud databases
+# will work. See:
+# * https://docs.rackspace.com/docs/cloud-databases/v1/general-api-info/using-ssl/
+# * http://ssl.rackspaceclouddb.com/rackspace-ca-2021.pem
+#
+# On Ubuntu machines, this means that the CA file(s) in
+# `/etc/ssl/certs/ca-certificates.crt` should be able to validate the SSL
+# certificate.
+#
+# In order to setup the `rackspace-ca-2021.pem` file, you should
+# place the `rackspace-ca-2021.pem` file in this folder
+# `/usr/local/share/ca-certificates` and rename it to end in `.crt`. Then
+# run `/usr/sbin/update-ca-certificates`
+
+if [ $# -ne 1 ] && [ $# -ne 2 ] && [ $# -ne 3 ]; then
+  echo "Usage: $0 <ip> [port] [ca_file]"
+  exit 100
+fi
+
+HOST=$1
+
+if [ $# -eq 2 ] || [ $# -eq 3 ]; then
+  PORT=$2
+else
+  PORT=443
+fi
+
+if [ $# -eq 3 ]; then
+  CA_FILE=$3
+  RESULT=`echo | openssl s_client -connect $HOST:$PORT 2>/dev/null | openssl x509 | openssl verify -CAfile $CA_FILE 2>/dev/null | awk '{ gsub("stdin: ", "") ; print $0 }'`
+else
+  CA_FILE=''
+  RESULT=`echo | openssl s_client -connect $HOST:$PORT 2>/dev/null | openssl x509 | openssl verify 2>/dev/null | awk '{ gsub("stdin: ", "") ; print $0 }'`
+fi
+
+echo "metric result string ${RESULT:-Error: No data}";
+echo "metric host string ${HOST:-Error: No data}";
+echo "metric port uint32 ${PORT:-Error: No data}";
+echo "metric ca_file string ${CA_FILE:-Error: No data}";
+exit 0

ubuntu_updates_check.sh

@@ -23,23 +23,55 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-OUTPUT=$(/usr/lib/update-notifier/apt-check 2>&1)
+OUTPUT=$(apt list --upgradable 2>/dev/null)
 
 if [ $? -ne 0 ]; then
     echo "Failed to retrieve a number of pending updates"
     exit 100
 fi
 
-PENDING_OTHER=$(echo "${OUTPUT}" | cut -d ";" -f 1)
-PENDING_SECURITY=$(echo "${OUTPUT}" | cut -d ";" -f 2)
+HELD=$(apt-mark showhold 2>/dev/null)
+
+# Remove any held packages from OUTPUT
+OLD_IFS=$IFS
+IFS=$'\n'
+OUT_ARR=($OUTPUT)
+HELD_ARR=($HELD)
+IFS=$OLD_IFS
+NEW_OUTPUT=""
+line_break='
+'
+
+for (( i=0; i<${#OUT_ARR[@]}; i++ )); do
+  line="${OUT_ARR[$i]}"
+  package=$(echo $line | awk -F'/' '{print $1}')
+  omit=0
+  for (( j=0; j<${#HELD_ARR[@]}; j++ )); do
+    held_package="${HELD_ARR[$j]}"
+    if [[ "$package" == "$held_package"  ]]; then
+      omit=1
+    fi
+  done
+  if [[ "$omit" == "0" ]]; then
+    if [[ "$i" == "0" ]]; then
+      NEW_OUTPUT="${NEW_OUTPUT}${line}"
+    else
+      NEW_OUTPUT="${NEW_OUTPUT}${line_break}${line}"
+    fi
+  fi
+done
+OUTPUT=$NEW_OUTPUT
+
+PENDING_OTHER=$(echo "${OUTPUT}" | grep -v "Listing..." | grep -v -P "(,|/)$(lsb_release -cs)-security" | wc -l)
+PENDING_SECURITY=$(echo "${OUTPUT}" | grep -v "Listing..." | grep -P "(,|/)$(lsb_release -cs)-security" | wc -l)
 REBOOT_REQUIRED="no"
 
 if [ -f "/var/run/reboot-required" ]; then
   REBOOT_REQUIRED="yes"
 fi
 
 if [ $((PENDING_OTHER+PENDING_SECURITY)) -gt 0 ]; then
-  UPGRADABLE_PACKAGES=$(apt list --upgradable 2>/dev/null | grep -v Listing | awk -F'/' '{print $1}' | paste -sd ',' -)
+  UPGRADABLE_PACKAGES=$(echo "${OUTPUT}" | grep -v Listing | awk -F'/' '{print $1}' | paste -sd ',' -)
 else
   UPGRADABLE_PACKAGES="none"
 fi
@@ -51,4 +83,4 @@ echo "metric pending_other uint32 ${PENDING_OTHER}"
 echo "metric reboot_required string ${REBOOT_REQUIRED}"
 echo "metric upgradable_packages string ${UPGRADABLE_PACKAGES}"
 
-exit 0
+exit 0