[RTT-6303] Gather D-Bus traffic and core dumps

to hopefully get some useful data related to various D-Bus
exceptions and potentially some other issues happening without
and obvious reason.

anabot-debug.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/bash
+
+log_dir="/tmp"
+
+while [ ! -e ${DBUS_SESSION_BUS_ADDRESS##*=} ]; do
+    echo "Waiting for session bus (${DBUS_SESSION_BUS_ADDRESS}) to become available."
+    sleep 1;
+done
+# capture traffic on the session bus for debugging purposes
+dbus-monitor --pcap > "${log_dir}/dbus.pcap" &
+
+# check if a core dump for a PID exists and if not, create it
+check_dump_pid() {
+    local pid="$1"
+    if [ ! -f ${log_dir}/dump-${pid}-*.info ]; then
+        # find corresponding executable name and replace slashes
+        local exe=$(coredumpctl --no-legend | awk -v pid=" *${pid} *" '
+            $5 ~ pid {
+                gsub("/", "-", $10)
+                print $10
+            }
+        ')
+        local dump_prefix="${log_dir}/dump-${pid}-${exe}"
+        coredumpctl info ${pid} &> "${dump_prefix}.info"
+
+        # upload the core dump to FTP (due to file size constraint in Beaker),
+        # we can just make use of the existing compressed core dump
+        local existing_dump=$(awk '/Storage:/ {print $2}' ${dump_prefix}.info)
+        existing_dump=${existing_dump/ /}
+        local dump_ext="${existing_dump##*.}"
+        local recipeid="$(cat /run/anabot/recipeid)"
+        local upload_filename="dump-${recipeid}-${pid}-${exe}.${dump_ext}"
+        if [[ -f /run/anabot/ftp_server && -f /run/anabot/ftp_user && \
+            -f /run/anabot/ftp_password ]]; then
+            local ftp_server=$(cat /run/anabot/ftp_server)
+            local ftp_user=$(cat /run/anabot/ftp_user)
+            local ftp_password=$(cat /run/anabot/ftp_password)
+            ftp -npv ${ftp_server} <<< "
+                user ${ftp_user} ${ftp_password}
+                put ${existing_dump} ${upload_filename}"
+        else
+            echo "Not uploading core dump for '${existing_dump}', FTP server settings \n
+are missing or incomplete!"
+        fi
+    fi
+}
+
+# have a look for possible core dumps (should be uploaded by Anabot,
+# together with the dbus traffic capture)
+old_dump_pids=""
+while sleep 1; do
+    # coredumpctl output may also contain unwanted output (besides the core dump
+    # records) under some circumstances, so we need to make sure to only filter the PIDs
+    new_dump_pids=$(coredumpctl --no-legend | awk '$5 ~ "^ *[0-9]+ *$" {print $5}')
+    if [ "${new_dump_pids}" != "${old_dump_pids}" ]; then
+        echo "old_dump_pids: '$old_dump_pids'; new_dump_pids: '$new_dump_pids'"
+        for pid in ${new_dump_pids}; do
+            check_dump_pid ${pid}
+        done
+        old_dump_pids="${new_dump_pids}"
+    fi
+done

etc/systemd/system/anabot-debug.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Anabot service for debug information collection
+After=anaconda.service
+
+[Service]
+ExecStart=sh -c "which Xorg || /opt/anabot-debug.sh"
+Environment="DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus"
+
+[Install]
+WantedBy=anaconda.service

etc/systemd/system/anaconda-direct.service.wants/anabot-debug.service

@@ -0,0 +1 @@
+../anabot-debug.service

etc/systemd/system/anaconda.service.wants/anabot-debug.service

@@ -0,0 +1 @@
+../anabot-debug.service

modules/anaconda_logfiles/__init__.py

@@ -14,6 +14,7 @@
 reporter = teres.Reporter.get_reporter()
 watched_files = [
     '/tmp/anaconda.log',
+    '/tmp/dbus.pcap',
     '/tmp/ifcfg.log',
     '/tmp/packaging.log',
     '/tmp/program.log',
@@ -22,6 +23,8 @@
 ]
 globfiles = [
     '/tmp/anaconda-tb-*',
+    # actual core dumps are uploaded to FTP by anabot-debug service
+    '/tmp/dump-*.info',
 ]
 
 def mainloop():

modules/beaker/__init__.py

@@ -22,7 +22,7 @@
 import teres
 import teres.bkr_handlers
 from anabot.exceptions import UnrelatedException
-from anabot.variables import set_variable, get_variable
+from anabot.variables import set_variable, get_variable, get_variables
 
 BEAKER = "https://%s" % get_variable('beaker_hub_hostname')
 EXTERNAL_VAR_PREFIX = "ANABOT_SUB_"
@@ -225,3 +225,18 @@ def get_param_names():
         (var_name, var_value))
     set_variable(var_name, var_value)
 
+# Ensure potential core dumps will get uploaded to FTP if possible.
+# The following settings are consumed by anabot-debug service:
+ftp_settings = {v:get_variable(v) for v in get_variables() if v.startswith("ftp_")}
+if ftp_settings:
+    required_ftp_vars = ("ftp_server", "ftp_user", "ftp_password")
+    if all([v in ftp_settings.keys() for v in required_ftp_vars]):
+        rep.log_debug("Using FTP server settings for core dumps upload")
+        for ftp_variable in ftp_settings:
+            with open(f"/run/anabot/{ftp_variable}", "w") as f:
+                f.write(ftp_settings[ftp_variable])
+        # We also need the recipe id to properly name the core dump file for upload
+        with open("/run/anabot/recipeid", "w") as f:
+            f.write(get_recipe_id())
+    else:
+        rep.log_warn("Not all settings for core dumps upload via FTP have been set!")