diff --git a/anabot-debug.sh b/anabot-debug.sh new file mode 100755 index 00000000..02ec292d --- /dev/null +++ b/anabot-debug.sh @@ -0,0 +1,62 @@ +#!/usr/bin/bash + +log_dir="/tmp" + +while [ ! -e ${DBUS_SESSION_BUS_ADDRESS##*=} ]; do + echo "Waiting for session bus (${DBUS_SESSION_BUS_ADDRESS}) to become available." + sleep 1; +done +# capture traffic on the session bus for debugging purposes +dbus-monitor --pcap > "${log_dir}/dbus.pcap" & + +# check if a core dump for a PID exists and if not, create it +check_dump_pid() { + local pid="$1" + if [ ! -f ${log_dir}/dump-${pid}-*.info ]; then + # find corresponding executable name and replace slashes + local exe=$(coredumpctl --no-legend | awk -v pid=" *${pid} *" ' + $5 ~ pid { + gsub("/", "-", $10) + print $10 + } + ') + local dump_prefix="${log_dir}/dump-${pid}-${exe}" + coredumpctl info ${pid} &> "${dump_prefix}.info" + + # upload the core dump to FTP (due to file size constraint in Beaker), + # we can just make use of the existing compressed core dump + local existing_dump=$(awk '/Storage:/ {print $2}' ${dump_prefix}.info) + existing_dump=${existing_dump/ /} + local dump_ext="${existing_dump##*.}" + local recipeid="$(cat /run/anabot/recipeid)" + local upload_filename="dump-${recipeid}-${pid}-${exe}.${dump_ext}" + if [[ -f /run/anabot/ftp_server && -f /run/anabot/ftp_user && \ + -f /run/anabot/ftp_password ]]; then + local ftp_server=$(cat /run/anabot/ftp_server) + local ftp_user=$(cat /run/anabot/ftp_user) + local ftp_password=$(cat /run/anabot/ftp_password) + ftp -npv ${ftp_server} <<< " + user ${ftp_user} ${ftp_password} + put ${existing_dump} ${upload_filename}" + else + echo "Not uploading core dump for '${existing_dump}', FTP server settings \n +are missing or incomplete!" + fi + fi +} + +# have a look for possible core dumps (should be uploaded by Anabot, +# together with the dbus traffic capture) +old_dump_pids="" +while sleep 1; do + # coredumpctl output may also contain unwanted output (besides the core dump + # records) under some circumstances, so we need to make sure to only filter the PIDs + new_dump_pids=$(coredumpctl --no-legend | awk '$5 ~ "^ *[0-9]+ *$" {print $5}') + if [ "${new_dump_pids}" != "${old_dump_pids}" ]; then + echo "old_dump_pids: '$old_dump_pids'; new_dump_pids: '$new_dump_pids'" + for pid in ${new_dump_pids}; do + check_dump_pid ${pid} + done + old_dump_pids="${new_dump_pids}" + fi +done diff --git a/etc/systemd/system/anabot-debug.service b/etc/systemd/system/anabot-debug.service new file mode 100644 index 00000000..2fb5d796 --- /dev/null +++ b/etc/systemd/system/anabot-debug.service @@ -0,0 +1,10 @@ +[Unit] +Description=Anabot service for debug information collection +After=anaconda.service + +[Service] +ExecStart=sh -c "which Xorg || /opt/anabot-debug.sh" +Environment="DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus" + +[Install] +WantedBy=anaconda.service diff --git a/etc/systemd/system/anaconda-direct.service.wants/anabot-debug.service b/etc/systemd/system/anaconda-direct.service.wants/anabot-debug.service new file mode 120000 index 00000000..db758df6 --- /dev/null +++ b/etc/systemd/system/anaconda-direct.service.wants/anabot-debug.service @@ -0,0 +1 @@ +../anabot-debug.service \ No newline at end of file diff --git a/etc/systemd/system/anaconda.service.wants/anabot-debug.service b/etc/systemd/system/anaconda.service.wants/anabot-debug.service new file mode 120000 index 00000000..db758df6 --- /dev/null +++ b/etc/systemd/system/anaconda.service.wants/anabot-debug.service @@ -0,0 +1 @@ +../anabot-debug.service \ No newline at end of file diff --git a/modules/anaconda_logfiles/__init__.py b/modules/anaconda_logfiles/__init__.py index 9eeabfa9..a592cd2d 100644 --- a/modules/anaconda_logfiles/__init__.py +++ b/modules/anaconda_logfiles/__init__.py @@ -14,6 +14,7 @@ reporter = teres.Reporter.get_reporter() watched_files = [ '/tmp/anaconda.log', + '/tmp/dbus.pcap', '/tmp/ifcfg.log', '/tmp/packaging.log', '/tmp/program.log', @@ -22,6 +23,8 @@ ] globfiles = [ '/tmp/anaconda-tb-*', + # actual core dumps are uploaded to FTP by anabot-debug service + '/tmp/dump-*.info', ] def mainloop(): diff --git a/modules/beaker/__init__.py b/modules/beaker/__init__.py index 68b5a1d1..c6a78feb 100644 --- a/modules/beaker/__init__.py +++ b/modules/beaker/__init__.py @@ -22,7 +22,7 @@ import teres import teres.bkr_handlers from anabot.exceptions import UnrelatedException -from anabot.variables import set_variable, get_variable +from anabot.variables import set_variable, get_variable, get_variables BEAKER = "https://%s" % get_variable('beaker_hub_hostname') EXTERNAL_VAR_PREFIX = "ANABOT_SUB_" @@ -225,3 +225,18 @@ def get_param_names(): (var_name, var_value)) set_variable(var_name, var_value) +# Ensure potential core dumps will get uploaded to FTP if possible. +# The following settings are consumed by anabot-debug service: +ftp_settings = {v:get_variable(v) for v in get_variables() if v.startswith("ftp_")} +if ftp_settings: + required_ftp_vars = ("ftp_server", "ftp_user", "ftp_password") + if all([v in ftp_settings.keys() for v in required_ftp_vars]): + rep.log_debug("Using FTP server settings for core dumps upload") + for ftp_variable in ftp_settings: + with open(f"/run/anabot/{ftp_variable}", "w") as f: + f.write(ftp_settings[ftp_variable]) + # We also need the recipe id to properly name the core dump file for upload + with open("/run/anabot/recipeid", "w") as f: + f.write(get_recipe_id()) + else: + rep.log_warn("Not all settings for core dumps upload via FTP have been set!")