From 3ef6bfd60af9b9fd168b691716ab986c56385911 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Tue, 12 Nov 2024 19:18:11 +0000 Subject: [PATCH 1/2] Fix dtrace system level scripts The dtrace scripts did not handle multiple session IDs. Refactor the scripts get-ds-state and get-lr-state to correctly handle a PID with multiple sessions and improved the printing time for getting a system summary output. --- tools/dtrace/get-ds-state.d | 23 +++++++++++++++++++++++ tools/dtrace/get-ds-state.sh | 21 ++++++++++++++------- tools/dtrace/get-lr-state.d | 26 ++++++++++++++++++++++++++ tools/dtrace/get-lr-state.sh | 23 +++++++++++++++-------- tools/make-dtrace.sh | 2 ++ 5 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 tools/dtrace/get-ds-state.d create mode 100755 tools/dtrace/get-lr-state.d diff --git a/tools/dtrace/get-ds-state.d b/tools/dtrace/get-ds-state.d new file mode 100644 index 000000000..74277f52b --- /dev/null +++ b/tools/dtrace/get-ds-state.d @@ -0,0 +1,23 @@ +/* + * Print a status line for a given PID. + * Exit after 5 seconds. + */ +#pragma D option quiet +#pragma D option strsize=1k + +crucible_upstairs*:::up-status +{ + my_sesh = json(copyinstr(arg1), "ok.session_id"); + + printf("%6d %8s %17s %17s %17s\n", + pid, + substr(my_sesh, 0, 8), + json(copyinstr(arg1), "ok.ds_state[0]"), + json(copyinstr(arg1), "ok.ds_state[1]"), + json(copyinstr(arg1), "ok.ds_state[2]")); +} + +tick-5s +{ + exit(0); +} diff --git a/tools/dtrace/get-ds-state.sh b/tools/dtrace/get-ds-state.sh index 7662c5fc1..feffd47b3 100755 --- a/tools/dtrace/get-ds-state.sh +++ b/tools/dtrace/get-ds-state.sh @@ -1,9 +1,16 @@ #!/bin/bash # -# This script will display the downstairs states for any propolis zones it -# finds running on a system. -for zzz in $(zoneadm list | grep propolis); do - echo -n "$zzz " - ppid=$(zlogin "$zzz" pgrep propolis-server) - dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %17s %17s %17s", pid, json(copyinstr(arg1), "ok.ds_state[0]"), json(copyinstr(arg1), "ok.ds_state[1]"), json(copyinstr(arg1), "ok.ds_state[2]")); exit(0); }' -done +# This script will display the downstairs states for each pid/session +# it finds running on a system. +filename='/tmp/get-ds-state.out' + +# Gather state on all running propolis servers, record summary to a file +dtrace -s /opt/oxide/crucible_dtrace/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename" +# Walk the lines in the file, append the zone name to each line. +while read p; do + # For each line in the file, pull out the PID we are looking at and + # print the zone that process is running in. + pid=$(echo $p | awk '{print $1}') + zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28) + echo "$zone $p" +done < "$filename" diff --git a/tools/dtrace/get-lr-state.d b/tools/dtrace/get-lr-state.d new file mode 100755 index 000000000..948336be7 --- /dev/null +++ b/tools/dtrace/get-lr-state.d @@ -0,0 +1,26 @@ +/* + * Print a live reapir status line. + * Exit after 5 seconds. + */ +#pragma D option quiet +#pragma D option strsize=1k + +crucible_upstairs*:::up-status +{ + my_sesh = json(copyinstr(arg1), "ok.session_id"); + + printf("%6d %8s %s %s %s %s %s %s\n", + pid, + substr(my_sesh, 0, 8), + json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"), + json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"), + json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"), + json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"), + json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"), + json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]")); +} + +tick-5s +{ + exit(0); +} diff --git a/tools/dtrace/get-lr-state.sh b/tools/dtrace/get-lr-state.sh index 690e20438..ab0f8d407 100755 --- a/tools/dtrace/get-lr-state.sh +++ b/tools/dtrace/get-lr-state.sh @@ -1,9 +1,16 @@ #!/bin/bash -# -# This script will log into every propolis zone it finds and get the -# DTrace live repair counters from propolis-server in each zone. -for zzz in $(zoneadm list | grep propolis); do - echo -n "$zzz " - ppid=$(zlogin "$zzz" pgrep propolis-server) - dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %s %s %s %s %s %s", pid, json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]")); exit(0); }' -done + +# This script will display the downstairs live repair for each +# pid/session it finds running on a system. +filename='/tmp/get-lr-state.out' + +# Gather state on all running propolis servers, record summary to a file +dtrace -s /opt/oxide/crucible_dtrace/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename" +# Walk the lines in the file, append the zone name to each line. +while read p; do + # For each line in the file, pull out the PID we are looking at and + # print the zone that process is running in. + pid=$(echo $p | awk '{print $1}') + zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28) + echo "$zone $p" +done < "$filename" diff --git a/tools/make-dtrace.sh b/tools/make-dtrace.sh index 8b1324234..d1c4aba64 100755 --- a/tools/make-dtrace.sh +++ b/tools/make-dtrace.sh @@ -23,7 +23,9 @@ tar cvf ../../out/crucible-dtrace.tar \ README.md \ all_downstairs.d \ downstairs_count.d \ + get-ds-state.d \ get-ds-state.sh \ + get-lr-state.d \ get-lr-state.sh \ perf-downstairs-os.d \ perf-downstairs-three.d \ From 5698e843f5bfd164775ff7ff0d12625b59494d67 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 15 Nov 2024 01:38:50 +0000 Subject: [PATCH 2/2] Fix issues from PR comments --- tools/dtrace/get-ds-state.d | 2 +- tools/dtrace/get-ds-state.sh | 2 +- tools/dtrace/get-lr-state.d | 2 +- tools/dtrace/get-lr-state.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dtrace/get-ds-state.d b/tools/dtrace/get-ds-state.d index 74277f52b..92aaf1dd5 100644 --- a/tools/dtrace/get-ds-state.d +++ b/tools/dtrace/get-ds-state.d @@ -1,5 +1,5 @@ /* - * Print a status line for a given PID. + * Print a status line for all matching probes. * Exit after 5 seconds. */ #pragma D option quiet diff --git a/tools/dtrace/get-ds-state.sh b/tools/dtrace/get-ds-state.sh index feffd47b3..5f5a13490 100755 --- a/tools/dtrace/get-ds-state.sh +++ b/tools/dtrace/get-ds-state.sh @@ -7,7 +7,7 @@ filename='/tmp/get-ds-state.out' # Gather state on all running propolis servers, record summary to a file dtrace -s /opt/oxide/crucible_dtrace/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename" # Walk the lines in the file, append the zone name to each line. -while read p; do +while read -r p; do # For each line in the file, pull out the PID we are looking at and # print the zone that process is running in. pid=$(echo $p | awk '{print $1}') diff --git a/tools/dtrace/get-lr-state.d b/tools/dtrace/get-lr-state.d index 948336be7..983628021 100755 --- a/tools/dtrace/get-lr-state.d +++ b/tools/dtrace/get-lr-state.d @@ -1,5 +1,5 @@ /* - * Print a live reapir status line. + * Print a live repair status line for all matching probes. * Exit after 5 seconds. */ #pragma D option quiet diff --git a/tools/dtrace/get-lr-state.sh b/tools/dtrace/get-lr-state.sh index ab0f8d407..5af032379 100755 --- a/tools/dtrace/get-lr-state.sh +++ b/tools/dtrace/get-lr-state.sh @@ -7,7 +7,7 @@ filename='/tmp/get-lr-state.out' # Gather state on all running propolis servers, record summary to a file dtrace -s /opt/oxide/crucible_dtrace/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename" # Walk the lines in the file, append the zone name to each line. -while read p; do +while read -r p; do # For each line in the file, pull out the PID we are looking at and # print the zone that process is running in. pid=$(echo $p | awk '{print $1}')