This repository has been archived by the owner on Mar 16, 2024. It is now read-only.
forked from getanteon/alaz
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathNotes
147 lines (111 loc) · 6.25 KB
/
Notes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
To clean debug print pipe -> cat /sys/kernel/debug/tracing/trace_pipe
When debugging, bpf_trace_printk's "fmt_size" is important. Otherwise, you can get errors like below from verifier:
load program: permission denied: invalid access to map value, value_size=31 off=14 size=20: R1 min value is outside of the allowed memory range (228 line(s) omitted)
// To install bpftool:
Clone https://github.com/torvalds/linux.git
// linux/tools/bpf/bpftool
// make
// gcc must be installed
To see bpf calls with strace:
sudo strace -o calls.txt -f -e trace=bpf,ioctl go run -exec sudo main.go bpf_bpfel.go
See "/sys/kernel/debug/tracing/events" to find available tracepoints
> cat /sys/kernel/debug/tracing/events/sock/inet_sock_set_state/format
To generate kernel headers files necessary:
> bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
See a tracepoints format:
> cat /sys/kernel/debug/tracing/events/task/task_newtask/format
// To debug program in vscode:
- First build the program with "go build -o myapp main.go bpf_bpfel.go"
- "mode": "exec",
"program": "${workspaceFolder}/prog/tcp_state/myapp",
"console": "integratedTerminal",
"asRoot": true,
// Compile bpf.c files with clang and llvm
> clang \
-target bpf \
-D __TARGET_ARCH_x86 \
-Wall \
-O2 -g -o hello-buffer-config.bpf.o -c kprobe.c
> llvm-strip -g hello-buffer-config.bpf.o (strip executable from debugging symbols)
BPF_CLANG=clang-14 BPF_CFLAGS="-O2 -g -Wall -Werror -D__TARGET_ARCH_x86" go generate ./...
// To see available tracepoints:
> cat /sys/kernel/tracing/available_events (SEC definition matches with this)
// How to take arguments of a function attached to a tracepoint:
1) Look at the format from :
> cat /sys/kernel/debug/tracing/events/sock/inet_sock_set_state/format
or
> use bcc's tplife tool:
> sudo tplist-bpfcc -v | grep inet_sock_set_state -A 20
2) search corresponding struct in vmlinux.h file, if exists copy into your headers
If not, create your own struct, be careful with paddings
cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_connect/format
// Look for syscall:
> man 2 connect
int connect(int sockfd, const struct sockaddr *addr,
socklen_t addrlen);
// Look format of the tracepoint:
> cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_connect/format
name: sys_enter_connect
ID: 1473
format:
field:unsigned short common_type; offset:0; size:2; signed:0;
field:unsigned char common_flags; offset:2; size:1; signed:0;
field:unsigned char common_preempt_count; offset:3; size:1; signed:0;
field:int common_pid; offset:4; size:4; signed:1;
field:int __syscall_nr; offset:8; size:4; signed:1;
field:int fd; offset:16; size:8; signed:0;
field:struct sockaddr * uservaddr; offset:24; size:8; signed:0;
field:int addrlen; offset:32; size:8; signed:0;
print fmt: "fd: 0x%08lx, uservaddr: 0x%08lx, addrlen: 0x%08lx", ((unsigned long)(REC->fd)), ((unsigned long)(REC->uservaddr)), ((unsigned long)(REC->addrlen))
args should be the same as in the format
Address Families : AF_INET, AF_INET6, AF_PACKET, AF_UNIX, AF_XDP
Protocol Families : PF_INET, PF_INET6, PF_PACKET, PF_UNIX, PF_XDP
Transport Protocols : IPPROTO_UDP or IPPROTO_TCP
Sock Types : SOCK_STREAM, SOCK_DGRAM, or SOCK_RAW.
Detaching bpf programs with bpftool
> bpftool perf list
then
> kill -9
in order to see map contents (human readable) with "bpftool map dump"
define as:
__type(key, void *);
__type(value, struct sk_info);
// sk_info icinde pointer olursa executing "common" at <$.TypeDeclaration>: error calling TypeDeclaration: Struct:"rw_args_t": field 1: type *btf.Pointer: not supported
// Ebpf error:
https://github.com/cilium/ebpf/blob/master/btf/core.go 0xbad2310
invalid func unknown#195896080:
195896080 -> 0xbad2310
"bad relo"
it means libbpf couldn't relocate a struct field reference against the current running kernel's layout
basically, your vmlinux.h defines a struct field that wasn't defined when your kernel was built
yani kerneldaki struct fieldina erismiyorsun. kendin tanimlamissin ama kernelde yok. bunu core_read yapmaya
calisirsan kernelda bulamadigi icin mapping de yapamiyor error veriyor.
exlude on strace:
sudo strace -o myapc.txt -f -e 'trace=!epoll_pwait,nanosleep,futex,setsockopt,getsockopt,epoll_ctl' -p 17555
filter on strace:
sudo strace -f -e trace=socket,connect,write,read,accept -p 805247
write, read , close
timeout case -> write , close ??
start_app_onpre(23005)---gunicorn(23118)-+-gunicorn(23123)---{gunicorn}(23155)
|-gunicorn(23124)---{gunicorn}(23154)
|-gunicorn(23125)---{gunicorn}(23152)
|-gunicorn(23126)---{gunicorn}(23197)
|-gunicorn(23127)---{gunicorn}(23153)
`-gunicorn(23128)---{gunicorn}(23198)
sudo strace -o backend-calls.txt -f -e 'trace=!epoll_pwait,select,fchmod,ioctl,geteuid,poll,openat,nanosleep,futex,setsockopt,getsockopt,epoll_ctl' -p 23123 -p 23124 -p 23125 -p 23126 -p 23127 -p 23128 -p 23128
sudo strace -o backend-calls.txt -f -e 'trace=!epoll_pwait,select,fchmod,ioctl,geteuid,openat,nanosleep,futex,setsockopt,getsockopt,epoll_ctl' -p 23123 -p 23124 -p 23125 -p 23126 -p 23127 -p 23128 -p 23128
sudo strace -o hammerdebug-calls.txt -f -e 'trace=!select,fchmod,ioctl,geteuid,nanosleep,futex,setsockopt,getsockopt,epoll_ctl' -p 30703
sudo strace -o hammermanager-calls.txt -f -e 'trace=!select,fchmod,ioctl,geteuid,nanosleep,futex,setsockopt,getsockopt,epoll_ctl' -p 22528 -p 22617 -p 22618 -p 22619 -p 22620 -p 22621 -p 22622
// bcc tools
pip3 install bcc
sudo yum install bcc-tools
tcptracer
tcplife
// tcptracer shows opening and closing of TCP connections
// tcptop summarizes throughput of TCP connections
sudo python3 tcptop.py | grep 5672 | grep hammer
// tcpdump inside container and get pcap and analyze with wireshark on local
> tcpdump -i eth0 src port 57036 -w capture.pcap
tcpdump -i eth0 -w capture.pcap
// see already established connections
// sudo conntrack -L -p tcp | grep 192.168.52.61 | grep 5672