Skip to content

Commit

Permalink
new(driver/modern_bpf): home-made bpf_loop for sendmmsg and recvmmsg.
Browse files Browse the repository at this point in the history 
With this one weird trick, bpf hates us!

Signed-off-by: Federico Di Pierro <[email protected]>
  • Loading branch information
@FedeDP
FedeDP committed Jan 15, 2025
1 parent d327a3b commit dd30172
Show file tree
Hide file tree
Showing 2 changed files with 169 additions and 18 deletions.
82 changes: 79 additions & 3 deletions driver/modern_bpf/programs/tail_called/events/syscall_dispatched_events/recvmmsg.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,91 @@ int BPF_PROG(recvmmsg_e, struct pt_regs *regs, long id) {

/*=============================== EXIT EVENT ===========================*/

static __always_inline long handle_exit(uint32_t index, void *ctx);

typedef struct {
uint32_t fd;
struct mmsghdr *mmh;
struct pt_regs *regs;
void *ctx;
} recvmmsg_data_t;

// This is some pre-processor magic (X_MACROs) to allow us to mimic `bpf_loop` behavior
// without using the helper, that triggers a verifier issue;
// See
// https://lore.kernel.org/bpf/CAGQdkDt9zyQwr5JyftXqL=OLKscNcqUtEteY4hvOkx2S4GdEkQ@mail.gmail.com/T/#u.

#define RECVMMSG_EXTRA_TAIL_CALLS \
X(0) \
X(1) \
X(2) \
X(3) \
X(4) \
X(5) \
X(6) \
X(7) \
X(8)

#define TAIL_CALL(ctx, value) \
bpf_tail_call(ctx, &extra_recvmmsg_calls, RECVMMSG_EXTRA_TAIL_CALL_##value)

enum extra_recvmmsg_codes {
#define X(value) RECVMMSG_EXTRA_TAIL_CALL_##value,
RECVMMSG_EXTRA_TAIL_CALLS
#undef X
RECVMMSG_EXTRA_TAIL_CALL_MAX
};

/*
* FORWARD DECLARATIONS:
* See the `BPF_PROG` macro in libbpf `libbpf/src/bpf_tracing.h`
* #define BPF_PROG(name, args...) \
* name(unsigned long long *ctx); \
*/
#define X(value) int recvmmsg_t_##value(unsigned long long *ctx);
RECVMMSG_EXTRA_TAIL_CALLS
#undef X

struct {
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
__uint(max_entries, RECVMMSG_EXTRA_TAIL_CALL_MAX);
__uint(key_size, sizeof(__u32));
__array(values, int(void *));
} extra_recvmmsg_calls SEC(".maps") = {
.values =
{
#define X(value) [value] = (void *)&recvmmsg_t_##value,
RECVMMSG_EXTRA_TAIL_CALLS
#undef X
},
};

#define X(value) \
SEC("tp_btf/sys_exit") \
int BPF_PROG(recvmmsg_t_##value, struct pt_regs *regs, long ret) { \
unsigned long args[2]; \
extract__network_args(args, 2, regs); \
recvmmsg_data_t data = { \
.fd = args[0], \
.mmh = (struct mmsghdr *)args[1], \
.regs = regs, \
.ctx = ctx, \
}; \
int i; \
int start = value * MAX_SENDMMSG_RECVMMSG_SIZE; \
for(i = start; i < ret && i < start + MAX_SENDMMSG_RECVMMSG_SIZE; i++) { \
handle_exit(i, &data); \
} \
if(i == ret) \
return 0; \
if(value + 1 == RECVMMSG_EXTRA_TAIL_CALL_MAX) \
return 0; \
TAIL_CALL(ctx, value + 1); \
return 0; \
}
RECVMMSG_EXTRA_TAIL_CALLS
#undef X

static __always_inline long handle_exit(uint32_t index, void *ctx) {
recvmmsg_data_t *data = (recvmmsg_data_t *)ctx;
struct mmsghdr mmh = {0};
Expand Down Expand Up @@ -166,9 +244,7 @@ int BPF_PROG(recvmmsg_x, struct pt_regs *regs, long ret) {
uint32_t nr_loops = ret < 1024 ? ret : 1024;
bpf_loop(nr_loops, handle_exit, &data, 0);
} else {*/
for(int i = 0; i < ret && i < MAX_SENDMMSG_RECVMMSG_SIZE; i++) {
handle_exit(i, &data);
}
TAIL_CALL(ctx, 0);
//}

return 0;
Expand Down
105 changes: 90 additions & 15 deletions driver/modern_bpf/programs/tail_called/events/syscall_dispatched_events/sendmmsg.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,93 @@ int BPF_PROG(sendmmsg_e, struct pt_regs *regs, long id) {

/*=============================== EXIT EVENT ===========================*/

static __always_inline long handle_exit(uint32_t index, void *ctx);

typedef struct {
uint32_t fd;
struct mmsghdr *mmh;
struct pt_regs *regs;
void *ctx;
} sendmmsg_exit_t;
} sendmmsg_data_t;

// This is some pre-processor magic (X_MACROs) to allow us to mimic `bpf_loop` behavior
// without using the helper, that triggers a verifier issue;
// See
// https://lore.kernel.org/bpf/CAGQdkDt9zyQwr5JyftXqL=OLKscNcqUtEteY4hvOkx2S4GdEkQ@mail.gmail.com/T/#u.

#define SENDMMSG_EXTRA_TAIL_CALLS \
X(0) \
X(1) \
X(2) \
X(3) \
X(4) \
X(5) \
X(6) \
X(7) \
X(8)

#define TAIL_CALL(ctx, value) \
bpf_tail_call(ctx, &extra_sendmmsg_calls, SENDMMSG_EXTRA_TAIL_CALL_##value)

enum extra_sendmmsg_codes {
#define X(value) SENDMMSG_EXTRA_TAIL_CALL_##value,
SENDMMSG_EXTRA_TAIL_CALLS
#undef X
SENDMMSG_EXTRA_TAIL_CALL_MAX
};

/*
* FORWARD DECLARATIONS:
* See the `BPF_PROG` macro in libbpf `libbpf/src/bpf_tracing.h`
* #define BPF_PROG(name, args...) \
* name(unsigned long long *ctx); \
*/
#define X(value) int sendmmsg_t_##value(unsigned long long *ctx);
SENDMMSG_EXTRA_TAIL_CALLS
#undef X

struct {
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
__uint(max_entries, SENDMMSG_EXTRA_TAIL_CALL_MAX);
__uint(key_size, sizeof(__u32));
__array(values, int(void *));
} extra_sendmmsg_calls SEC(".maps") = {
.values =
{
#define X(value) [value] = (void *)&sendmmsg_t_##value,
SENDMMSG_EXTRA_TAIL_CALLS
#undef X
},
};

#define X(value) \
SEC("tp_btf/sys_exit") \
int BPF_PROG(sendmmsg_t_##value, struct pt_regs *regs, long ret) { \
unsigned long args[2]; \
extract__network_args(args, 2, regs); \
sendmmsg_data_t data = { \
.fd = args[0], \
.mmh = (struct mmsghdr *)args[1], \
.regs = regs, \
.ctx = ctx, \
}; \
int i; \
int start = value * MAX_SENDMMSG_RECVMMSG_SIZE; \
for(i = start; i < ret && i < start + MAX_SENDMMSG_RECVMMSG_SIZE; i++) { \
handle_exit(i, &data); \
} \
if(i == ret) \
return 0; \
if(value + 1 == SENDMMSG_EXTRA_TAIL_CALL_MAX) \
return 0; \
TAIL_CALL(ctx, value + 1); \
return 0; \
}
SENDMMSG_EXTRA_TAIL_CALLS
#undef X

static __always_inline long handle_exit(uint32_t index, void *ctx) {
sendmmsg_exit_t *data = (sendmmsg_exit_t *)ctx;
sendmmsg_data_t *data = (sendmmsg_data_t *)ctx;
struct mmsghdr mmh = {0};

if(bpf_probe_read_user((void *)&mmh,
Expand Down Expand Up @@ -142,26 +220,23 @@ int BPF_PROG(sendmmsg_x, struct pt_regs *regs, long ret) {
return 0;
}

/* Collect parameters at the beginning to manage socketcalls */
unsigned long args[2];
extract__network_args(args, 2, regs);
sendmmsg_exit_t data = {
.fd = args[0],
.mmh = (struct mmsghdr *)args[1],
.regs = regs,
.ctx = ctx,
};

// We can't use bpf_loop() helper since the below check triggers a verifier failure:
// see
// https://lore.kernel.org/bpf/CAGQdkDt9zyQwr5JyftXqL=OLKscNcqUtEteY4hvOkx2S4GdEkQ@mail.gmail.com/T/#u
/*if(bpf_core_enum_value_exists(enum bpf_func_id, BPF_FUNC_loop)) {
// Collect parameters at the beginning to manage socketcalls
unsigned long args[2];
extract__network_args(args, 2, regs);
sendmmsg_exit_t data = {
.fd = args[0],
.mmh = (struct mmsghdr *)args[1],
.regs = regs,
.ctx = ctx,
};
uint32_t nr_loops = ret < 1024 ? ret : 1024;
bpf_loop(nr_loops, handle_exit, &data, 0);
} else {*/
for(int i = 0; i < ret && i < MAX_SENDMMSG_RECVMMSG_SIZE; i++) {
handle_exit(i, &data);
}
TAIL_CALL(ctx, 0);
//}

return 0;
Expand Down

0 comments on commit dd30172

Please sign in to comment.