summaryrefslogtreecommitdiffstats
path: root/tools/lib/bpf/libbpf.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib/bpf/libbpf.c')
-rw-r--r--tools/lib/bpf/libbpf.c252
1 files changed, 219 insertions, 33 deletions
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index f515cf264a..5401f2df46 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -132,6 +132,7 @@ static const char * const attach_type_name[] = {
[BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi",
[BPF_NETKIT_PRIMARY] = "netkit_primary",
[BPF_NETKIT_PEER] = "netkit_peer",
+ [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
};
static const char * const link_type_name[] = {
@@ -149,6 +150,7 @@ static const char * const link_type_name[] = {
[BPF_LINK_TYPE_TCX] = "tcx",
[BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi",
[BPF_LINK_TYPE_NETKIT] = "netkit",
+ [BPF_LINK_TYPE_SOCKMAP] = "sockmap",
};
static const char * const map_type_name[] = {
@@ -1126,17 +1128,46 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
const struct btf_type *mtype, *kern_mtype;
__u32 mtype_id, kern_mtype_id;
void *mdata, *kern_mdata;
+ struct bpf_program *prog;
__s64 msize, kern_msize;
__u32 moff, kern_moff;
__u32 kern_member_idx;
const char *mname;
mname = btf__name_by_offset(btf, member->name_off);
+ moff = member->offset / 8;
+ mdata = data + moff;
+ msize = btf__resolve_size(btf, member->type);
+ if (msize < 0) {
+ pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
+ map->name, mname);
+ return msize;
+ }
+
kern_member = find_member_by_name(kern_btf, kern_type, mname);
if (!kern_member) {
- pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
+ if (!libbpf_is_mem_zeroed(mdata, msize)) {
+ pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
+ map->name, mname);
+ return -ENOTSUP;
+ }
+
+ if (st_ops->progs[i]) {
+ /* If we had declaratively set struct_ops callback, we need to
+ * force its autoload to false, because it doesn't have
+ * a chance of succeeding from POV of the current struct_ops map.
+ * If this program is still referenced somewhere else, though,
+ * then bpf_object_adjust_struct_ops_autoload() will update its
+ * autoload accordingly.
+ */
+ st_ops->progs[i]->autoload = false;
+ st_ops->progs[i] = NULL;
+ }
+
+ /* Skip all-zero/NULL fields if they are not present in the kernel BTF */
+ pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
map->name, mname);
- return -ENOTSUP;
+ continue;
}
kern_member_idx = kern_member - btf_members(kern_type);
@@ -1147,10 +1178,7 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
return -ENOTSUP;
}
- moff = member->offset / 8;
kern_moff = kern_member->offset / 8;
-
- mdata = data + moff;
kern_mdata = kern_data + kern_moff;
mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
@@ -1165,13 +1193,19 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
}
if (btf_is_ptr(mtype)) {
- struct bpf_program *prog;
+ prog = *(void **)mdata;
+ /* just like for !kern_member case above, reset declaratively
+ * set (at compile time) program's autload to false,
+ * if user replaced it with another program or NULL
+ */
+ if (st_ops->progs[i] && st_ops->progs[i] != prog)
+ st_ops->progs[i]->autoload = false;
/* Update the value from the shadow type */
- prog = *(void **)mdata;
st_ops->progs[i] = prog;
if (!prog)
continue;
+
if (!is_valid_st_ops_program(obj, prog)) {
pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n",
map->name, mname);
@@ -1230,9 +1264,8 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
continue;
}
- msize = btf__resolve_size(btf, mtype_id);
kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
- if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
+ if (kern_msize < 0 || msize != kern_msize) {
pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
map->name, mname, (ssize_t)msize,
(ssize_t)kern_msize);
@@ -1956,6 +1989,20 @@ static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
return NULL;
}
+static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj,
+ const void *name, int len)
+{
+ const char *ext_name;
+ int i;
+
+ for (i = 0; i < obj->nr_extern; i++) {
+ ext_name = obj->externs[i].name;
+ if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0)
+ return &obj->externs[i];
+ }
+ return NULL;
+}
+
static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
char value)
{
@@ -7325,7 +7372,11 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
__u32 log_level = prog->log_level;
int ret, err;
- if (prog->type == BPF_PROG_TYPE_UNSPEC) {
+ /* Be more helpful by rejecting programs that can't be validated early
+ * with more meaningful and actionable error message.
+ */
+ switch (prog->type) {
+ case BPF_PROG_TYPE_UNSPEC:
/*
* The program type must be set. Most likely we couldn't find a proper
* section definition at load time, and thus we didn't infer the type.
@@ -7333,6 +7384,15 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
prog->name, prog->sec_name);
return -EINVAL;
+ case BPF_PROG_TYPE_STRUCT_OPS:
+ if (prog->attach_btf_id == 0) {
+ pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n",
+ prog->name);
+ return -EINVAL;
+ }
+ break;
+ default:
+ break;
}
if (!insns || !insns_cnt)
@@ -7972,7 +8032,10 @@ static int bpf_object__sanitize_maps(struct bpf_object *obj)
return 0;
}
-int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
+typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
+ const char *sym_name, void *ctx);
+
+static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
{
char sym_type, sym_name[500];
unsigned long long sym_addr;
@@ -8012,8 +8075,13 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
struct bpf_object *obj = ctx;
const struct btf_type *t;
struct extern_desc *ext;
+ char *res;
- ext = find_extern_by_name(obj, sym_name);
+ res = strstr(sym_name, ".llvm.");
+ if (sym_type == 'd' && res)
+ ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);
+ else
+ ext = find_extern_by_name(obj, sym_name);
if (!ext || ext->type != EXT_KSYM)
return 0;
@@ -8562,6 +8630,11 @@ int bpf_map__pin(struct bpf_map *map, const char *path)
return libbpf_err(-EINVAL);
}
+ if (map->fd < 0) {
+ pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
+ return libbpf_err(-EINVAL);
+ }
+
if (map->pin_path) {
if (path && strcmp(path, map->pin_path)) {
pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
@@ -9230,6 +9303,7 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_lin
static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
+static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
@@ -9246,6 +9320,7 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
+ SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
@@ -9297,6 +9372,7 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
+ SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),
SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
@@ -9815,16 +9891,28 @@ static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
enum bpf_attach_type attach_type,
int *btf_obj_fd, int *btf_type_id)
{
- int ret, i;
+ int ret, i, mod_len;
+ const char *fn_name, *mod_name = NULL;
- ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
- if (ret > 0) {
- *btf_obj_fd = 0; /* vmlinux BTF */
- *btf_type_id = ret;
- return 0;
+ fn_name = strchr(attach_name, ':');
+ if (fn_name) {
+ mod_name = attach_name;
+ mod_len = fn_name - mod_name;
+ fn_name++;
+ }
+
+ if (!mod_name || strncmp(mod_name, "vmlinux", mod_len) == 0) {
+ ret = find_attach_btf_id(obj->btf_vmlinux,
+ mod_name ? fn_name : attach_name,
+ attach_type);
+ if (ret > 0) {
+ *btf_obj_fd = 0; /* vmlinux BTF */
+ *btf_type_id = ret;
+ return 0;
+ }
+ if (ret != -ENOENT)
+ return ret;
}
- if (ret != -ENOENT)
- return ret;
ret = load_module_btfs(obj);
if (ret)
@@ -9833,7 +9921,12 @@ static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
for (i = 0; i < obj->btf_module_cnt; i++) {
const struct module_btf *mod = &obj->btf_modules[i];
- ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
+ if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)
+ continue;
+
+ ret = find_attach_btf_id(mod->btf,
+ mod_name ? fn_name : attach_name,
+ attach_type);
if (ret > 0) {
*btf_obj_fd = mod->fd;
*btf_type_id = ret;
@@ -10306,6 +10399,11 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,
return -EINVAL;
}
+ if (map->fd < 0) {
+ pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
+ return -EINVAL;
+ }
+
if (!check_value_sz)
return 0;
@@ -10418,8 +10516,15 @@ long libbpf_get_error(const void *ptr)
int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
{
int ret;
+ int prog_fd = bpf_program__fd(prog);
+
+ if (prog_fd < 0) {
+ pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
+ prog->name);
+ return libbpf_err(-EINVAL);
+ }
- ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
+ ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);
return libbpf_err_errno(ret);
}
@@ -10613,7 +10718,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p
}
prog_fd = bpf_program__fd(prog);
if (prog_fd < 0) {
- pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
+ pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
prog->name);
return libbpf_err_ptr(-EINVAL);
}
@@ -11325,18 +11430,26 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
struct kprobe_multi_resolve res = {
.pattern = pattern,
};
+ enum bpf_attach_type attach_type;
struct bpf_link *link = NULL;
char errmsg[STRERR_BUFSIZE];
const unsigned long *addrs;
int err, link_fd, prog_fd;
+ bool retprobe, session;
const __u64 *cookies;
const char **syms;
- bool retprobe;
size_t cnt;
if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
return libbpf_err_ptr(-EINVAL);
+ prog_fd = bpf_program__fd(prog);
+ if (prog_fd < 0) {
+ pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
syms = OPTS_GET(opts, syms, false);
addrs = OPTS_GET(opts, addrs, false);
cnt = OPTS_GET(opts, cnt, false);
@@ -11363,6 +11476,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
}
retprobe = OPTS_GET(opts, retprobe, false);
+ session = OPTS_GET(opts, session, false);
+
+ if (retprobe && session)
+ return libbpf_err_ptr(-EINVAL);
+
+ attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI;
lopts.kprobe_multi.syms = syms;
lopts.kprobe_multi.addrs = addrs;
@@ -11377,8 +11496,7 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
}
link->detach = &bpf_link__detach_fd;
- prog_fd = bpf_program__fd(prog);
- link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
+ link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
if (link_fd < 0) {
err = -errno;
pr_warn("prog '%s': failed to attach: %s\n",
@@ -11484,6 +11602,32 @@ static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, stru
return libbpf_get_error(*link);
}
+static int attach_kprobe_session(const struct bpf_program *prog, long cookie,
+ struct bpf_link **link)
+{
+ LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true);
+ const char *spec;
+ char *pattern;
+ int n;
+
+ *link = NULL;
+
+ /* no auto-attach for SEC("kprobe.session") */
+ if (strcmp(prog->sec_name, "kprobe.session") == 0)
+ return 0;
+
+ spec = prog->sec_name + sizeof("kprobe.session/") - 1;
+ n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
+ if (n < 1) {
+ pr_warn("kprobe session pattern is invalid: %s\n", spec);
+ return -EINVAL;
+ }
+
+ *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
+ free(pattern);
+ return *link ? 0 : -errno;
+}
+
static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
{
char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
@@ -11760,6 +11904,13 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
return libbpf_err_ptr(-EINVAL);
+ prog_fd = bpf_program__fd(prog);
+ if (prog_fd < 0) {
+ pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
syms = OPTS_GET(opts, syms, NULL);
offsets = OPTS_GET(opts, offsets, NULL);
ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
@@ -11835,7 +11986,6 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
}
link->detach = &bpf_link__detach_fd;
- prog_fd = bpf_program__fd(prog);
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
if (link_fd < 0) {
err = -errno;
@@ -12079,7 +12229,7 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
return libbpf_err_ptr(-EINVAL);
if (bpf_program__fd(prog) < 0) {
- pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
+ pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
prog->name);
return libbpf_err_ptr(-EINVAL);
}
@@ -12270,13 +12420,19 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_lin
return libbpf_get_error(*link);
}
-struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
- const char *tp_name)
+struct bpf_link *
+bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
+ const char *tp_name,
+ struct bpf_raw_tracepoint_opts *opts)
{
+ LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
char errmsg[STRERR_BUFSIZE];
struct bpf_link *link;
int prog_fd, pfd;
+ if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))
+ return libbpf_err_ptr(-EINVAL);
+
prog_fd = bpf_program__fd(prog);
if (prog_fd < 0) {
pr_warn("prog '%s': can't attach before loaded\n", prog->name);
@@ -12288,7 +12444,9 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr
return libbpf_err_ptr(-ENOMEM);
link->detach = &bpf_link__detach_fd;
- pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
+ raw_opts.tp_name = tp_name;
+ raw_opts.cookie = OPTS_GET(opts, cookie, 0);
+ pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);
if (pfd < 0) {
pfd = -errno;
free(link);
@@ -12300,6 +12458,12 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr
return link;
}
+struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
+ const char *tp_name)
+{
+ return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
+}
+
static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
{
static const char *const prefixes[] = {
@@ -12453,6 +12617,12 @@ bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
}
+struct bpf_link *
+bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd)
+{
+ return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL);
+}
+
struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
{
/* target_fd/target_ifindex use the same field in LINK_CREATE */
@@ -12661,6 +12831,12 @@ struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
return libbpf_err_ptr(-EOPNOTSUPP);
+ if (bpf_program__fd(prog) < 0) {
+ pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
if (err)
return libbpf_err_ptr(err);
@@ -12701,8 +12877,13 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
__u32 zero = 0;
int err, fd;
- if (!bpf_map__is_struct_ops(map) || map->fd == -1)
+ if (!bpf_map__is_struct_ops(map))
+ return libbpf_err_ptr(-EINVAL);
+
+ if (map->fd < 0) {
+ pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
return libbpf_err_ptr(-EINVAL);
+ }
link = calloc(1, sizeof(*link));
if (!link)
@@ -12750,9 +12931,14 @@ int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map)
__u32 zero = 0;
int err;
- if (!bpf_map__is_struct_ops(map) || !map_is_created(map))
+ if (!bpf_map__is_struct_ops(map))
return -EINVAL;
+ if (map->fd < 0) {
+ pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
+ return -EINVAL;
+ }
+
st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
/* Ensure the type of a link is correct */
if (st_ops_link->map_fd < 0)
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-19 16:11:08 +0000