Commit 5902da6d authored by Jiri Olsa's avatar Jiri Olsa Committed by Alexei Starovoitov
Browse files

libbpf: Add uprobe multi link support to bpf_program__attach_usdt 



Adding support for usdt_manager_attach_usdt to use uprobe_multi
link to attach to usdt probes.

The uprobe_multi support is detected before the usdt program is
loaded and its expected_attach_type is set accordingly.

If uprobe_multi support is detected the usdt_manager_attach_usdt
gathers uprobes info and calls bpf_program__attach_uprobe to
create all needed uprobes.

If uprobe_multi support is not detected the old behaviour stays.

Also adding usdt.s program section for sleepable usdt probes.

Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20230809083440.3209381-18-jolsa@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 7e1b4681

tools/lib/bpf/libbpf.c

+11 −2
Original line number Original line Diff line number Diff line
@@ -367,6 +367,8 @@ enum sec_def_flags { 
	SEC_SLEEPABLE = 8,

	SEC_SLEEPABLE = 8,

	/* BPF program support non-linear XDP buffer */

	/* BPF program support non-linear XDP buffer */

	SEC_XDP_FRAGS = 16,

	SEC_XDP_FRAGS = 16,

	/* Setup proper attach type for usdt probes. */

	SEC_USDT = 32,

};

};





struct bpf_sec_def {

struct bpf_sec_def {
@@ -6828,6 +6830,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, 
	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))

	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))

		opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;

		opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;





	/* special check for usdt to use uprobe_multi link */

	if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))

		prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;



	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {

	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {

		int btf_obj_fd = 0, btf_type_id = 0, err;

		int btf_obj_fd = 0, btf_type_id = 0, err;

		const char *attach_name;

		const char *attach_name;
@@ -6896,7 +6902,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog 
	if (!insns || !insns_cnt)

	if (!insns || !insns_cnt)

		return -EINVAL;

		return -EINVAL;





	load_attr.expected_attach_type = prog->expected_attach_type;

	if (kernel_supports(obj, FEAT_PROG_NAME))

	if (kernel_supports(obj, FEAT_PROG_NAME))

		prog_name = prog->name;

		prog_name = prog->name;

	load_attr.attach_prog_fd = prog->attach_prog_fd;

	load_attr.attach_prog_fd = prog->attach_prog_fd;
@@ -6932,6 +6937,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog 
		insns_cnt = prog->insns_cnt;

		insns_cnt = prog->insns_cnt;

	}

	}





	/* allow prog_prepare_load_fn to change expected_attach_type */

	load_attr.expected_attach_type = prog->expected_attach_type;



	if (obj->gen_loader) {

	if (obj->gen_loader) {

		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,

		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,

				   license, insns, insns_cnt, &load_attr,

				   license, insns, insns_cnt, &load_attr,
@@ -8759,7 +8767,8 @@ static const struct bpf_sec_def section_defs[] = { 
	SEC_DEF("uretprobe.multi.s+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),

	SEC_DEF("uretprobe.multi.s+",	KPROBE,	BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),

	SEC_DEF("ksyscall+",		KPROBE,	0, SEC_NONE, attach_ksyscall),

	SEC_DEF("ksyscall+",		KPROBE,	0, SEC_NONE, attach_ksyscall),

	SEC_DEF("kretsyscall+",		KPROBE, 0, SEC_NONE, attach_ksyscall),

	SEC_DEF("kretsyscall+",		KPROBE, 0, SEC_NONE, attach_ksyscall),

	SEC_DEF("usdt+",		KPROBE,	0, SEC_NONE, attach_usdt),

	SEC_DEF("usdt+",		KPROBE,	0, SEC_USDT, attach_usdt),

	SEC_DEF("usdt.s+",		KPROBE,	0, SEC_USDT | SEC_SLEEPABLE, attach_usdt),

	SEC_DEF("tc/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */

	SEC_DEF("tc/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */

	SEC_DEF("tc/egress",		SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),  /* alias for tcx */

	SEC_DEF("tc/egress",		SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),  /* alias for tcx */

	SEC_DEF("tcx/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),

	SEC_DEF("tcx/ingress",		SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),

tools/lib/bpf/usdt.c

+71 −15
Original line number Original line Diff line number Diff line
@@ -250,6 +250,7 @@ struct usdt_manager { 




	bool has_bpf_cookie;

	bool has_bpf_cookie;

	bool has_sema_refcnt;

	bool has_sema_refcnt;

	bool has_uprobe_multi;

};

};





struct usdt_manager *usdt_manager_new(struct bpf_object *obj)

struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
@@ -284,6 +285,11 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) 
	 */

	 */

	man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;

	man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;





	/*

	 * Detect kernel support for uprobe multi link to be used for attaching

	 * usdt probes.

	 */

	man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);

	return man;

	return man;

}

}
@@ -808,6 +814,8 @@ struct bpf_link_usdt { 
		long abs_ip;

		long abs_ip;

		struct bpf_link *link;

		struct bpf_link *link;

	} *uprobes;

	} *uprobes;



	struct bpf_link *multi_link;

};

};





static int bpf_link_usdt_detach(struct bpf_link *link)

static int bpf_link_usdt_detach(struct bpf_link *link)
@@ -816,6 +824,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link) 
	struct usdt_manager *man = usdt_link->usdt_man;

	struct usdt_manager *man = usdt_link->usdt_man;

	int i;

	int i;





	bpf_link__destroy(usdt_link->multi_link);



	/* When having multi_link, uprobe_cnt is 0 */

	for (i = 0; i < usdt_link->uprobe_cnt; i++) {

	for (i = 0; i < usdt_link->uprobe_cnt; i++) {

		/* detach underlying uprobe link */

		/* detach underlying uprobe link */

		bpf_link__destroy(usdt_link->uprobes[i].link);

		bpf_link__destroy(usdt_link->uprobes[i].link);
@@ -946,11 +957,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct 
					  const char *usdt_provider, const char *usdt_name,

					  const char *usdt_provider, const char *usdt_name,

					  __u64 usdt_cookie)

					  __u64 usdt_cookie)

{

{

	unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;

	int i, err, spec_map_fd, ip_map_fd;

	int i, err, spec_map_fd, ip_map_fd;

	LIBBPF_OPTS(bpf_uprobe_opts, opts);

	LIBBPF_OPTS(bpf_uprobe_opts, opts);

	struct hashmap *specs_hash = NULL;

	struct hashmap *specs_hash = NULL;

	struct bpf_link_usdt *link = NULL;

	struct bpf_link_usdt *link = NULL;

	struct usdt_target *targets = NULL;

	struct usdt_target *targets = NULL;

	__u64 *cookies = NULL;

	struct elf_fd elf_fd;

	struct elf_fd elf_fd;

	size_t target_cnt;

	size_t target_cnt;
@@ -997,11 +1010,22 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct 
	link->link.detach = &bpf_link_usdt_detach;

	link->link.detach = &bpf_link_usdt_detach;

	link->link.dealloc = &bpf_link_usdt_dealloc;

	link->link.dealloc = &bpf_link_usdt_dealloc;





	if (man->has_uprobe_multi) {

		offsets = calloc(target_cnt, sizeof(*offsets));

		cookies = calloc(target_cnt, sizeof(*cookies));

		ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));



		if (!offsets || !ref_ctr_offsets || !cookies) {

			err = -ENOMEM;

			goto err_out;

		}

	} else {

		link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));

		link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));

		if (!link->uprobes) {

		if (!link->uprobes) {

			err = -ENOMEM;

			err = -ENOMEM;

			goto err_out;

			goto err_out;

		}

		}

	}





	for (i = 0; i < target_cnt; i++) {

	for (i = 0; i < target_cnt; i++) {

		struct usdt_target *target = &targets[i];

		struct usdt_target *target = &targets[i];
@@ -1041,6 +1065,11 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct 
			goto err_out;

			goto err_out;

		}

		}





		if (man->has_uprobe_multi) {

			offsets[i] = target->rel_ip;

			ref_ctr_offsets[i] = target->sema_off;

			cookies[i] = spec_id;

		} else {

			opts.ref_ctr_offset = target->sema_off;

			opts.ref_ctr_offset = target->sema_off;

			opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;

			opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;

			uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,

			uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
@@ -1056,6 +1085,29 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct 
			link->uprobes[i].abs_ip = target->abs_ip;

			link->uprobes[i].abs_ip = target->abs_ip;

			link->uprobe_cnt++;

			link->uprobe_cnt++;

		}

		}

	}



	if (man->has_uprobe_multi) {

		LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,

			.ref_ctr_offsets = ref_ctr_offsets,

			.offsets = offsets,

			.cookies = cookies,

			.cnt = target_cnt,

		);



		link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,

								    NULL, &opts_multi);

		if (!link->multi_link) {

			err = -errno;

			pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",

				usdt_provider, usdt_name, path, err);

			goto err_out;

		}



		free(offsets);

		free(ref_ctr_offsets);

		free(cookies);

	}





	free(targets);

	free(targets);

	hashmap__free(specs_hash);

	hashmap__free(specs_hash);
@@ -1063,6 +1115,10 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct 
	return &link->link;

	return &link->link;





err_out:

err_out:

	free(offsets);

	free(ref_ctr_offsets);

	free(cookies);



	if (link)

	if (link)

		bpf_link__destroy(&link->link);

		bpf_link__destroy(&link->link);

	free(targets);

	free(targets);