tracing: probeevent: Return consumed bytes of dynamic area (9178412d) · Commits · jan.koester / Linux

kernel/trace/trace_kprobe.c

+27 −30

Original line number	Original line	Diff line number	Diff line
	@@ -853,8 +853,8 @@ static const struct file_operations kprobe_profile_ops = {
	/* Kprobe specific fetch functions */		/* Kprobe specific fetch functions */

	/* Return the length of string -- including null terminal byte */		/* Return the length of string -- including null terminal byte */
	static nokprobe_inline void		static nokprobe_inline int
	fetch_store_strlen(unsigned long addr, void *dest)		fetch_store_strlen(unsigned long addr)
	{		{
	mm_segment_t old_fs;		mm_segment_t old_fs;
	int ret, len = 0;		int ret, len = 0;
	@@ -872,47 +872,40 @@ fetch_store_strlen(unsigned long addr, void *dest)
	pagefault_enable();		pagefault_enable();
	set_fs(old_fs);		set_fs(old_fs);

	if (ret < 0) /* Failed to check the length */		return (ret < 0) ? ret : len;
	(u32 )dest = 0;
	else
	(u32 )dest = len;
	}		}

	/*		/*
	* Fetch a null-terminated string. Caller MUST set (u32 )buf with max		* Fetch a null-terminated string. Caller MUST set (u32 )buf with max
	* length and relative data location.		* length and relative data location.
	*/		*/
	static nokprobe_inline void		static nokprobe_inline int
	fetch_store_string(unsigned long addr, void *dest)		fetch_store_string(unsigned long addr, void dest, void base)
	{		{
	int maxlen = get_rloc_len((u32 )dest);		int maxlen = get_loc_len((u32 )dest);
	u8 *dst = get_rloc_data(dest);		u8 *dst = get_loc_data(dest, base);
	long ret;		long ret;

	if (!maxlen)		if (unlikely(!maxlen))
	return;		return -ENOMEM;

	/*		/*
	* Try to get string again, since the string can be changed while		* Try to get string again, since the string can be changed while
	* probing.		* probing.
	*/		*/
	ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);		ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);

	if (ret < 0) { /* Failed to fetch string */		if (ret >= 0)
	dst[0] = '\0';		(u32 )dest = make_data_loc(ret, (void *)dst - base);
	(u32 )dest = make_data_rloc(0, get_rloc_offs((u32 )dest));		return ret;
	} else {
	(u32 )dest = make_data_rloc(ret, get_rloc_offs((u32 )dest));
	}
	}		}

	/* Note that we don't verify it, since the code does not come from user space */		/* Note that we don't verify it, since the code does not come from user space */
	static int		static int
	process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,		process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	bool pre)		void *base)
	{		{
	unsigned long val;		unsigned long val;
	int ret;		int ret = 0;

	/* 1st stage: get value from context */		/* 1st stage: get value from context */
	switch (code->op) {		switch (code->op) {
	@@ -949,6 +942,13 @@ process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	}		}

	/* 3rd stage: store value to buffer */		/* 3rd stage: store value to buffer */
			if (unlikely(!dest)) {
			if (code->op == FETCH_OP_ST_STRING)
			return fetch_store_strlen(val + code->offset);
			else
			return -EILSEQ;
			}

	switch (code->op) {		switch (code->op) {
	case FETCH_OP_ST_RAW:		case FETCH_OP_ST_RAW:
	fetch_store_raw(val, code, dest);		fetch_store_raw(val, code, dest);
	@@ -957,10 +957,7 @@ process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	probe_kernel_read(dest, (void *)val + code->offset, code->size);		probe_kernel_read(dest, (void *)val + code->offset, code->size);
	break;		break;
	case FETCH_OP_ST_STRING:		case FETCH_OP_ST_STRING:
	if (pre)		ret = fetch_store_string(val + code->offset, dest, base);
	fetch_store_strlen(val + code->offset, dest);
	else
	fetch_store_string(val + code->offset, dest);
	break;		break;
	default:		default:
	return -EILSEQ;		return -EILSEQ;
	@@ -973,7 +970,7 @@ process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	code++;		code++;
	}		}

	return code->op == FETCH_OP_END ? 0 : -EILSEQ;		return code->op == FETCH_OP_END ? ret : -EILSEQ;
	}		}
	NOKPROBE_SYMBOL(process_fetch_insn)		NOKPROBE_SYMBOL(process_fetch_insn)

	@@ -1008,7 +1005,7 @@ __kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs,

	entry = ring_buffer_event_data(event);		entry = ring_buffer_event_data(event);
	entry->ip = (unsigned long)tk->rp.kp.addr;		entry->ip = (unsigned long)tk->rp.kp.addr;
	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);		store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);

	event_trigger_unlock_commit_regs(trace_file, buffer, event,		event_trigger_unlock_commit_regs(trace_file, buffer, event,
	entry, irq_flags, pc, regs);		entry, irq_flags, pc, regs);
	@@ -1057,7 +1054,7 @@ __kretprobe_trace_func(struct trace_kprobe tk, struct kretprobe_instance ri,
	entry = ring_buffer_event_data(event);		entry = ring_buffer_event_data(event);
	entry->func = (unsigned long)tk->rp.kp.addr;		entry->func = (unsigned long)tk->rp.kp.addr;
	entry->ret_ip = (unsigned long)ri->ret_addr;		entry->ret_ip = (unsigned long)ri->ret_addr;
	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);		store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);

	event_trigger_unlock_commit_regs(trace_file, buffer, event,		event_trigger_unlock_commit_regs(trace_file, buffer, event,
	entry, irq_flags, pc, regs);		entry, irq_flags, pc, regs);
	@@ -1203,7 +1200,7 @@ kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs)

	entry->ip = (unsigned long)tk->rp.kp.addr;		entry->ip = (unsigned long)tk->rp.kp.addr;
	memset(&entry[1], 0, dsize);		memset(&entry[1], 0, dsize);
	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);		store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,		perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
	head, NULL);		head, NULL);
	return 0;		return 0;
	@@ -1239,7 +1236,7 @@ kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri,

	entry->func = (unsigned long)tk->rp.kp.addr;		entry->func = (unsigned long)tk->rp.kp.addr;
	entry->ret_ip = (unsigned long)ri->ret_addr;		entry->ret_ip = (unsigned long)ri->ret_addr;
	store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);		store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,		perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
	head, NULL);		head, NULL);
	}		}

kernel/trace/trace_probe.h

+6 −20

Original line number	Original line	Diff line number	Diff line
	@@ -54,29 +54,15 @@
	#define TP_FLAG_PROFILE 2		#define TP_FLAG_PROFILE 2
	#define TP_FLAG_REGISTERED 4		#define TP_FLAG_REGISTERED 4

			/* data_loc: data location, compatible with u32 */
			#define make_data_loc(len, offs) \
			(((u32)(len) << 16) \| ((u32)(offs) & 0xffff))
			#define get_loc_len(dl) ((u32)(dl) >> 16)
			#define get_loc_offs(dl) ((u32)(dl) & 0xffff)

	/* data_rloc: data relative location, compatible with u32 */
	#define make_data_rloc(len, roffs) \
	(((u32)(len) << 16) \| ((u32)(roffs) & 0xffff))
	#define get_rloc_len(dl) ((u32)(dl) >> 16)
	#define get_rloc_offs(dl) ((u32)(dl) & 0xffff)

	/*
	* Convert data_rloc to data_loc:
	* data_rloc stores the offset from data_rloc itself, but data_loc
	* stores the offset from event entry.
	*/
	#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))

	static nokprobe_inline void get_rloc_data(u32 dl)
	{
	return (u8 )dl + get_rloc_offs(dl);
	}

	/* For data_loc conversion */
	static nokprobe_inline void get_loc_data(u32 dl, void *ent)		static nokprobe_inline void get_loc_data(u32 dl, void *ent)
	{		{
	return (u8 )ent + get_rloc_offs(dl);		return (u8 )ent + get_loc_offs(dl);
	}		}

	/* Printing function type */		/* Printing function type */

kernel/trace/trace_probe_tmpl.h

+25 −29

Original line number	Original line	Diff line number	Diff line
	@@ -48,23 +48,27 @@ fetch_apply_bitfield(struct fetch_insn code, void buf)
	}		}
	}		}

	/* Define this for each callsite */		/*
			* This must be defined for each callsite.
			* Return consumed dynamic data size (>= 0), or error (< 0).
			* If dest is NULL, don't store result and return required dynamic data size.
			*/
	static int		static int
	process_fetch_insn(struct fetch_insn code, struct pt_regs regs,		process_fetch_insn(struct fetch_insn code, struct pt_regs regs,
	void *dest, bool pre);		void dest, void base);

	/* Sum up total data length for dynamic arraies (strings) */		/* Sum up total data length for dynamic arraies (strings) */
	static nokprobe_inline int		static nokprobe_inline int
	__get_data_size(struct trace_probe tp, struct pt_regs regs)		__get_data_size(struct trace_probe tp, struct pt_regs regs)
	{		{
	struct probe_arg *arg;		struct probe_arg *arg;
	int i, ret = 0;		int i, len, ret = 0;
	u32 len;

	for (i = 0; i < tp->nr_args; i++) {		for (i = 0; i < tp->nr_args; i++) {
	arg = tp->args + i;		arg = tp->args + i;
	if (unlikely(arg->dynamic)) {		if (unlikely(arg->dynamic)) {
	process_fetch_insn(arg->code, regs, &len, true);		len = process_fetch_insn(arg->code, regs, NULL, NULL);
			if (len > 0)
	ret += len;		ret += len;
	}		}
	}		}
	@@ -74,34 +78,26 @@ __get_data_size(struct trace_probe tp, struct pt_regs regs)

	/* Store the value of each argument */		/* Store the value of each argument */
	static nokprobe_inline void		static nokprobe_inline void
	store_trace_args(int ent_size, struct trace_probe tp, struct pt_regs regs,		store_trace_args(void data, struct trace_probe tp, struct pt_regs *regs,
	u8 *data, int maxlen)		int header_size, int maxlen)
	{		{
	struct probe_arg *arg;		struct probe_arg *arg;
	u32 end = tp->size;		void *base = data - header_size;
	u32 dl; / Data (relative) location */		void *dyndata = data + tp->size;
	int i;		u32 dl; / Data location */
			int ret, i;

	for (i = 0; i < tp->nr_args; i++) {		for (i = 0; i < tp->nr_args; i++) {
	arg = tp->args + i;		arg = tp->args + i;
	if (unlikely(arg->dynamic)) {		dl = data + arg->offset;
	/*		/* Point the dynamic data area if needed */
	* First, we set the relative location and		if (unlikely(arg->dynamic))
	* maximum data length to *dl		*dl = make_data_loc(maxlen, dyndata - base);
	*/		ret = process_fetch_insn(arg->code, regs, dl, base);
	dl = (u32 *)(data + arg->offset);		if (unlikely(ret < 0 && arg->dynamic))
	*dl = make_data_rloc(maxlen, end - arg->offset);		*dl = make_data_loc(0, dyndata - base);
	/* Then try to fetch string or dynamic array data */		else
	process_fetch_insn(arg->code, regs, dl, false);		dyndata += ret;
	/* Reduce maximum length */
	end += get_rloc_len(*dl);
	maxlen -= get_rloc_len(*dl);
	/* Trick here, convert data_rloc to data_loc */
	dl = convert_rloc_to_loc(dl, ent_size + arg->offset);
	} else
	/* Just fetching data normally */
	process_fetch_insn(arg->code, regs, data + arg->offset,
	false);
	}		}
	}		}

kernel/trace/trace_uprobe.c

+30 −31

Original line number	Original line	Diff line number	Diff line
	@@ -111,43 +111,38 @@ probe_user_read(void dest, void src, size_t size)
	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max		* Fetch a null-terminated string. Caller MUST set (u32 )dest with max
	* length and relative data location.		* length and relative data location.
	*/		*/
	static nokprobe_inline void		static nokprobe_inline int
	fetch_store_string(unsigned long addr, void *dest)		fetch_store_string(unsigned long addr, void dest, void base)
	{		{
	long ret;		long ret;
	u32 rloc = (u32 )dest;		u32 loc = (u32 )dest;
	int maxlen = get_rloc_len(rloc);		int maxlen = get_loc_len(loc);
	u8 *dst = get_rloc_data(dest);		u8 *dst = get_loc_data(dest, base);
	void __user src = (void __force __user ) addr;		void __user src = (void __force __user ) addr;

	if (!maxlen)		if (unlikely(!maxlen))
	return;		return -ENOMEM;

	ret = strncpy_from_user(dst, src, maxlen);		ret = strncpy_from_user(dst, src, maxlen);
			if (ret >= 0) {
	if (ret == maxlen)		if (ret == maxlen)
	dst[--ret] = '\0';		dst[ret - 1] = '\0';
			(u32 )dest = make_data_loc(ret, (void *)dst - base);
	if (ret < 0) { /* Failed to fetch string */
	((u8 *)get_rloc_data(dest))[0] = '\0';
	(u32 )dest = make_data_rloc(0, get_rloc_offs(rloc));
	} else {
	(u32 )dest = make_data_rloc(ret, get_rloc_offs(rloc));
	}		}

			return ret;
	}		}

	/* Return the length of string -- including null terminal byte */		/* Return the length of string -- including null terminal byte */
	static nokprobe_inline void		static nokprobe_inline int
	fetch_store_strlen(unsigned long addr, void *dest)		fetch_store_strlen(unsigned long addr)
	{		{
	int len;		int len;
	void __user vaddr = (void __force __user ) addr;		void __user vaddr = (void __force __user ) addr;

	len = strnlen_user(vaddr, MAX_STRING_SIZE);		len = strnlen_user(vaddr, MAX_STRING_SIZE);

	if (len == 0 \|\| len > MAX_STRING_SIZE) /* Failed to check length */		return (len > MAX_STRING_SIZE) ? 0 : len;
	(u32 )dest = 0;
	else
	(u32 )dest = len;
	}		}

	static unsigned long translate_user_vaddr(unsigned long file_offset)		static unsigned long translate_user_vaddr(unsigned long file_offset)
	@@ -164,10 +159,10 @@ static unsigned long translate_user_vaddr(unsigned long file_offset)
	/* Note that we don't verify it, since the code does not come from user space */		/* Note that we don't verify it, since the code does not come from user space */
	static int		static int
	process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,		process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	bool pre)		void *base)
	{		{
	unsigned long val;		unsigned long val;
	int ret;		int ret = 0;

	/* 1st stage: get value from context */		/* 1st stage: get value from context */
	switch (code->op) {		switch (code->op) {
	@@ -204,18 +199,22 @@ process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	}		}

	/* 3rd stage: store value to buffer */		/* 3rd stage: store value to buffer */
			if (unlikely(!dest)) {
			if (code->op == FETCH_OP_ST_STRING)
			return fetch_store_strlen(val + code->offset);
			else
			return -EILSEQ;
			}

	switch (code->op) {		switch (code->op) {
	case FETCH_OP_ST_RAW:		case FETCH_OP_ST_RAW:
	fetch_store_raw(val, code, dest);		fetch_store_raw(val, code, dest);
	break;		break;
	case FETCH_OP_ST_MEM:		case FETCH_OP_ST_MEM:
	probe_user_read(dest, (void *)val + code->offset, code->size);		probe_kernel_read(dest, (void *)val + code->offset, code->size);
	break;		break;
	case FETCH_OP_ST_STRING:		case FETCH_OP_ST_STRING:
	if (pre)		ret = fetch_store_string(val + code->offset, dest, base);
	fetch_store_strlen(val + code->offset, dest);
	else
	fetch_store_string(val + code->offset, dest);
	break;		break;
	default:		default:
	return -EILSEQ;		return -EILSEQ;
	@@ -228,7 +227,7 @@ process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	code++;		code++;
	}		}

	return code->op == FETCH_OP_END ? 0 : -EILSEQ;		return code->op == FETCH_OP_END ? ret : -EILSEQ;
	}		}
	NOKPROBE_SYMBOL(process_fetch_insn)		NOKPROBE_SYMBOL(process_fetch_insn)

	@@ -1300,7 +1299,7 @@ static int uprobe_dispatcher(struct uprobe_consumer con, struct pt_regs regs)
	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));		esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));

	ucb = uprobe_buffer_get();		ucb = uprobe_buffer_get();
	store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);		store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);

	if (tu->tp.flags & TP_FLAG_TRACE)		if (tu->tp.flags & TP_FLAG_TRACE)
	ret \|= uprobe_trace_func(tu, regs, ucb, dsize);		ret \|= uprobe_trace_func(tu, regs, ucb, dsize);
	@@ -1335,7 +1334,7 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));		esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));

	ucb = uprobe_buffer_get();		ucb = uprobe_buffer_get();
	store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize);		store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);

	if (tu->tp.flags & TP_FLAG_TRACE)		if (tu->tp.flags & TP_FLAG_TRACE)
	uretprobe_trace_func(tu, func, regs, ucb, dsize);		uretprobe_trace_func(tu, func, regs, ucb, dsize);