Commit 86087383 authored by Steven Rostedt (Google)'s avatar Steven Rostedt (Google)
Browse files

tracing/hist: Call hist functions directly via a switch statement 

Due to retpolines, indirect calls are much more expensive than direct
calls. The histograms have a select set of functions it uses for the
histograms, instead of using function pointers to call them, create a
hist_fn_call() function that uses a switch statement to call the histogram
functions directly. This gives a 13% speedup to the histogram logic.

Using the histogram benchmark:

Before:

 # event histogram
 #
 # trigger info: hist:keys=delta:vals=hitcount:sort=delta:size=2048 if delta > 0 [active]
 #

{ delta:        129 } hitcount:       2213
{ delta:        130 } hitcount:     285965
{ delta:        131 } hitcount:    1146545
{ delta:        132 } hitcount:    51854322
{ delta:        133 } hitcount:   19896215
{ delta:        134 } hitcount:   53118616
{ delta:        135 } hitcount:   83816709
{ delta:        136 } hitcount:   68329562
{ delta:        137 } hitcount:   41859349
{ delta:        138 } hitcount:   46257797
{ delta:        139 } hitcount:   54400831
{ delta:        140 } hitcount:   72875007
{ delta:        141 } hitcount:   76193272
{ delta:        142 } hitcount:   49504263
{ delta:        143 } hitcount:   38821072
{ delta:        144 } hitcount:   47702679
{ delta:        145 } hitcount:   41357297
{ delta:        146 } hitcount:   22058238
{ delta:        147 } hitcount:    9720002
{ delta:        148 } hitcount:    3193542
{ delta:        149 } hitcount:     927030
{ delta:        150 } hitcount:     850772
{ delta:        151 } hitcount:    1477380
{ delta:        152 } hitcount:    2687977
{ delta:        153 } hitcount:    2865985
{ delta:        154 } hitcount:    1977492
{ delta:        155 } hitcount:    2475607
{ delta:        156 } hitcount:    3403612

After:

 # event histogram
 #
 # trigger info: hist:keys=delta:vals=hitcount:sort=delta:size=2048 if delta > 0 [active]
 #

{ delta:        113 } hitcount:        272
{ delta:        114 } hitcount:        840
{ delta:        118 } hitcount:        344
{ delta:        119 } hitcount:      25428
{ delta:        120 } hitcount:     350590
{ delta:        121 } hitcount:    1892484
{ delta:        122 } hitcount:    6205004
{ delta:        123 } hitcount:   11583521
{ delta:        124 } hitcount:   37590979
{ delta:        125 } hitcount:  108308504
{ delta:        126 } hitcount:  131672461
{ delta:        127 } hitcount:   88700598
{ delta:        128 } hitcount:   65939870
{ delta:        129 } hitcount:   45055004
{ delta:        130 } hitcount:   33174464
{ delta:        131 } hitcount:   31813493
{ delta:        132 } hitcount:   29011676
{ delta:        133 } hitcount:   22798782
{ delta:        134 } hitcount:   22072486
{ delta:        135 } hitcount:   17034113
{ delta:        136 } hitcount:    8982490
{ delta:        137 } hitcount:    2865908
{ delta:        138 } hitcount:     980382
{ delta:        139 } hitcount:    1651944
{ delta:        140 } hitcount:    4112073
{ delta:        141 } hitcount:    3963269
{ delta:        142 } hitcount:    1712508
{ delta:        143 } hitcount:     575941
{ delta:        144 } hitcount:     351427
{ delta:        145 } hitcount:     218077
{ delta:        146 } hitcount:     167297
{ delta:        147 } hitcount:     146198
{ delta:        148 } hitcount:     116122
{ delta:        149 } hitcount:      58993
{ delta:        150 } hitcount:      40228

The delta above is in nanoseconds. It brings the fastest time down from
129ns to 113ns, and the peak from 141ns to 126ns.

Link: https://lkml.kernel.org/r/20220906225529.411545333@goodmis.org



Cc: Ingo Molnar <mingo@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Tom Zanussi <zanussi@kernel.org>
Signed-off-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
parent b7b037eb

kernel/trace/trace_events_hist.c

+169 −77
Original line number Original line Diff line number Diff line
@@ -104,6 +104,38 @@ enum field_op_id { 
	FIELD_OP_MULT,

	FIELD_OP_MULT,

};

};





enum hist_field_fn {

	HIST_FIELD_FN_NOP,

	HIST_FIELD_FN_VAR_REF,

	HIST_FIELD_FN_COUNTER,

	HIST_FIELD_FN_CONST,

	HIST_FIELD_FN_LOG2,

	HIST_FIELD_FN_BUCKET,

	HIST_FIELD_FN_TIMESTAMP,

	HIST_FIELD_FN_CPU,

	HIST_FIELD_FN_STRING,

	HIST_FIELD_FN_DYNSTRING,

	HIST_FIELD_FN_RELDYNSTRING,

	HIST_FIELD_FN_PSTRING,

	HIST_FIELD_FN_S64,

	HIST_FIELD_FN_U64,

	HIST_FIELD_FN_S32,

	HIST_FIELD_FN_U32,

	HIST_FIELD_FN_S16,

	HIST_FIELD_FN_U16,

	HIST_FIELD_FN_S8,

	HIST_FIELD_FN_U8,

	HIST_FIELD_FN_UMINUS,

	HIST_FIELD_FN_MINUS,

	HIST_FIELD_FN_PLUS,

	HIST_FIELD_FN_DIV,

	HIST_FIELD_FN_MULT,

	HIST_FIELD_FN_DIV_POWER2,

	HIST_FIELD_FN_DIV_NOT_POWER2,

	HIST_FIELD_FN_DIV_MULT_SHIFT,

	HIST_FIELD_FN_EXECNAME,

};



/*

/*

 * A hist_var (histogram variable) contains variable information for

 * A hist_var (histogram variable) contains variable information for

 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF

 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
@@ -123,15 +155,15 @@ struct hist_var { 
struct hist_field {

struct hist_field {

	struct ftrace_event_field	*field;

	struct ftrace_event_field	*field;

	unsigned long			flags;

	unsigned long			flags;

	hist_field_fn_t			fn;

	unsigned int			ref;

	unsigned int			size;

	unsigned int			offset;

	unsigned int                    is_signed;

	unsigned long			buckets;

	unsigned long			buckets;

	const char			*type;

	const char			*type;

	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];

	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];

	struct hist_trigger_data	*hist_data;

	struct hist_trigger_data	*hist_data;

	enum hist_field_fn		fn_num;

	unsigned int			ref;

	unsigned int			size;

	unsigned int			offset;

	unsigned int                    is_signed;





	/*

	/*

	 * Variable fields contain variable-specific info in var.

	 * Variable fields contain variable-specific info in var.
@@ -166,14 +198,11 @@ struct hist_field { 
	u64				div_multiplier;

	u64				div_multiplier;

};

};





static u64 hist_field_none(struct hist_field *field,

static u64 hist_fn_call(struct hist_field *hist_field,

			struct tracing_map_elt *elt,

			struct tracing_map_elt *elt,

			struct trace_buffer *buffer,

			struct trace_buffer *buffer,

			struct ring_buffer_event *rbe,

			struct ring_buffer_event *rbe,

			   void *event)

			void *event);

{

	return 0;

}





static u64 hist_field_const(struct hist_field *field,

static u64 hist_field_const(struct hist_field *field,

			   struct tracing_map_elt *elt,

			   struct tracing_map_elt *elt,
@@ -250,7 +279,7 @@ static u64 hist_field_log2(struct hist_field *hist_field, 
{

{

	struct hist_field *operand = hist_field->operands[0];

	struct hist_field *operand = hist_field->operands[0];





	u64 val = operand->fn(operand, elt, buffer, rbe, event);

	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);





	return (u64) ilog2(roundup_pow_of_two(val));

	return (u64) ilog2(roundup_pow_of_two(val));

}

}
@@ -264,7 +293,7 @@ static u64 hist_field_bucket(struct hist_field *hist_field, 
	struct hist_field *operand = hist_field->operands[0];

	struct hist_field *operand = hist_field->operands[0];

	unsigned long buckets = hist_field->buckets;

	unsigned long buckets = hist_field->buckets;





	u64 val = operand->fn(operand, elt, buffer, rbe, event);

	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);





	if (WARN_ON_ONCE(!buckets))

	if (WARN_ON_ONCE(!buckets))

		return val;

		return val;
@@ -285,8 +314,8 @@ static u64 hist_field_plus(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);

	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);

	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);





	return val1 + val2;

	return val1 + val2;

}

}
@@ -300,8 +329,8 @@ static u64 hist_field_minus(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);

	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);

	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);





	return val1 - val2;

	return val1 - val2;

}

}
@@ -315,8 +344,8 @@ static u64 hist_field_div(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);

	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);

	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);





	/* Return -1 for the undefined case */

	/* Return -1 for the undefined case */

	if (!val2)

	if (!val2)
@@ -338,7 +367,7 @@ static u64 div_by_power_of_two(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);





	return val1 >> __ffs64(operand2->constant);

	return val1 >> __ffs64(operand2->constant);

}

}
@@ -352,7 +381,7 @@ static u64 div_by_not_power_of_two(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);





	return div64_u64(val1, operand2->constant);

	return div64_u64(val1, operand2->constant);

}

}
@@ -366,7 +395,7 @@ static u64 div_by_mult_and_shift(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);





	/*

	/*

	 * If the divisor is a constant, do a multiplication and shift instead.

	 * If the divisor is a constant, do a multiplication and shift instead.
@@ -400,8 +429,8 @@ static u64 hist_field_mult(struct hist_field *hist_field, 
	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand1 = hist_field->operands[0];

	struct hist_field *operand2 = hist_field->operands[1];

	struct hist_field *operand2 = hist_field->operands[1];





	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);

	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);

	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);

	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);





	return val1 * val2;

	return val1 * val2;

}

}
@@ -414,7 +443,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field, 
{

{

	struct hist_field *operand = hist_field->operands[0];

	struct hist_field *operand = hist_field->operands[0];





	s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);

	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);

	u64 val = (u64)-sval;

	u64 val = (u64)-sval;





	return val;

	return val;
@@ -657,19 +686,19 @@ struct snapshot_context { 
 * Returns the specific division function to use if the divisor

 * Returns the specific division function to use if the divisor

 * is constant. This avoids extra branches when the trigger is hit.

 * is constant. This avoids extra branches when the trigger is hit.

 */

 */

static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)

static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)

{

{

	u64 div = divisor->constant;

	u64 div = divisor->constant;





	if (!(div & (div - 1)))

	if (!(div & (div - 1)))

		return div_by_power_of_two;

		return HIST_FIELD_FN_DIV_POWER2;





	/* If the divisor is too large, do a regular division */

	/* If the divisor is too large, do a regular division */

	if (div > (1 << HIST_DIV_SHIFT))

	if (div > (1 << HIST_DIV_SHIFT))

		return div_by_not_power_of_two;

		return HIST_FIELD_FN_DIV_NOT_POWER2;





	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);

	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);

	return div_by_mult_and_shift;

	return HIST_FIELD_FN_DIV_MULT_SHIFT;

}

}





static void track_data_free(struct track_data *track_data)

static void track_data_free(struct track_data *track_data)
@@ -1334,38 +1363,32 @@ static const char *hist_field_name(struct hist_field *field, 
	return field_name;

	return field_name;

}

}





static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)

static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)

{

{

	hist_field_fn_t fn = NULL;



	switch (field_size) {

	switch (field_size) {

	case 8:

	case 8:

		if (field_is_signed)

		if (field_is_signed)

			fn = hist_field_s64;

			return HIST_FIELD_FN_S64;

		else

		else

			fn = hist_field_u64;

			return HIST_FIELD_FN_U64;

		break;

	case 4:

	case 4:

		if (field_is_signed)

		if (field_is_signed)

			fn = hist_field_s32;

			return HIST_FIELD_FN_S32;

		else

		else

			fn = hist_field_u32;

			return HIST_FIELD_FN_U32;

		break;

	case 2:

	case 2:

		if (field_is_signed)

		if (field_is_signed)

			fn = hist_field_s16;

			return HIST_FIELD_FN_S16;

		else

		else

			fn = hist_field_u16;

			return HIST_FIELD_FN_U16;

		break;

	case 1:

	case 1:

		if (field_is_signed)

		if (field_is_signed)

			fn = hist_field_s8;

			return HIST_FIELD_FN_S8;

		else

		else

			fn = hist_field_u8;

			return HIST_FIELD_FN_U8;

		break;

	}

	}





	return fn;

	return HIST_FIELD_FN_NOP;

}

}





static int parse_map_size(char *str)

static int parse_map_size(char *str)
@@ -1922,19 +1945,19 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 
		goto out; /* caller will populate */

		goto out; /* caller will populate */





	if (flags & HIST_FIELD_FL_VAR_REF) {

	if (flags & HIST_FIELD_FL_VAR_REF) {

		hist_field->fn = hist_field_var_ref;

		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;

		goto out;

		goto out;

	}

	}





	if (flags & HIST_FIELD_FL_HITCOUNT) {

	if (flags & HIST_FIELD_FL_HITCOUNT) {

		hist_field->fn = hist_field_counter;

		hist_field->fn_num = HIST_FIELD_FN_COUNTER;

		hist_field->size = sizeof(u64);

		hist_field->size = sizeof(u64);

		hist_field->type = "u64";

		hist_field->type = "u64";

		goto out;

		goto out;

	}

	}





	if (flags & HIST_FIELD_FL_CONST) {

	if (flags & HIST_FIELD_FL_CONST) {

		hist_field->fn = hist_field_const;

		hist_field->fn_num = HIST_FIELD_FN_CONST;

		hist_field->size = sizeof(u64);

		hist_field->size = sizeof(u64);

		hist_field->type = kstrdup("u64", GFP_KERNEL);

		hist_field->type = kstrdup("u64", GFP_KERNEL);

		if (!hist_field->type)

		if (!hist_field->type)
@@ -1943,14 +1966,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 
	}

	}





	if (flags & HIST_FIELD_FL_STACKTRACE) {

	if (flags & HIST_FIELD_FL_STACKTRACE) {

		hist_field->fn = hist_field_none;

		hist_field->fn_num = HIST_FIELD_FN_NOP;

		goto out;

		goto out;

	}

	}





	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {

	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {

		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);

		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);

		hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :

		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :

			hist_field_bucket;

			HIST_FIELD_FN_BUCKET;

		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);

		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);

		hist_field->size = hist_field->operands[0]->size;

		hist_field->size = hist_field->operands[0]->size;

		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);

		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
@@ -1960,14 +1983,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 
	}

	}





	if (flags & HIST_FIELD_FL_TIMESTAMP) {

	if (flags & HIST_FIELD_FL_TIMESTAMP) {

		hist_field->fn = hist_field_timestamp;

		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;

		hist_field->size = sizeof(u64);

		hist_field->size = sizeof(u64);

		hist_field->type = "u64";

		hist_field->type = "u64";

		goto out;

		goto out;

	}

	}





	if (flags & HIST_FIELD_FL_CPU) {

	if (flags & HIST_FIELD_FL_CPU) {

		hist_field->fn = hist_field_cpu;

		hist_field->fn_num = HIST_FIELD_FN_CPU;

		hist_field->size = sizeof(int);

		hist_field->size = sizeof(int);

		hist_field->type = "unsigned int";

		hist_field->type = "unsigned int";

		goto out;

		goto out;
@@ -1987,14 +2010,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 
			goto free;

			goto free;





		if (field->filter_type == FILTER_STATIC_STRING) {

		if (field->filter_type == FILTER_STATIC_STRING) {

			hist_field->fn = hist_field_string;

			hist_field->fn_num = HIST_FIELD_FN_STRING;

			hist_field->size = field->size;

			hist_field->size = field->size;

		} else if (field->filter_type == FILTER_DYN_STRING) {

		} else if (field->filter_type == FILTER_DYN_STRING) {

			hist_field->fn = hist_field_dynstring;

			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;

		} else if (field->filter_type == FILTER_RDYN_STRING)

		} else if (field->filter_type == FILTER_RDYN_STRING)

			hist_field->fn = hist_field_reldynstring;

			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;

		else

		else

			hist_field->fn = hist_field_pstring;

			hist_field->fn_num = HIST_FIELD_FN_PSTRING;

	} else {

	} else {

		hist_field->size = field->size;

		hist_field->size = field->size;

		hist_field->is_signed = field->is_signed;

		hist_field->is_signed = field->is_signed;
@@ -2002,9 +2025,9 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 
		if (!hist_field->type)

		if (!hist_field->type)

			goto free;

			goto free;





		hist_field->fn = select_value_fn(field->size,

		hist_field->fn_num = select_value_fn(field->size,

						     field->is_signed);

						     field->is_signed);

		if (!hist_field->fn) {

		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {

			destroy_hist_field(hist_field, 0);

			destroy_hist_field(hist_field, 0);

			return NULL;

			return NULL;

		}

		}
@@ -2340,7 +2363,7 @@ static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 
	if (!alias)

	if (!alias)

		return NULL;

		return NULL;





	alias->fn = var_ref->fn;

	alias->fn_num = var_ref->fn_num;

	alias->operands[0] = var_ref;

	alias->operands[0] = var_ref;





	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {

	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
@@ -2523,7 +2546,7 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 




	expr->flags |= operand1->flags &

	expr->flags |= operand1->flags &

		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);

		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);

	expr->fn = hist_field_unary_minus;

	expr->fn_num = HIST_FIELD_FN_UMINUS;

	expr->operands[0] = operand1;

	expr->operands[0] = operand1;

	expr->size = operand1->size;

	expr->size = operand1->size;

	expr->is_signed = operand1->is_signed;

	expr->is_signed = operand1->is_signed;
@@ -2595,7 +2618,7 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 
	unsigned long operand_flags, operand2_flags;

	unsigned long operand_flags, operand2_flags;

	int field_op, ret = -EINVAL;

	int field_op, ret = -EINVAL;

	char *sep, *operand1_str;

	char *sep, *operand1_str;

	hist_field_fn_t op_fn;

	enum hist_field_fn op_fn;

	bool combine_consts;

	bool combine_consts;





	if (*n_subexprs > 3) {

	if (*n_subexprs > 3) {
@@ -2654,16 +2677,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 




	switch (field_op) {

	switch (field_op) {

	case FIELD_OP_MINUS:

	case FIELD_OP_MINUS:

		op_fn = hist_field_minus;

		op_fn = HIST_FIELD_FN_MINUS;

		break;

		break;

	case FIELD_OP_PLUS:

	case FIELD_OP_PLUS:

		op_fn = hist_field_plus;

		op_fn = HIST_FIELD_FN_PLUS;

		break;

		break;

	case FIELD_OP_DIV:

	case FIELD_OP_DIV:

		op_fn = hist_field_div;

		op_fn = HIST_FIELD_FN_DIV;

		break;

		break;

	case FIELD_OP_MULT:

	case FIELD_OP_MULT:

		op_fn = hist_field_mult;

		op_fn = HIST_FIELD_FN_MULT;

		break;

		break;

	default:

	default:

		ret = -EINVAL;

		ret = -EINVAL;
@@ -2719,13 +2742,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 
		op_fn = hist_field_get_div_fn(operand2);

		op_fn = hist_field_get_div_fn(operand2);

	}

	}





	expr->fn_num = op_fn;



	if (combine_consts) {

	if (combine_consts) {

		if (var1)

		if (var1)

			expr->operands[0] = var1;

			expr->operands[0] = var1;

		if (var2)

		if (var2)

			expr->operands[1] = var2;

			expr->operands[1] = var2;





		expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);

		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);

		expr->fn_num = HIST_FIELD_FN_CONST;





		expr->operands[0] = NULL;

		expr->operands[0] = NULL;

		expr->operands[1] = NULL;

		expr->operands[1] = NULL;
@@ -2739,8 +2765,6 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 




		expr->name = expr_str(expr, 0);

		expr->name = expr_str(expr, 0);

	} else {

	} else {

		expr->fn = op_fn;



		/* The operand sizes should be the same, so just pick one */

		/* The operand sizes should be the same, so just pick one */

		expr->size = operand1->size;

		expr->size = operand1->size;

		expr->is_signed = operand1->is_signed;

		expr->is_signed = operand1->is_signed;
@@ -3065,7 +3089,7 @@ static inline void __update_field_vars(struct tracing_map_elt *elt, 
		struct hist_field *var = field_var->var;

		struct hist_field *var = field_var->var;

		struct hist_field *val = field_var->val;

		struct hist_field *val = field_var->val;





		var_val = val->fn(val, elt, buffer, rbe, rec);

		var_val = hist_fn_call(val, elt, buffer, rbe, rec);

		var_idx = var->var.idx;

		var_idx = var->var.idx;





		if (val->flags & HIST_FIELD_FL_STRING) {

		if (val->flags & HIST_FIELD_FL_STRING) {
@@ -4186,6 +4210,74 @@ static u64 hist_field_execname(struct hist_field *hist_field, 
	return (u64)(unsigned long)(elt_data->comm);

	return (u64)(unsigned long)(elt_data->comm);

}

}





static u64 hist_fn_call(struct hist_field *hist_field,

			struct tracing_map_elt *elt,

			struct trace_buffer *buffer,

			struct ring_buffer_event *rbe,

			void *event)

{

	switch (hist_field->fn_num) {

	case HIST_FIELD_FN_VAR_REF:

		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_COUNTER:

		return hist_field_counter(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_CONST:

		return hist_field_const(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_LOG2:

		return hist_field_log2(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_BUCKET:

		return hist_field_bucket(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_TIMESTAMP:

		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_CPU:

		return hist_field_cpu(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_STRING:

		return hist_field_string(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_DYNSTRING:

		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_RELDYNSTRING:

		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_PSTRING:

		return hist_field_pstring(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_S64:

		return hist_field_s64(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_U64:

		return hist_field_u64(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_S32:

		return hist_field_s32(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_U32:

		return hist_field_u32(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_S16:

		return hist_field_s16(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_U16:

		return hist_field_u16(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_S8:

		return hist_field_s8(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_U8:

		return hist_field_u8(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_UMINUS:

		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_MINUS:

		return hist_field_minus(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_PLUS:

		return hist_field_plus(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_DIV:

		return hist_field_div(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_MULT:

		return hist_field_mult(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_DIV_POWER2:

		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_DIV_NOT_POWER2:

		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_DIV_MULT_SHIFT:

		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);

	case HIST_FIELD_FN_EXECNAME:

		return hist_field_execname(hist_field, elt, buffer, rbe, event);

	default:

		return 0;

	}

}



/* Convert a var that points to common_pid.execname to a string */

/* Convert a var that points to common_pid.execname to a string */

static void update_var_execname(struct hist_field *hist_field)

static void update_var_execname(struct hist_field *hist_field)

{

{
@@ -4197,7 +4289,7 @@ static void update_var_execname(struct hist_field *hist_field) 
	kfree_const(hist_field->type);

	kfree_const(hist_field->type);

	hist_field->type = "char[]";

	hist_field->type = "char[]";





	hist_field->fn = hist_field_execname;

	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;

}

}





static int create_var_field(struct hist_trigger_data *hist_data,

static int create_var_field(struct hist_trigger_data *hist_data,
@@ -4956,7 +5048,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 




	for_each_hist_val_field(i, hist_data) {

	for_each_hist_val_field(i, hist_data) {

		hist_field = hist_data->fields[i];

		hist_field = hist_data->fields[i];

		hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);

		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);

		if (hist_field->flags & HIST_FIELD_FL_VAR) {

		if (hist_field->flags & HIST_FIELD_FL_VAR) {

			var_idx = hist_field->var.idx;

			var_idx = hist_field->var.idx;
@@ -4987,7 +5079,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 
	for_each_hist_key_field(i, hist_data) {

	for_each_hist_key_field(i, hist_data) {

		hist_field = hist_data->fields[i];

		hist_field = hist_data->fields[i];

		if (hist_field->flags & HIST_FIELD_FL_VAR) {

		if (hist_field->flags & HIST_FIELD_FL_VAR) {

			hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);

			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);

			var_idx = hist_field->var.idx;

			var_idx = hist_field->var.idx;

			tracing_map_set_var(elt, var_idx, hist_val);

			tracing_map_set_var(elt, var_idx, hist_val);

		}

		}
@@ -5062,7 +5154,7 @@ static void event_hist_trigger(struct event_trigger_data *data, 
					 HIST_STACKTRACE_SKIP);

					 HIST_STACKTRACE_SKIP);

			key = entries;

			key = entries;

		} else {

		} else {

			field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);

			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);

			if (key_field->flags & HIST_FIELD_FL_STRING) {

			if (key_field->flags & HIST_FIELD_FL_STRING) {

				key = (void *)(unsigned long)field_contents;

				key = (void *)(unsigned long)field_contents;

				use_compound_key = true;

				use_compound_key = true;