1// SPDX-License-Identifier: GPL-2.0
2/*
3 * unlikely profiler
4 *
5 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 */
7#include <linux/kallsyms.h>
8#include <linux/seq_file.h>
9#include <linux/spinlock.h>
10#include <linux/irqflags.h>
11#include <linux/uaccess.h>
12#include <linux/module.h>
13#include <linux/ftrace.h>
14#include <linux/hash.h>
15#include <linux/fs.h>
16#include <asm/local.h>
17
18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
21
22#ifdef CONFIG_BRANCH_TRACER
23
24static struct tracer branch_trace;
25static int branch_tracing_enabled __read_mostly;
26static DEFINE_MUTEX(branch_tracing_mutex);
27
28static struct trace_array *branch_tracer;
29
30static void
31probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
32{
33	struct trace_event_call *call = &event_branch;
34	struct trace_array *tr = branch_tracer;
35	struct trace_buffer *buffer;
36	struct trace_array_cpu *data;
37	struct ring_buffer_event *event;
38	struct trace_branch *entry;
39	unsigned long flags;
40	unsigned int trace_ctx;
41	const char *p;
42
43	if (current->trace_recursion & TRACE_BRANCH_BIT)
44		return;
45
46	/*
47	 * I would love to save just the ftrace_likely_data pointer, but
48	 * this code can also be used by modules. Ugly things can happen
49	 * if the module is unloaded, and then we go and read the
50	 * pointer.  This is slower, but much safer.
51	 */
52
53	if (unlikely(!tr))
54		return;
55
56	raw_local_irq_save(flags);
57	current->trace_recursion |= TRACE_BRANCH_BIT;
58	data = this_cpu_ptr(tr->array_buffer.data);
59	if (atomic_read(&data->disabled))
60		goto out;
61
62	trace_ctx = tracing_gen_ctx_flags(flags);
63	buffer = tr->array_buffer.buffer;
64	event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
65					  sizeof(*entry), trace_ctx);
66	if (!event)
67		goto out;
68
69	entry	= ring_buffer_event_data(event);
70
71	/* Strip off the path, only save the file */
72	p = f->data.file + strlen(f->data.file);
73	while (p >= f->data.file && *p != '/')
74		p--;
75	p++;
76
77	strncpy(entry->func, f->data.func, TRACE_FUNC_SIZE);
78	strncpy(entry->file, p, TRACE_FILE_SIZE);
79	entry->func[TRACE_FUNC_SIZE] = 0;
80	entry->file[TRACE_FILE_SIZE] = 0;
81	entry->constant = f->constant;
82	entry->line = f->data.line;
83	entry->correct = val == expect;
84
85	if (!call_filter_check_discard(call, entry, buffer, event))
86		trace_buffer_unlock_commit_nostack(buffer, event);
87
88 out:
89	current->trace_recursion &= ~TRACE_BRANCH_BIT;
90	raw_local_irq_restore(flags);
91}
92
93static inline
94void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
95{
96	if (!branch_tracing_enabled)
97		return;
98
99	probe_likely_condition(f, val, expect);
100}
101
102int enable_branch_tracing(struct trace_array *tr)
103{
104	mutex_lock(&branch_tracing_mutex);
105	branch_tracer = tr;
106	/*
107	 * Must be seen before enabling. The reader is a condition
108	 * where we do not need a matching rmb()
109	 */
110	smp_wmb();
111	branch_tracing_enabled++;
112	mutex_unlock(&branch_tracing_mutex);
113
114	return 0;
115}
116
117void disable_branch_tracing(void)
118{
119	mutex_lock(&branch_tracing_mutex);
120
121	if (!branch_tracing_enabled)
122		goto out_unlock;
123
124	branch_tracing_enabled--;
125
126 out_unlock:
127	mutex_unlock(&branch_tracing_mutex);
128}
129
130static int branch_trace_init(struct trace_array *tr)
131{
132	return enable_branch_tracing(tr);
133}
134
135static void branch_trace_reset(struct trace_array *tr)
136{
137	disable_branch_tracing();
138}
139
140static enum print_line_t trace_branch_print(struct trace_iterator *iter,
141					    int flags, struct trace_event *event)
142{
143	struct trace_branch *field;
144
145	trace_assign_type(field, iter->ent);
146
147	trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
148			 field->correct ? "  ok  " : " MISS ",
149			 field->func,
150			 field->file,
151			 field->line);
152
153	return trace_handle_return(&iter->seq);
154}
155
156static void branch_print_header(struct seq_file *s)
157{
158	seq_puts(s, "#           TASK-PID    CPU#    TIMESTAMP  CORRECT"
159		    "  FUNC:FILE:LINE\n"
160		    "#              | |       |          |         |   "
161		    "    |\n");
162}
163
164static struct trace_event_functions trace_branch_funcs = {
165	.trace		= trace_branch_print,
166};
167
168static struct trace_event trace_branch_event = {
169	.type		= TRACE_BRANCH,
170	.funcs		= &trace_branch_funcs,
171};
172
173static struct tracer branch_trace __read_mostly =
174{
175	.name		= "branch",
176	.init		= branch_trace_init,
177	.reset		= branch_trace_reset,
178#ifdef CONFIG_FTRACE_SELFTEST
179	.selftest	= trace_selftest_startup_branch,
180#endif /* CONFIG_FTRACE_SELFTEST */
181	.print_header	= branch_print_header,
182};
183
184__init static int init_branch_tracer(void)
185{
186	int ret;
187
188	ret = register_trace_event(&trace_branch_event);
189	if (!ret) {
190		printk(KERN_WARNING "Warning: could not register "
191				    "branch events\n");
192		return 1;
193	}
194	return register_tracer(&branch_trace);
195}
196core_initcall(init_branch_tracer);
197
198#else
199static inline
200void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
201{
202}
203#endif /* CONFIG_BRANCH_TRACER */
204
205void ftrace_likely_update(struct ftrace_likely_data *f, int val,
206			  int expect, int is_constant)
207{
208	unsigned long flags = user_access_save();
209
210	/* A constant is always correct */
211	if (is_constant) {
212		f->constant++;
213		val = expect;
214	}
215	/*
216	 * I would love to have a trace point here instead, but the
217	 * trace point code is so inundated with unlikely and likely
218	 * conditions that the recursive nightmare that exists is too
219	 * much to try to get working. At least for now.
220	 */
221	trace_likely_condition(f, val, expect);
222
223	/* FIXME: Make this atomic! */
224	if (val == expect)
225		f->data.correct++;
226	else
227		f->data.incorrect++;
228
229	user_access_restore(flags);
230}
231EXPORT_SYMBOL(ftrace_likely_update);
232
233extern unsigned long __start_annotated_branch_profile[];
234extern unsigned long __stop_annotated_branch_profile[];
235
236static int annotated_branch_stat_headers(struct seq_file *m)
237{
238	seq_puts(m, " correct incorrect  % "
239		    "       Function                "
240		    "  File              Line\n"
241		    " ------- ---------  - "
242		    "       --------                "
243		    "  ----              ----\n");
244	return 0;
245}
246
247static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
248{
249	long percent;
250
251	if (p->correct) {
252		percent = p->incorrect * 100;
253		percent /= p->correct + p->incorrect;
254	} else
255		percent = p->incorrect ? 100 : -1;
256
257	return percent;
258}
259
260static const char *branch_stat_process_file(struct ftrace_branch_data *p)
261{
262	const char *f;
263
264	/* Only print the file, not the path */
265	f = p->file + strlen(p->file);
266	while (f >= p->file && *f != '/')
267		f--;
268	return ++f;
269}
270
271static void branch_stat_show(struct seq_file *m,
272			     struct ftrace_branch_data *p, const char *f)
273{
274	long percent;
275
276	/*
277	 * The miss is overlayed on correct, and hit on incorrect.
278	 */
279	percent = get_incorrect_percent(p);
280
281	if (percent < 0)
282		seq_puts(m, "  X ");
283	else
284		seq_printf(m, "%3ld ", percent);
285
286	seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
287}
288
289static int branch_stat_show_normal(struct seq_file *m,
290				   struct ftrace_branch_data *p, const char *f)
291{
292	seq_printf(m, "%8lu %8lu ",  p->correct, p->incorrect);
293	branch_stat_show(m, p, f);
294	return 0;
295}
296
297static int annotate_branch_stat_show(struct seq_file *m, void *v)
298{
299	struct ftrace_likely_data *p = v;
300	const char *f;
301	int l;
302
303	f = branch_stat_process_file(&p->data);
304
305	if (!p->constant)
306		return branch_stat_show_normal(m, &p->data, f);
307
308	l = snprintf(NULL, 0, "/%lu", p->constant);
309	l = l > 8 ? 0 : 8 - l;
310
311	seq_printf(m, "%8lu/%lu %*lu ",
312		   p->data.correct, p->constant, l, p->data.incorrect);
313	branch_stat_show(m, &p->data, f);
314	return 0;
315}
316
317static void *annotated_branch_stat_start(struct tracer_stat *trace)
318{
319	return __start_annotated_branch_profile;
320}
321
322static void *
323annotated_branch_stat_next(void *v, int idx)
324{
325	struct ftrace_likely_data *p = v;
326
327	++p;
328
329	if ((void *)p >= (void *)__stop_annotated_branch_profile)
330		return NULL;
331
332	return p;
333}
334
335static int annotated_branch_stat_cmp(const void *p1, const void *p2)
336{
337	const struct ftrace_branch_data *a = p1;
338	const struct ftrace_branch_data *b = p2;
339
340	long percent_a, percent_b;
341
342	percent_a = get_incorrect_percent(a);
343	percent_b = get_incorrect_percent(b);
344
345	if (percent_a < percent_b)
346		return -1;
347	if (percent_a > percent_b)
348		return 1;
349
350	if (a->incorrect < b->incorrect)
351		return -1;
352	if (a->incorrect > b->incorrect)
353		return 1;
354
355	/*
356	 * Since the above shows worse (incorrect) cases
357	 * first, we continue that by showing best (correct)
358	 * cases last.
359	 */
360	if (a->correct > b->correct)
361		return -1;
362	if (a->correct < b->correct)
363		return 1;
364
365	return 0;
366}
367
368static struct tracer_stat annotated_branch_stats = {
369	.name = "branch_annotated",
370	.stat_start = annotated_branch_stat_start,
371	.stat_next = annotated_branch_stat_next,
372	.stat_cmp = annotated_branch_stat_cmp,
373	.stat_headers = annotated_branch_stat_headers,
374	.stat_show = annotate_branch_stat_show
375};
376
377__init static int init_annotated_branch_stats(void)
378{
379	int ret;
380
381	ret = register_stat_tracer(&annotated_branch_stats);
382	if (!ret) {
383		printk(KERN_WARNING "Warning: could not register "
384				    "annotated branches stats\n");
385		return 1;
386	}
387	return 0;
388}
389fs_initcall(init_annotated_branch_stats);
390
391#ifdef CONFIG_PROFILE_ALL_BRANCHES
392
393extern unsigned long __start_branch_profile[];
394extern unsigned long __stop_branch_profile[];
395
396static int all_branch_stat_headers(struct seq_file *m)
397{
398	seq_puts(m, "   miss      hit    % "
399		    "       Function                "
400		    "  File              Line\n"
401		    " ------- ---------  - "
402		    "       --------                "
403		    "  ----              ----\n");
404	return 0;
405}
406
407static void *all_branch_stat_start(struct tracer_stat *trace)
408{
409	return __start_branch_profile;
410}
411
412static void *
413all_branch_stat_next(void *v, int idx)
414{
415	struct ftrace_branch_data *p = v;
416
417	++p;
418
419	if ((void *)p >= (void *)__stop_branch_profile)
420		return NULL;
421
422	return p;
423}
424
425static int all_branch_stat_show(struct seq_file *m, void *v)
426{
427	struct ftrace_branch_data *p = v;
428	const char *f;
429
430	f = branch_stat_process_file(p);
431	return branch_stat_show_normal(m, p, f);
432}
433
434static struct tracer_stat all_branch_stats = {
435	.name = "branch_all",
436	.stat_start = all_branch_stat_start,
437	.stat_next = all_branch_stat_next,
438	.stat_headers = all_branch_stat_headers,
439	.stat_show = all_branch_stat_show
440};
441
442__init static int all_annotated_branch_stats(void)
443{
444	int ret;
445
446	ret = register_stat_tracer(&all_branch_stats);
447	if (!ret) {
448		printk(KERN_WARNING "Warning: could not register "
449				    "all branches stats\n");
450		return 1;
451	}
452	return 0;
453}
454fs_initcall(all_annotated_branch_stats);
455#endif /* CONFIG_PROFILE_ALL_BRANCHES */
456