Kernel

Kernel Git Source Tree

Root/kernel/trace/trace_kprobe.c

1	/*
2	* Kprobes-based tracing events
3	*
4	* Created by Masami Hiramatsu <mhiramat@redhat.com>
5	*
6	* This program is free software; you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License version 2 as
8	* published by the Free Software Foundation.
9	*
10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.
14	*
15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software
17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18	*/
19
20	#include <linux/module.h>
21	#include <linux/uaccess.h>
22	#include <linux/kprobes.h>
23	#include <linux/seq_file.h>
24	#include <linux/slab.h>
25	#include <linux/smp.h>
26	#include <linux/debugfs.h>
27	#include <linux/types.h>
28	#include <linux/string.h>
29	#include <linux/ctype.h>
30	#include <linux/ptrace.h>
31	#include <linux/perf_event.h>
32	#include <linux/stringify.h>
33	#include <linux/limits.h>
34	#include <asm/bitsperlong.h>
35
36	#include "trace.h"
37	#include "trace_output.h"
38
39	#define MAX_TRACE_ARGS 128
40	#define MAX_ARGSTR_LEN 63
41	#define MAX_EVENT_NAME_LEN 64
42	#define MAX_STRING_SIZE PATH_MAX
43	#define KPROBE_EVENT_SYSTEM "kprobes"
44
45	/* Reserved field names */
46	#define FIELD_STRING_IP "__probe_ip"
47	#define FIELD_STRING_RETIP "__probe_ret_ip"
48	#define FIELD_STRING_FUNC "__probe_func"
49
50	const char *reserved_field_names[] = {
51	"common_type",
52	"common_flags",
53	"common_preempt_count",
54	"common_pid",
55	"common_tgid",
56	"common_lock_depth",
57	FIELD_STRING_IP,
58	FIELD_STRING_RETIP,
59	FIELD_STRING_FUNC,
60	};
61
62	/* Printing function type */
63	typedef int (print_type_func_t)(struct trace_seq , const char , void ,
64	void *);
65	#define PRINT_TYPE_FUNC_NAME(type) print_type_##type
66	#define PRINT_TYPE_FMT_NAME(type) print_type_format_##type
67
68	/* Printing in basic type function template */
69	#define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \
70	static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
71	const char *name, \
72	void data, void ent)\
73	{ \
74	return trace_seq_printf(s, " %s=" fmt, name, (cast)(type )data);\
75	} \
76	static const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
77
78	DEFINE_BASIC_PRINT_TYPE_FUNC(u8, "%x", unsigned int)
79	DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "%x", unsigned int)
80	DEFINE_BASIC_PRINT_TYPE_FUNC(u32, "%lx", unsigned long)
81	DEFINE_BASIC_PRINT_TYPE_FUNC(u64, "%llx", unsigned long long)
82	DEFINE_BASIC_PRINT_TYPE_FUNC(s8, "%d", int)
83	DEFINE_BASIC_PRINT_TYPE_FUNC(s16, "%d", int)
84	DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%ld", long)
85	DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%lld", long long)
86
87	/* data_rloc: data relative location, compatible with u32 */
88	#define make_data_rloc(len, roffs) \
89	(((u32)(len) << 16) \| ((u32)(roffs) & 0xffff))
90	#define get_rloc_len(dl) ((u32)(dl) >> 16)
91	#define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
92
93	static inline void get_rloc_data(u32 dl)
94	{
95	return (u8 )dl + get_rloc_offs(dl);
96	}
97
98	/* For data_loc conversion */
99	static inline void get_loc_data(u32 dl, void *ent)
100	{
101	return (u8 )ent + get_rloc_offs(dl);
102	}
103
104	/*
105	* Convert data_rloc to data_loc:
106	* data_rloc stores the offset from data_rloc itself, but data_loc
107	* stores the offset from event entry.
108	*/
109	#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
110
111	/* For defining macros, define string/string_size types */
112	typedef u32 string;
113	typedef u32 string_size;
114
115	/* Print type function for string type */
116	static __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s,
117	const char *name,
118	void data, void ent)
119	{
120	int len = (u32 )data >> 16;
121
122	if (!len)
123	return trace_seq_printf(s, " %s=(fault)", name);
124	else
125	return trace_seq_printf(s, " %s=\"%s\"", name,
126	(const char *)get_loc_data(data, ent));
127	}
128	static const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
129
130	/* Data fetch function type */
131	typedef void (fetch_func_t)(struct pt_regs , void , void );
132
133	struct fetch_param {
134	fetch_func_t fn;
135	void *data;
136	};
137
138	static __kprobes void call_fetch(struct fetch_param *fprm,
139	struct pt_regs regs, void dest)
140	{
141	return fprm->fn(regs, fprm->data, dest);
142	}
143
144	#define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
145	/*
146	* Define macro for basic types - we don't need to define s* types, because
147	* we have to care only about bitwidth at recording time.
148	*/
149	#define DEFINE_BASIC_FETCH_FUNCS(method) \
150	DEFINE_FETCH_##method(u8) \
151	DEFINE_FETCH_##method(u16) \
152	DEFINE_FETCH_##method(u32) \
153	DEFINE_FETCH_##method(u64)
154
155	#define CHECK_FETCH_FUNCS(method, fn) \
156	(((FETCH_FUNC_NAME(method, u8) == fn) \|\| \
157	(FETCH_FUNC_NAME(method, u16) == fn) \|\| \
158	(FETCH_FUNC_NAME(method, u32) == fn) \|\| \
159	(FETCH_FUNC_NAME(method, u64) == fn) \|\| \
160	(FETCH_FUNC_NAME(method, string) == fn) \|\| \
161	(FETCH_FUNC_NAME(method, string_size) == fn)) \
162	&& (fn != NULL))
163
164	/* Data fetch function templates */
165	#define DEFINE_FETCH_reg(type) \
166	static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
167	void offset, void dest) \
168	{ \
169	(type )dest = (type)regs_get_register(regs, \
170	(unsigned int)((unsigned long)offset)); \
171	}
172	DEFINE_BASIC_FETCH_FUNCS(reg)
173	/* No string on the register */
174	#define fetch_reg_string NULL
175	#define fetch_reg_string_size NULL
176
177	#define DEFINE_FETCH_stack(type) \
178	static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
179	void offset, void dest) \
180	{ \
181	(type )dest = (type)regs_get_kernel_stack_nth(regs, \
182	(unsigned int)((unsigned long)offset)); \
183	}
184	DEFINE_BASIC_FETCH_FUNCS(stack)
185	/* No string on the stack entry */
186	#define fetch_stack_string NULL
187	#define fetch_stack_string_size NULL
188
189	#define DEFINE_FETCH_retval(type) \
190	static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\
191	void dummy, void dest) \
192	{ \
193	(type )dest = (type)regs_return_value(regs); \
194	}
195	DEFINE_BASIC_FETCH_FUNCS(retval)
196	/* No string on the retval */
197	#define fetch_retval_string NULL
198	#define fetch_retval_string_size NULL
199
200	#define DEFINE_FETCH_memory(type) \
201	static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
202	void addr, void dest) \
203	{ \
204	type retval; \
205	if (probe_kernel_address(addr, retval)) \
206	(type )dest = 0; \
207	else \
208	(type )dest = retval; \
209	}
210	DEFINE_BASIC_FETCH_FUNCS(memory)
211	/*
212	* Fetch a null-terminated string. Caller MUST set (u32 )dest with max
213	* length and relative data location.
214	*/
215	static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
216	void addr, void dest)
217	{
218	long ret;
219	int maxlen = get_rloc_len((u32 )dest);
220	u8 *dst = get_rloc_data(dest);
221	u8 *src = addr;
222	mm_segment_t old_fs = get_fs();
223	if (!maxlen)
224	return;
225	/*
226	* Try to get string again, since the string can be changed while
227	* probing.
228	*/
229	set_fs(KERNEL_DS);
230	pagefault_disable();
231	do
232	ret = __copy_from_user_inatomic(dst++, src++, 1);
233	while (dst[-1] && ret == 0 && src - (u8 *)addr < maxlen);
234	dst[-1] = '\0';
235	pagefault_enable();
236	set_fs(old_fs);
237
238	if (ret < 0) { /* Failed to fetch string */
239	((u8 *)get_rloc_data(dest))[0] = '\0';
240	(u32 )dest = make_data_rloc(0, get_rloc_offs((u32 )dest));
241	} else
242	(u32 )dest = make_data_rloc(src - (u8 *)addr,
243	get_rloc_offs((u32 )dest));
244	}
245	/* Return the length of string -- including null terminal byte */
246	static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
247	void addr, void dest)
248	{
249	int ret, len = 0;
250	u8 c;
251	mm_segment_t old_fs = get_fs();
252
253	set_fs(KERNEL_DS);
254	pagefault_disable();
255	do {
256	ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
257	len++;
258	} while (c && ret == 0 && len < MAX_STRING_SIZE);
259	pagefault_enable();
260	set_fs(old_fs);
261
262	if (ret < 0) /* Failed to check the length */
263	(u32 )dest = 0;
264	else
265	(u32 )dest = len;
266	}
267
268	/* Memory fetching by symbol */
269	struct symbol_cache {
270	char *symbol;
271	long offset;
272	unsigned long addr;
273	};
274
275	static unsigned long update_symbol_cache(struct symbol_cache *sc)
276	{
277	sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
278	if (sc->addr)
279	sc->addr += sc->offset;
280	return sc->addr;
281	}
282
283	static void free_symbol_cache(struct symbol_cache *sc)
284	{
285	kfree(sc->symbol);
286	kfree(sc);
287	}
288
289	static struct symbol_cache alloc_symbol_cache(const char sym, long offset)
290	{
291	struct symbol_cache *sc;
292
293	if (!sym \|\| strlen(sym) == 0)
294	return NULL;
295	sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
296	if (!sc)
297	return NULL;
298
299	sc->symbol = kstrdup(sym, GFP_KERNEL);
300	if (!sc->symbol) {
301	kfree(sc);
302	return NULL;
303	}
304	sc->offset = offset;
305
306	update_symbol_cache(sc);
307	return sc;
308	}
309
310	#define DEFINE_FETCH_symbol(type) \
311	static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\
312	void data, void dest) \
313	{ \
314	struct symbol_cache *sc = data; \
315	if (sc->addr) \
316	fetch_memory_##type(regs, (void *)sc->addr, dest); \
317	else \
318	(type )dest = 0; \
319	}
320	DEFINE_BASIC_FETCH_FUNCS(symbol)
321	DEFINE_FETCH_symbol(string)
322	DEFINE_FETCH_symbol(string_size)
323
324	/* Dereference memory access function */
325	struct deref_fetch_param {
326	struct fetch_param orig;
327	long offset;
328	};
329
330	#define DEFINE_FETCH_deref(type) \
331	static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\
332	void data, void dest) \
333	{ \
334	struct deref_fetch_param *dprm = data; \
335	unsigned long addr; \
336	call_fetch(&dprm->orig, regs, &addr); \
337	if (addr) { \
338	addr += dprm->offset; \
339	fetch_memory_##type(regs, (void *)addr, dest); \
340	} else \
341	(type )dest = 0; \
342	}
343	DEFINE_BASIC_FETCH_FUNCS(deref)
344	DEFINE_FETCH_deref(string)
345	DEFINE_FETCH_deref(string_size)
346
347	static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data)
348	{
349	if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
350	free_deref_fetch_param(data->orig.data);
351	else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
352	free_symbol_cache(data->orig.data);
353	kfree(data);
354	}
355
356	/* Default (unsigned long) fetch type */
357	#define __DEFAULT_FETCH_TYPE(t) u##t
358	#define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
359	#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
360	#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
361
362	/* Fetch types */
363	enum {
364	FETCH_MTD_reg = 0,
365	FETCH_MTD_stack,
366	FETCH_MTD_retval,
367	FETCH_MTD_memory,
368	FETCH_MTD_symbol,
369	FETCH_MTD_deref,
370	FETCH_MTD_END,
371	};
372
373	#define ASSIGN_FETCH_FUNC(method, type) \
374	[FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type)
375
376	#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
377	{.name = _name, \
378	.size = _size, \
379	.is_signed = sign, \
380	.print = PRINT_TYPE_FUNC_NAME(ptype), \
381	.fmt = PRINT_TYPE_FMT_NAME(ptype), \
382	.fmttype = _fmttype, \
383	.fetch = { \
384	ASSIGN_FETCH_FUNC(reg, ftype), \
385	ASSIGN_FETCH_FUNC(stack, ftype), \
386	ASSIGN_FETCH_FUNC(retval, ftype), \
387	ASSIGN_FETCH_FUNC(memory, ftype), \
388	ASSIGN_FETCH_FUNC(symbol, ftype), \
389	ASSIGN_FETCH_FUNC(deref, ftype), \
390	} \
391	}
392
393	#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
394	__ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype)
395
396	#define FETCH_TYPE_STRING 0
397	#define FETCH_TYPE_STRSIZE 1
398
399	/* Fetch type information table */
400	static const struct fetch_type {
401	const char name; / Name of type */
402	size_t size; /* Byte size of type */
403	int is_signed; /* Signed flag */
404	print_type_func_t print; /* Print functions */
405	const char fmt; / Fromat string */
406	const char fmttype; / Name in format file */
407	/* Fetch functions */
408	fetch_func_t fetch[FETCH_MTD_END];
409	} fetch_type_table[] = {
410	/* Special types */
411	[FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
412	sizeof(u32), 1, "__data_loc char[]"),
413	[FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
414	string_size, sizeof(u32), 0, "u32"),
415	/* Basic types */
416	ASSIGN_FETCH_TYPE(u8, u8, 0),
417	ASSIGN_FETCH_TYPE(u16, u16, 0),
418	ASSIGN_FETCH_TYPE(u32, u32, 0),
419	ASSIGN_FETCH_TYPE(u64, u64, 0),
420	ASSIGN_FETCH_TYPE(s8, u8, 1),
421	ASSIGN_FETCH_TYPE(s16, u16, 1),
422	ASSIGN_FETCH_TYPE(s32, u32, 1),
423	ASSIGN_FETCH_TYPE(s64, u64, 1),
424	};
425
426	static const struct fetch_type find_fetch_type(const char type)
427	{
428	int i;
429
430	if (!type)
431	type = DEFAULT_FETCH_TYPE_STR;
432
433	for (i = 0; i < ARRAY_SIZE(fetch_type_table); i++)
434	if (strcmp(type, fetch_type_table[i].name) == 0)
435	return &fetch_type_table[i];
436	return NULL;
437	}
438
439	/* Special function : only accept unsigned long */
440	static __kprobes void fetch_stack_address(struct pt_regs *regs,
441	void dummy, void dest)
442	{
443	(unsigned long )dest = kernel_stack_pointer(regs);
444	}
445
446	static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
447	fetch_func_t orig_fn)
448	{
449	int i;
450
451	if (type != &fetch_type_table[FETCH_TYPE_STRING])
452	return NULL; /* Only string type needs size function */
453	for (i = 0; i < FETCH_MTD_END; i++)
454	if (type->fetch[i] == orig_fn)
455	return fetch_type_table[FETCH_TYPE_STRSIZE].fetch[i];
456
457	WARN_ON(1); /* This should not happen */
458	return NULL;
459	}
460
461	/**
462	* Kprobe event core functions
463	*/
464
465	struct probe_arg {
466	struct fetch_param fetch;
467	struct fetch_param fetch_size;
468	unsigned int offset; /* Offset from argument entry */
469	const char name; / Name of this argument */
470	const char comm; / Command of this argument */
471	const struct fetch_type type; / Type of this argument */
472	};
473
474	/* Flags for trace_probe */
475	#define TP_FLAG_TRACE 1
476	#define TP_FLAG_PROFILE 2
477
478	struct trace_probe {
479	struct list_head list;
480	struct kretprobe rp; /* Use rp.kp for kprobe use */
481	unsigned long nhit;
482	unsigned int flags; /* For TP_FLAG_* */
483	const char symbol; / symbol name */
484	struct ftrace_event_class class;
485	struct ftrace_event_call call;
486	ssize_t size; /* trace entry size */
487	unsigned int nr_args;
488	struct probe_arg args[];
489	};
490
491	#define SIZEOF_TRACE_PROBE(n) \
492	(offsetof(struct trace_probe, args) + \
493	(sizeof(struct probe_arg) * (n)))
494
495
496	static __kprobes int probe_is_return(struct trace_probe *tp)
497	{
498	return tp->rp.handler != NULL;
499	}
500
501	static __kprobes const char probe_symbol(struct trace_probe tp)
502	{
503	return tp->symbol ? tp->symbol : "unknown";
504	}
505
506	static int register_probe_event(struct trace_probe *tp);
507	static void unregister_probe_event(struct trace_probe *tp);
508
509	static DEFINE_MUTEX(probe_lock);
510	static LIST_HEAD(probe_list);
511
512	static int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs);
513	static int kretprobe_dispatcher(struct kretprobe_instance *ri,
514	struct pt_regs *regs);
515
516	/* Check the name is good for event/group/fields */
517	static int is_good_name(const char *name)
518	{
519	if (!isalpha(name) && name != '_')
520	return 0;
521	while (*++name != '\0') {
522	if (!isalpha(name) && !isdigit(name) && *name != '_')
523	return 0;
524	}
525	return 1;
526	}
527
528	/*
529	* Allocate new trace_probe and initialize it (including kprobes).
530	*/
531	static struct trace_probe alloc_trace_probe(const char group,
532	const char *event,
533	void *addr,
534	const char *symbol,
535	unsigned long offs,
536	int nargs, int is_return)
537	{
538	struct trace_probe *tp;
539	int ret = -ENOMEM;
540
541	tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL);
542	if (!tp)
543	return ERR_PTR(ret);
544
545	if (symbol) {
546	tp->symbol = kstrdup(symbol, GFP_KERNEL);
547	if (!tp->symbol)
548	goto error;
549	tp->rp.kp.symbol_name = tp->symbol;
550	tp->rp.kp.offset = offs;
551	} else
552	tp->rp.kp.addr = addr;
553
554	if (is_return)
555	tp->rp.handler = kretprobe_dispatcher;
556	else
557	tp->rp.kp.pre_handler = kprobe_dispatcher;
558
559	if (!event \|\| !is_good_name(event)) {
560	ret = -EINVAL;
561	goto error;
562	}
563
564	tp->call.class = &tp->class;
565	tp->call.name = kstrdup(event, GFP_KERNEL);
566	if (!tp->call.name)
567	goto error;
568
569	if (!group \|\| !is_good_name(group)) {
570	ret = -EINVAL;
571	goto error;
572	}
573
574	tp->class.system = kstrdup(group, GFP_KERNEL);
575	if (!tp->class.system)
576	goto error;
577
578	INIT_LIST_HEAD(&tp->list);
579	return tp;
580	error:
581	kfree(tp->call.name);
582	kfree(tp->symbol);
583	kfree(tp);
584	return ERR_PTR(ret);
585	}
586
587	static void free_probe_arg(struct probe_arg *arg)
588	{
589	if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
590	free_deref_fetch_param(arg->fetch.data);
591	else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
592	free_symbol_cache(arg->fetch.data);
593	kfree(arg->name);
594	kfree(arg->comm);
595	}
596
597	static void free_trace_probe(struct trace_probe *tp)
598	{
599	int i;
600
601	for (i = 0; i < tp->nr_args; i++)
602	free_probe_arg(&tp->args[i]);
603
604	kfree(tp->call.class->system);
605	kfree(tp->call.name);
606	kfree(tp->symbol);
607	kfree(tp);
608	}
609
610	static struct trace_probe find_probe_event(const char event,
611	const char *group)
612	{
613	struct trace_probe *tp;
614
615	list_for_each_entry(tp, &probe_list, list)
616	if (strcmp(tp->call.name, event) == 0 &&
617	strcmp(tp->call.class->system, group) == 0)
618	return tp;
619	return NULL;
620	}
621
622	/* Unregister a trace_probe and probe_event: call with locking probe_lock */
623	static void unregister_trace_probe(struct trace_probe *tp)
624	{
625	if (probe_is_return(tp))
626	unregister_kretprobe(&tp->rp);
627	else
628	unregister_kprobe(&tp->rp.kp);
629	list_del(&tp->list);
630	unregister_probe_event(tp);
631	}
632
633	/* Register a trace_probe and probe_event */
634	static int register_trace_probe(struct trace_probe *tp)
635	{
636	struct trace_probe *old_tp;
637	int ret;
638
639	mutex_lock(&probe_lock);
640
641	/* register as an event */
642	old_tp = find_probe_event(tp->call.name, tp->call.class->system);
643	if (old_tp) {
644	/* delete old event */
645	unregister_trace_probe(old_tp);
646	free_trace_probe(old_tp);
647	}
648	ret = register_probe_event(tp);
649	if (ret) {
650	pr_warning("Failed to register probe event(%d)\n", ret);
651	goto end;
652	}
653
654	tp->rp.kp.flags \|= KPROBE_FLAG_DISABLED;
655	if (probe_is_return(tp))
656	ret = register_kretprobe(&tp->rp);
657	else
658	ret = register_kprobe(&tp->rp.kp);
659
660	if (ret) {
661	pr_warning("Could not insert probe(%d)\n", ret);
662	if (ret == -EILSEQ) {
663	pr_warning("Probing address(0x%p) is not an "
664	"instruction boundary.\n",
665	tp->rp.kp.addr);
666	ret = -EINVAL;
667	}
668	unregister_probe_event(tp);
669	} else
670	list_add_tail(&tp->list, &probe_list);
671	end:
672	mutex_unlock(&probe_lock);
673	return ret;
674	}
675
676	/* Split symbol and offset. */
677	static int split_symbol_offset(char symbol, unsigned long offset)
678	{
679	char *tmp;
680	int ret;
681
682	if (!offset)
683	return -EINVAL;
684
685	tmp = strchr(symbol, '+');
686	if (tmp) {
687	/* skip sign because strict_strtol doesn't accept '+' */
688	ret = strict_strtoul(tmp + 1, 0, offset);
689	if (ret)
690	return ret;
691	*tmp = '\0';
692	} else
693	*offset = 0;
694	return 0;
695	}
696
697	#define PARAM_MAX_ARGS 16
698	#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
699
700	static int parse_probe_vars(char arg, const struct fetch_type t,
701	struct fetch_param *f, int is_return)
702	{
703	int ret = 0;
704	unsigned long param;
705
706	if (strcmp(arg, "retval") == 0) {
707	if (is_return)
708	f->fn = t->fetch[FETCH_MTD_retval];
709	else
710	ret = -EINVAL;
711	} else if (strncmp(arg, "stack", 5) == 0) {
712	if (arg[5] == '\0') {
713	if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR) == 0)
714	f->fn = fetch_stack_address;
715	else
716	ret = -EINVAL;
717	} else if (isdigit(arg[5])) {
718	ret = strict_strtoul(arg + 5, 10, &param);
719	if (ret \|\| param > PARAM_MAX_STACK)
720	ret = -EINVAL;
721	else {
722	f->fn = t->fetch[FETCH_MTD_stack];
723	f->data = (void *)param;
724	}
725	} else
726	ret = -EINVAL;
727	} else
728	ret = -EINVAL;
729	return ret;
730	}
731
732	/* Recursive argument parser */
733	static int __parse_probe_arg(char arg, const struct fetch_type t,
734	struct fetch_param *f, int is_return)
735	{
736	int ret = 0;
737	unsigned long param;
738	long offset;
739	char *tmp;
740
741	switch (arg[0]) {
742	case '$':
743	ret = parse_probe_vars(arg + 1, t, f, is_return);
744	break;
745	case '%': /* named register */
746	ret = regs_query_register_offset(arg + 1);
747	if (ret >= 0) {
748	f->fn = t->fetch[FETCH_MTD_reg];
749	f->data = (void *)(unsigned long)ret;
750	ret = 0;
751	}
752	break;
753	case '@': /* memory or symbol */
754	if (isdigit(arg[1])) {
755	ret = strict_strtoul(arg + 1, 0, &param);
756	if (ret)
757	break;
758	f->fn = t->fetch[FETCH_MTD_memory];
759	f->data = (void *)param;
760	} else {
761	ret = split_symbol_offset(arg + 1, &offset);
762	if (ret)
763	break;
764	f->data = alloc_symbol_cache(arg + 1, offset);
765	if (f->data)
766	f->fn = t->fetch[FETCH_MTD_symbol];
767	}
768	break;
769	case '+': /* deref memory */
770	case '-':
771	tmp = strchr(arg, '(');
772	if (!tmp)
773	break;
774	*tmp = '\0';
775	ret = strict_strtol(arg + 1, 0, &offset);
776	if (ret)
777	break;
778	if (arg[0] == '-')
779	offset = -offset;
780	arg = tmp + 1;
781	tmp = strrchr(arg, ')');
782	if (tmp) {
783	struct deref_fetch_param *dprm;
784	const struct fetch_type *t2 = find_fetch_type(NULL);
785	*tmp = '\0';
786	dprm = kzalloc(sizeof(struct deref_fetch_param),
787	GFP_KERNEL);
788	if (!dprm)
789	return -ENOMEM;
790	dprm->offset = offset;
791	ret = __parse_probe_arg(arg, t2, &dprm->orig,
792	is_return);
793	if (ret)
794	kfree(dprm);
795	else {
796	f->fn = t->fetch[FETCH_MTD_deref];
797	f->data = (void *)dprm;
798	}
799	}
800	break;
801	}
802	if (!ret && !f->fn) { /* Parsed, but do not find fetch method */
803	pr_info("%s type has no corresponding fetch method.\n",
804	t->name);
805	ret = -EINVAL;
806	}
807	return ret;
808	}
809
810	/* String length checking wrapper */
811	static int parse_probe_arg(char arg, struct trace_probe tp,
812	struct probe_arg *parg, int is_return)
813	{
814	const char *t;
815	int ret;
816
817	if (strlen(arg) > MAX_ARGSTR_LEN) {
818	pr_info("Argument is too long.: %s\n", arg);
819	return -ENOSPC;
820	}
821	parg->comm = kstrdup(arg, GFP_KERNEL);
822	if (!parg->comm) {
823	pr_info("Failed to allocate memory for command '%s'.\n", arg);
824	return -ENOMEM;
825	}
826	t = strchr(parg->comm, ':');
827	if (t) {
828	arg[t - parg->comm] = '\0';
829	t++;
830	}
831	parg->type = find_fetch_type(t);
832	if (!parg->type) {
833	pr_info("Unsupported type: %s\n", t);
834	return -EINVAL;
835	}
836	parg->offset = tp->size;
837	tp->size += parg->type->size;
838	ret = __parse_probe_arg(arg, parg->type, &parg->fetch, is_return);
839	if (ret >= 0) {
840	parg->fetch_size.fn = get_fetch_size_function(parg->type,
841	parg->fetch.fn);
842	parg->fetch_size.data = parg->fetch.data;
843	}
844	return ret;
845	}
846
847	/* Return 1 if name is reserved or already used by another argument */
848	static int conflict_field_name(const char *name,
849	struct probe_arg *args, int narg)
850	{
851	int i;
852	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
853	if (strcmp(reserved_field_names[i], name) == 0)
854	return 1;
855	for (i = 0; i < narg; i++)
856	if (strcmp(args[i].name, name) == 0)
857	return 1;
858	return 0;
859	}
860
861	static int create_trace_probe(int argc, char **argv)
862	{
863	/*
864	* Argument syntax:
865	* - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]\|KADDR [FETCHARGS]
866	* - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
867	* Fetch args:
868	* $retval : fetch return value
869	* $stack : fetch stack address
870	* $stackN : fetch Nth of stack (N:0-)
871	* @ADDR : fetch memory at ADDR (ADDR should be in kernel)
872	* @SYM[+\|-offs] : fetch memory at SYM +\|- offs (SYM is a data symbol)
873	* %REG : fetch register REG
874	* Dereferencing memory fetch:
875	* +\|-offs(ARG) : fetch memory at ARG +\|- offs address.
876	* Alias name of args:
877	* NAME=FETCHARG : set NAME as alias of FETCHARG.
878	* Type of args:
879	* FETCHARG:TYPE : use TYPE instead of unsigned long.
880	*/
881	struct trace_probe *tp;
882	int i, ret = 0;
883	int is_return = 0, is_delete = 0;
884	char symbol = NULL, event = NULL, *group = NULL;
885	char *arg;
886	unsigned long offset = 0;
887	void *addr = NULL;
888	char buf[MAX_EVENT_NAME_LEN];
889
890	/* argc must be >= 1 */
891	if (argv[0][0] == 'p')
892	is_return = 0;
893	else if (argv[0][0] == 'r')
894	is_return = 1;
895	else if (argv[0][0] == '-')
896	is_delete = 1;
897	else {
898	pr_info("Probe definition must be started with 'p', 'r' or"
899	" '-'.\n");
900	return -EINVAL;
901	}
902
903	if (argv[0][1] == ':') {
904	event = &argv[0][2];
905	if (strchr(event, '/')) {
906	group = event;
907	event = strchr(group, '/') + 1;
908	event[-1] = '\0';
909	if (strlen(group) == 0) {
910	pr_info("Group name is not specified\n");
911	return -EINVAL;
912	}
913	}
914	if (strlen(event) == 0) {
915	pr_info("Event name is not specified\n");
916	return -EINVAL;
917	}
918	}
919	if (!group)
920	group = KPROBE_EVENT_SYSTEM;
921
922	if (is_delete) {
923	if (!event) {
924	pr_info("Delete command needs an event name.\n");
925	return -EINVAL;
926	}
927	mutex_lock(&probe_lock);
928	tp = find_probe_event(event, group);
929	if (!tp) {
930	mutex_unlock(&probe_lock);
931	pr_info("Event %s/%s doesn't exist.\n", group, event);
932	return -ENOENT;
933	}
934	/* delete an event */
935	unregister_trace_probe(tp);
936	free_trace_probe(tp);
937	mutex_unlock(&probe_lock);
938	return 0;
939	}
940
941	if (argc < 2) {
942	pr_info("Probe point is not specified.\n");
943	return -EINVAL;
944	}
945	if (isdigit(argv[1][0])) {
946	if (is_return) {
947	pr_info("Return probe point must be a symbol.\n");
948	return -EINVAL;
949	}
950	/* an address specified */
951	ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr);
952	if (ret) {
953	pr_info("Failed to parse address.\n");
954	return ret;
955	}
956	} else {
957	/* a symbol specified */
958	symbol = argv[1];
959	/* TODO: support .init module functions */
960	ret = split_symbol_offset(symbol, &offset);
961	if (ret) {
962	pr_info("Failed to parse symbol.\n");
963	return ret;
964	}
965	if (offset && is_return) {
966	pr_info("Return probe must be used without offset.\n");
967	return -EINVAL;
968	}
969	}
970	argc -= 2; argv += 2;
971
972	/* setup a probe */
973	if (!event) {
974	/* Make a new event name */
975	if (symbol)
976	snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
977	is_return ? 'r' : 'p', symbol, offset);
978	else
979	snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
980	is_return ? 'r' : 'p', addr);
981	event = buf;
982	}
983	tp = alloc_trace_probe(group, event, addr, symbol, offset, argc,
984	is_return);
985	if (IS_ERR(tp)) {
986	pr_info("Failed to allocate trace_probe.(%d)\n",
987	(int)PTR_ERR(tp));
988	return PTR_ERR(tp);
989	}
990
991	/* parse arguments */
992	ret = 0;
993	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
994	/* Increment count for freeing args in error case */
995	tp->nr_args++;
996
997	/* Parse argument name */
998	arg = strchr(argv[i], '=');
999	if (arg) {
1000	*arg++ = '\0';
1001	tp->args[i].name = kstrdup(argv[i], GFP_KERNEL);
1002	} else {
1003	arg = argv[i];
1004	/* If argument name is omitted, set "argN" */
1005	snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
1006	tp->args[i].name = kstrdup(buf, GFP_KERNEL);
1007	}
1008
1009	if (!tp->args[i].name) {
1010	pr_info("Failed to allocate argument[%d] name.\n", i);
1011	ret = -ENOMEM;
1012	goto error;
1013	}
1014
1015	if (!is_good_name(tp->args[i].name)) {
1016	pr_info("Invalid argument[%d] name: %s\n",
1017	i, tp->args[i].name);
1018	ret = -EINVAL;
1019	goto error;
1020	}
1021
1022	if (conflict_field_name(tp->args[i].name, tp->args, i)) {
1023	pr_info("Argument[%d] name '%s' conflicts with "
1024	"another field.\n", i, argv[i]);
1025	ret = -EINVAL;
1026	goto error;
1027	}
1028
1029	/* Parse fetch argument */
1030	ret = parse_probe_arg(arg, tp, &tp->args[i], is_return);
1031	if (ret) {
1032	pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
1033	goto error;
1034	}
1035	}
1036
1037	ret = register_trace_probe(tp);
1038	if (ret)
1039	goto error;
1040	return 0;
1041
1042	error:
1043	free_trace_probe(tp);
1044	return ret;
1045	}
1046
1047	static void cleanup_all_probes(void)
1048	{
1049	struct trace_probe *tp;
1050
1051	mutex_lock(&probe_lock);
1052	/* TODO: Use batch unregistration */
1053	while (!list_empty(&probe_list)) {
1054	tp = list_entry(probe_list.next, struct trace_probe, list);
1055	unregister_trace_probe(tp);
1056	free_trace_probe(tp);
1057	}
1058	mutex_unlock(&probe_lock);
1059	}
1060
1061
1062	/* Probes listing interfaces */
1063	static void probes_seq_start(struct seq_file m, loff_t *pos)
1064	{
1065	mutex_lock(&probe_lock);
1066	return seq_list_start(&probe_list, *pos);
1067	}
1068
1069	static void probes_seq_next(struct seq_file m, void v, loff_t pos)
1070	{
1071	return seq_list_next(v, &probe_list, pos);
1072	}
1073
1074	static void probes_seq_stop(struct seq_file m, void v)
1075	{
1076	mutex_unlock(&probe_lock);
1077	}
1078
1079	static int probes_seq_show(struct seq_file m, void v)
1080	{
1081	struct trace_probe *tp = v;
1082	int i;
1083
1084	seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p');
1085	seq_printf(m, ":%s/%s", tp->call.class->system, tp->call.name);
1086
1087	if (!tp->symbol)
1088	seq_printf(m, " 0x%p", tp->rp.kp.addr);
1089	else if (tp->rp.kp.offset)
1090	seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset);
1091	else
1092	seq_printf(m, " %s", probe_symbol(tp));
1093
1094	for (i = 0; i < tp->nr_args; i++)
1095	seq_printf(m, " %s=%s", tp->args[i].name, tp->args[i].comm);
1096	seq_printf(m, "\n");
1097
1098	return 0;
1099	}
1100
1101	static const struct seq_operations probes_seq_op = {
1102	.start = probes_seq_start,
1103	.next = probes_seq_next,
1104	.stop = probes_seq_stop,
1105	.show = probes_seq_show
1106	};
1107
1108	static int probes_open(struct inode inode, struct file file)
1109	{
1110	if ((file->f_mode & FMODE_WRITE) &&
1111	(file->f_flags & O_TRUNC))
1112	cleanup_all_probes();
1113
1114	return seq_open(file, &probes_seq_op);
1115	}
1116
1117	static int command_trace_probe(const char *buf)
1118	{
1119	char **argv;
1120	int argc = 0, ret = 0;
1121
1122	argv = argv_split(GFP_KERNEL, buf, &argc);
1123	if (!argv)
1124	return -ENOMEM;
1125
1126	if (argc)
1127	ret = create_trace_probe(argc, argv);
1128
1129	argv_free(argv);
1130	return ret;
1131	}
1132
1133	#define WRITE_BUFSIZE 128
1134
1135	static ssize_t probes_write(struct file file, const char __user buffer,
1136	size_t count, loff_t *ppos)
1137	{
1138	char kbuf, tmp;
1139	int ret;
1140	size_t done;
1141	size_t size;
1142
1143	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
1144	if (!kbuf)
1145	return -ENOMEM;
1146
1147	ret = done = 0;
1148	while (done < count) {
1149	size = count - done;
1150	if (size >= WRITE_BUFSIZE)
1151	size = WRITE_BUFSIZE - 1;
1152	if (copy_from_user(kbuf, buffer + done, size)) {
1153	ret = -EFAULT;
1154	goto out;
1155	}
1156	kbuf[size] = '\0';
1157	tmp = strchr(kbuf, '\n');
1158	if (tmp) {
1159	*tmp = '\0';
1160	size = tmp - kbuf + 1;
1161	} else if (done + size < count) {
1162	pr_warning("Line length is too long: "
1163	"Should be less than %d.", WRITE_BUFSIZE);
1164	ret = -EINVAL;
1165	goto out;
1166	}
1167	done += size;
1168	/* Remove comments */
1169	tmp = strchr(kbuf, '#');
1170	if (tmp)
1171	*tmp = '\0';
1172
1173	ret = command_trace_probe(kbuf);
1174	if (ret)
1175	goto out;
1176	}
1177	ret = done;
1178	out:
1179	kfree(kbuf);
1180	return ret;
1181	}
1182
1183	static const struct file_operations kprobe_events_ops = {
1184	.owner = THIS_MODULE,
1185	.open = probes_open,
1186	.read = seq_read,
1187	.llseek = seq_lseek,
1188	.release = seq_release,
1189	.write = probes_write,
1190	};
1191
1192	/* Probes profiling interfaces */
1193	static int probes_profile_seq_show(struct seq_file m, void v)
1194	{
1195	struct trace_probe *tp = v;
1196
1197	seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit,
1198	tp->rp.kp.nmissed);
1199
1200	return 0;
1201	}
1202
1203	static const struct seq_operations profile_seq_op = {
1204	.start = probes_seq_start,
1205	.next = probes_seq_next,
1206	.stop = probes_seq_stop,
1207	.show = probes_profile_seq_show
1208	};
1209
1210	static int profile_open(struct inode inode, struct file file)
1211	{
1212	return seq_open(file, &profile_seq_op);
1213	}
1214
1215	static const struct file_operations kprobe_profile_ops = {
1216	.owner = THIS_MODULE,
1217	.open = profile_open,
1218	.read = seq_read,
1219	.llseek = seq_lseek,
1220	.release = seq_release,
1221	};
1222
1223	/* Sum up total data length for dynamic arraies (strings) */
1224	static __kprobes int __get_data_size(struct trace_probe *tp,
1225	struct pt_regs *regs)
1226	{
1227	int i, ret = 0;
1228	u32 len;
1229
1230	for (i = 0; i < tp->nr_args; i++)
1231	if (unlikely(tp->args[i].fetch_size.fn)) {
1232	call_fetch(&tp->args[i].fetch_size, regs, &len);
1233	ret += len;
1234	}
1235
1236	return ret;
1237	}
1238
1239	/* Store the value of each argument */
1240	static __kprobes void store_trace_args(int ent_size, struct trace_probe *tp,
1241	struct pt_regs *regs,
1242	u8 *data, int maxlen)
1243	{
1244	int i;
1245	u32 end = tp->size;
1246	u32 dl; / Data (relative) location */
1247
1248	for (i = 0; i < tp->nr_args; i++) {
1249	if (unlikely(tp->args[i].fetch_size.fn)) {
1250	/*
1251	* First, we set the relative location and
1252	* maximum data length to *dl
1253	*/
1254	dl = (u32 *)(data + tp->args[i].offset);
1255	*dl = make_data_rloc(maxlen, end - tp->args[i].offset);
1256	/* Then try to fetch string or dynamic array data */
1257	call_fetch(&tp->args[i].fetch, regs, dl);
1258	/* Reduce maximum length */
1259	end += get_rloc_len(*dl);
1260	maxlen -= get_rloc_len(*dl);
1261	/* Trick here, convert data_rloc to data_loc */
1262	dl = convert_rloc_to_loc(dl,
1263	ent_size + tp->args[i].offset);
1264	} else
1265	/* Just fetching data normally */
1266	call_fetch(&tp->args[i].fetch, regs,
1267	data + tp->args[i].offset);
1268	}
1269	}
1270
1271	/* Kprobe handler */
1272	static __kprobes void kprobe_trace_func(struct kprobe kp, struct pt_regs regs)
1273	{
1274	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1275	struct kprobe_trace_entry_head *entry;
1276	struct ring_buffer_event *event;
1277	struct ring_buffer *buffer;
1278	int size, dsize, pc;
1279	unsigned long irq_flags;
1280	struct ftrace_event_call *call = &tp->call;
1281
1282	tp->nhit++;
1283
1284	local_save_flags(irq_flags);
1285	pc = preempt_count();
1286
1287	dsize = __get_data_size(tp, regs);
1288	size = sizeof(*entry) + tp->size + dsize;
1289
1290	event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
1291	size, irq_flags, pc);
1292	if (!event)
1293	return;
1294
1295	entry = ring_buffer_event_data(event);
1296	entry->ip = (unsigned long)kp->addr;
1297	store_trace_args(sizeof(entry), tp, regs, (u8 )&entry[1], dsize);
1298
1299	if (!filter_current_check_discard(buffer, call, entry, event))
1300	trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
1301	}
1302
1303	/* Kretprobe handler */
1304	static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri,
1305	struct pt_regs *regs)
1306	{
1307	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1308	struct kretprobe_trace_entry_head *entry;
1309	struct ring_buffer_event *event;
1310	struct ring_buffer *buffer;
1311	int size, pc, dsize;
1312	unsigned long irq_flags;
1313	struct ftrace_event_call *call = &tp->call;
1314
1315	local_save_flags(irq_flags);
1316	pc = preempt_count();
1317
1318	dsize = __get_data_size(tp, regs);
1319	size = sizeof(*entry) + tp->size + dsize;
1320
1321	event = trace_current_buffer_lock_reserve(&buffer, call->event.type,
1322	size, irq_flags, pc);
1323	if (!event)
1324	return;
1325
1326	entry = ring_buffer_event_data(event);
1327	entry->func = (unsigned long)tp->rp.kp.addr;
1328	entry->ret_ip = (unsigned long)ri->ret_addr;
1329	store_trace_args(sizeof(entry), tp, regs, (u8 )&entry[1], dsize);
1330
1331	if (!filter_current_check_discard(buffer, call, entry, event))
1332	trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
1333	}
1334
1335	/* Event entry printers */
1336	enum print_line_t
1337	print_kprobe_event(struct trace_iterator *iter, int flags,
1338	struct trace_event *event)
1339	{
1340	struct kprobe_trace_entry_head *field;
1341	struct trace_seq *s = &iter->seq;
1342	struct trace_probe *tp;
1343	u8 *data;
1344	int i;
1345
1346	field = (struct kprobe_trace_entry_head *)iter->ent;
1347	tp = container_of(event, struct trace_probe, call.event);
1348
1349	if (!trace_seq_printf(s, "%s: (", tp->call.name))
1350	goto partial;
1351
1352	if (!seq_print_ip_sym(s, field->ip, flags \| TRACE_ITER_SYM_OFFSET))
1353	goto partial;
1354
1355	if (!trace_seq_puts(s, ")"))
1356	goto partial;
1357
1358	data = (u8 *)&field[1];
1359	for (i = 0; i < tp->nr_args; i++)
1360	if (!tp->args[i].type->print(s, tp->args[i].name,
1361	data + tp->args[i].offset, field))
1362	goto partial;
1363
1364	if (!trace_seq_puts(s, "\n"))
1365	goto partial;
1366
1367	return TRACE_TYPE_HANDLED;
1368	partial:
1369	return TRACE_TYPE_PARTIAL_LINE;
1370	}
1371
1372	enum print_line_t
1373	print_kretprobe_event(struct trace_iterator *iter, int flags,
1374	struct trace_event *event)
1375	{
1376	struct kretprobe_trace_entry_head *field;
1377	struct trace_seq *s = &iter->seq;
1378	struct trace_probe *tp;
1379	u8 *data;
1380	int i;
1381
1382	field = (struct kretprobe_trace_entry_head *)iter->ent;
1383	tp = container_of(event, struct trace_probe, call.event);
1384
1385	if (!trace_seq_printf(s, "%s: (", tp->call.name))
1386	goto partial;
1387
1388	if (!seq_print_ip_sym(s, field->ret_ip, flags \| TRACE_ITER_SYM_OFFSET))
1389	goto partial;
1390
1391	if (!trace_seq_puts(s, " <- "))
1392	goto partial;
1393
1394	if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1395	goto partial;
1396
1397	if (!trace_seq_puts(s, ")"))
1398	goto partial;
1399
1400	data = (u8 *)&field[1];
1401	for (i = 0; i < tp->nr_args; i++)
1402	if (!tp->args[i].type->print(s, tp->args[i].name,
1403	data + tp->args[i].offset, field))
1404	goto partial;
1405
1406	if (!trace_seq_puts(s, "\n"))
1407	goto partial;
1408
1409	return TRACE_TYPE_HANDLED;
1410	partial:
1411	return TRACE_TYPE_PARTIAL_LINE;
1412	}
1413
1414	static int probe_event_enable(struct ftrace_event_call *call)
1415	{
1416	struct trace_probe tp = (struct trace_probe )call->data;
1417
1418	tp->flags \|= TP_FLAG_TRACE;
1419	if (probe_is_return(tp))
1420	return enable_kretprobe(&tp->rp);
1421	else
1422	return enable_kprobe(&tp->rp.kp);
1423	}
1424
1425	static void probe_event_disable(struct ftrace_event_call *call)
1426	{
1427	struct trace_probe tp = (struct trace_probe )call->data;
1428
1429	tp->flags &= ~TP_FLAG_TRACE;
1430	if (!(tp->flags & (TP_FLAG_TRACE \| TP_FLAG_PROFILE))) {
1431	if (probe_is_return(tp))
1432	disable_kretprobe(&tp->rp);
1433	else
1434	disable_kprobe(&tp->rp.kp);
1435	}
1436	}
1437
1438	#undef DEFINE_FIELD
1439	#define DEFINE_FIELD(type, item, name, is_signed) \
1440	do { \
1441	ret = trace_define_field(event_call, #type, name, \
1442	offsetof(typeof(field), item), \
1443	sizeof(field.item), is_signed, \
1444	FILTER_OTHER); \
1445	if (ret) \
1446	return ret; \
1447	} while (0)
1448
1449	static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
1450	{
1451	int ret, i;
1452	struct kprobe_trace_entry_head field;
1453	struct trace_probe tp = (struct trace_probe )event_call->data;
1454
1455	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1456	/* Set argument names as fields */
1457	for (i = 0; i < tp->nr_args; i++) {
1458	ret = trace_define_field(event_call, tp->args[i].type->fmttype,
1459	tp->args[i].name,
1460	sizeof(field) + tp->args[i].offset,
1461	tp->args[i].type->size,
1462	tp->args[i].type->is_signed,
1463	FILTER_OTHER);
1464	if (ret)
1465	return ret;
1466	}
1467	return 0;
1468	}
1469
1470	static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
1471	{
1472	int ret, i;
1473	struct kretprobe_trace_entry_head field;
1474	struct trace_probe tp = (struct trace_probe )event_call->data;
1475
1476	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1477	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1478	/* Set argument names as fields */
1479	for (i = 0; i < tp->nr_args; i++) {
1480	ret = trace_define_field(event_call, tp->args[i].type->fmttype,
1481	tp->args[i].name,
1482	sizeof(field) + tp->args[i].offset,
1483	tp->args[i].type->size,
1484	tp->args[i].type->is_signed,
1485	FILTER_OTHER);
1486	if (ret)
1487	return ret;
1488	}
1489	return 0;
1490	}
1491
1492	static int __set_print_fmt(struct trace_probe tp, char buf, int len)
1493	{
1494	int i;
1495	int pos = 0;
1496
1497	const char fmt, arg;
1498
1499	if (!probe_is_return(tp)) {
1500	fmt = "(%lx)";
1501	arg = "REC->" FIELD_STRING_IP;
1502	} else {
1503	fmt = "(%lx <- %lx)";
1504	arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1505	}
1506
1507	/* When len=0, we just calculate the needed length */
1508	#define LEN_OR_ZERO (len ? len - pos : 0)
1509
1510	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1511
1512	for (i = 0; i < tp->nr_args; i++) {
1513	pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s",
1514	tp->args[i].name, tp->args[i].type->fmt);
1515	}
1516
1517	pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
1518
1519	for (i = 0; i < tp->nr_args; i++) {
1520	if (strcmp(tp->args[i].type->name, "string") == 0)
1521	pos += snprintf(buf + pos, LEN_OR_ZERO,
1522	", __get_str(%s)",
1523	tp->args[i].name);
1524	else
1525	pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
1526	tp->args[i].name);
1527	}
1528
1529	#undef LEN_OR_ZERO
1530
1531	/* return the length of print_fmt */
1532	return pos;
1533	}
1534
1535	static int set_print_fmt(struct trace_probe *tp)
1536	{
1537	int len;
1538	char *print_fmt;
1539
1540	/* First: called with 0 length to calculate the needed length */
1541	len = __set_print_fmt(tp, NULL, 0);
1542	print_fmt = kmalloc(len + 1, GFP_KERNEL);
1543	if (!print_fmt)
1544	return -ENOMEM;
1545
1546	/* Second: actually write the @print_fmt */
1547	__set_print_fmt(tp, print_fmt, len + 1);
1548	tp->call.print_fmt = print_fmt;
1549
1550	return 0;
1551	}
1552
1553	#ifdef CONFIG_PERF_EVENTS
1554
1555	/* Kprobe profile handler */
1556	static __kprobes void kprobe_perf_func(struct kprobe *kp,
1557	struct pt_regs *regs)
1558	{
1559	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1560	struct ftrace_event_call *call = &tp->call;
1561	struct kprobe_trace_entry_head *entry;
1562	struct hlist_head *head;
1563	int size, __size, dsize;
1564	int rctx;
1565
1566	dsize = __get_data_size(tp, regs);
1567	__size = sizeof(*entry) + tp->size + dsize;
1568	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1569	size -= sizeof(u32);
1570	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
1571	"profile buffer not large enough"))
1572	return;
1573
1574	entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
1575	if (!entry)
1576	return;
1577
1578	entry->ip = (unsigned long)kp->addr;
1579	memset(&entry[1], 0, dsize);
1580	store_trace_args(sizeof(entry), tp, regs, (u8 )&entry[1], dsize);
1581
1582	head = this_cpu_ptr(call->perf_events);
1583	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
1584	}
1585
1586	/* Kretprobe profile handler */
1587	static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
1588	struct pt_regs *regs)
1589	{
1590	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1591	struct ftrace_event_call *call = &tp->call;
1592	struct kretprobe_trace_entry_head *entry;
1593	struct hlist_head *head;
1594	int size, __size, dsize;
1595	int rctx;
1596
1597	dsize = __get_data_size(tp, regs);
1598	__size = sizeof(*entry) + tp->size + dsize;
1599	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1600	size -= sizeof(u32);
1601	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
1602	"profile buffer not large enough"))
1603	return;
1604
1605	entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
1606	if (!entry)
1607	return;
1608
1609	entry->func = (unsigned long)tp->rp.kp.addr;
1610	entry->ret_ip = (unsigned long)ri->ret_addr;
1611	store_trace_args(sizeof(entry), tp, regs, (u8 )&entry[1], dsize);
1612
1613	head = this_cpu_ptr(call->perf_events);
1614	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
1615	}
1616
1617	static int probe_perf_enable(struct ftrace_event_call *call)
1618	{
1619	struct trace_probe tp = (struct trace_probe )call->data;
1620
1621	tp->flags \|= TP_FLAG_PROFILE;
1622
1623	if (probe_is_return(tp))
1624	return enable_kretprobe(&tp->rp);
1625	else
1626	return enable_kprobe(&tp->rp.kp);
1627	}
1628
1629	static void probe_perf_disable(struct ftrace_event_call *call)
1630	{
1631	struct trace_probe tp = (struct trace_probe )call->data;
1632
1633	tp->flags &= ~TP_FLAG_PROFILE;
1634
1635	if (!(tp->flags & TP_FLAG_TRACE)) {
1636	if (probe_is_return(tp))
1637	disable_kretprobe(&tp->rp);
1638	else
1639	disable_kprobe(&tp->rp.kp);
1640	}
1641	}
1642	#endif /* CONFIG_PERF_EVENTS */
1643
1644	static __kprobes
1645	int kprobe_register(struct ftrace_event_call *event, enum trace_reg type)
1646	{
1647	switch (type) {
1648	case TRACE_REG_REGISTER:
1649	return probe_event_enable(event);
1650	case TRACE_REG_UNREGISTER:
1651	probe_event_disable(event);
1652	return 0;
1653
1654	#ifdef CONFIG_PERF_EVENTS
1655	case TRACE_REG_PERF_REGISTER:
1656	return probe_perf_enable(event);
1657	case TRACE_REG_PERF_UNREGISTER:
1658	probe_perf_disable(event);
1659	return 0;
1660	#endif
1661	}
1662	return 0;
1663	}
1664
1665	static __kprobes
1666	int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs)
1667	{
1668	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1669
1670	if (tp->flags & TP_FLAG_TRACE)
1671	kprobe_trace_func(kp, regs);
1672	#ifdef CONFIG_PERF_EVENTS
1673	if (tp->flags & TP_FLAG_PROFILE)
1674	kprobe_perf_func(kp, regs);
1675	#endif
1676	return 0; /* We don't tweek kernel, so just return 0 */
1677	}
1678
1679	static __kprobes
1680	int kretprobe_dispatcher(struct kretprobe_instance ri, struct pt_regs regs)
1681	{
1682	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1683
1684	if (tp->flags & TP_FLAG_TRACE)
1685	kretprobe_trace_func(ri, regs);
1686	#ifdef CONFIG_PERF_EVENTS
1687	if (tp->flags & TP_FLAG_PROFILE)
1688	kretprobe_perf_func(ri, regs);
1689	#endif
1690	return 0; /* We don't tweek kernel, so just return 0 */
1691	}
1692
1693	static struct trace_event_functions kretprobe_funcs = {
1694	.trace = print_kretprobe_event
1695	};
1696
1697	static struct trace_event_functions kprobe_funcs = {
1698	.trace = print_kprobe_event
1699	};
1700
1701	static int register_probe_event(struct trace_probe *tp)
1702	{
1703	struct ftrace_event_call *call = &tp->call;
1704	int ret;
1705
1706	/* Initialize ftrace_event_call */
1707	INIT_LIST_HEAD(&call->class->fields);
1708	if (probe_is_return(tp)) {
1709	call->event.funcs = &kretprobe_funcs;
1710	call->class->define_fields = kretprobe_event_define_fields;
1711	} else {
1712	call->event.funcs = &kprobe_funcs;
1713	call->class->define_fields = kprobe_event_define_fields;
1714	}
1715	if (set_print_fmt(tp) < 0)
1716	return -ENOMEM;
1717	ret = register_ftrace_event(&call->event);
1718	if (!ret) {
1719	kfree(call->print_fmt);
1720	return -ENODEV;
1721	}
1722	call->flags = 0;
1723	call->class->reg = kprobe_register;
1724	call->data = tp;
1725	ret = trace_add_event_call(call);
1726	if (ret) {
1727	pr_info("Failed to register kprobe event: %s\n", call->name);
1728	kfree(call->print_fmt);
1729	unregister_ftrace_event(&call->event);
1730	}
1731	return ret;
1732	}
1733
1734	static void unregister_probe_event(struct trace_probe *tp)
1735	{
1736	/* tp->event is unregistered in trace_remove_event_call() */
1737	trace_remove_event_call(&tp->call);
1738	kfree(tp->call.print_fmt);
1739	}
1740
1741	/* Make a debugfs interface for controling probe points */
1742	static __init int init_kprobe_trace(void)
1743	{
1744	struct dentry *d_tracer;
1745	struct dentry *entry;
1746
1747	d_tracer = tracing_init_dentry();
1748	if (!d_tracer)
1749	return 0;
1750
1751	entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1752	NULL, &kprobe_events_ops);
1753
1754	/* Event list interface */
1755	if (!entry)
1756	pr_warning("Could not create debugfs "
1757	"'kprobe_events' entry\n");
1758
1759	/* Profile interface */
1760	entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1761	NULL, &kprobe_profile_ops);
1762
1763	if (!entry)
1764	pr_warning("Could not create debugfs "
1765	"'kprobe_profile' entry\n");
1766	return 0;
1767	}
1768	fs_initcall(init_kprobe_trace);
1769
1770
1771	#ifdef CONFIG_FTRACE_STARTUP_TEST
1772
1773	static int kprobe_trace_selftest_target(int a1, int a2, int a3,
1774	int a4, int a5, int a6)
1775	{
1776	return a1 + a2 + a3 + a4 + a5 + a6;
1777	}
1778
1779	static __init int kprobe_trace_self_tests_init(void)
1780	{
1781	int ret, warn = 0;
1782	int (*target)(int, int, int, int, int, int);
1783	struct trace_probe *tp;
1784
1785	target = kprobe_trace_selftest_target;
1786
1787	pr_info("Testing kprobe tracing: ");
1788
1789	ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1790	"$stack $stack0 +0($stack)");
1791	if (WARN_ON_ONCE(ret)) {
1792	pr_warning("error on probing function entry.\n");
1793	warn++;
1794	} else {
1795	/* Enable trace point */
1796	tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM);
1797	if (WARN_ON_ONCE(tp == NULL)) {
1798	pr_warning("error on getting new probe.\n");
1799	warn++;
1800	} else
1801	probe_event_enable(&tp->call);
1802	}
1803
1804	ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1805	"$retval");
1806	if (WARN_ON_ONCE(ret)) {
1807	pr_warning("error on probing function return.\n");
1808	warn++;
1809	} else {
1810	/* Enable trace point */
1811	tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM);
1812	if (WARN_ON_ONCE(tp == NULL)) {
1813	pr_warning("error on getting new probe.\n");
1814	warn++;
1815	} else
1816	probe_event_enable(&tp->call);
1817	}
1818
1819	if (warn)
1820	goto end;
1821
1822	ret = target(1, 2, 3, 4, 5, 6);
1823
1824	ret = command_trace_probe("-:testprobe");
1825	if (WARN_ON_ONCE(ret)) {
1826	pr_warning("error on deleting a probe.\n");
1827	warn++;
1828	}
1829
1830	ret = command_trace_probe("-:testprobe2");
1831	if (WARN_ON_ONCE(ret)) {
1832	pr_warning("error on deleting a probe.\n");
1833	warn++;
1834	}
1835
1836	end:
1837	cleanup_all_probes();
1838	if (warn)
1839	pr_cont("NG: Some tests are failed. Please check them.\n");
1840	else
1841	pr_cont("OK\n");
1842	return 0;
1843	}
1844
1845	late_initcall(kprobe_trace_self_tests_init);
1846
1847	#endif
1848

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/kernel/trace/trace_kprobe.c

interactive