Kernel

Kernel Git Source Tree

Root/kernel/kprobes.c

1	/*
2	* Kernel Probes (KProbes)
3	* kernel/kprobes.c
4	*
5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License as published by
7	* the Free Software Foundation; either version 2 of the License, or
8	* (at your option) any later version.
9	*
10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.
14	*
15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software
17	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18	*
19	* Copyright (C) IBM Corporation, 2002, 2004
20	*
21	* 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22	* Probes initial implementation (includes suggestions from
23	* Rusty Russell).
24	* 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25	* hlists and exceptions notifier as suggested by Andi Kleen.
26	* 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27	* interface to access function arguments.
28	* 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29	* exceptions notifier to be first on the priority list.
30	* 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31	* <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32	* <prasanna@in.ibm.com> added function-return probes.
33	*/
34	#include <linux/kprobes.h>
35	#include <linux/hash.h>
36	#include <linux/init.h>
37	#include <linux/slab.h>
38	#include <linux/stddef.h>
39	#include <linux/module.h>
40	#include <linux/moduleloader.h>
41	#include <linux/kallsyms.h>
42	#include <linux/freezer.h>
43	#include <linux/seq_file.h>
44	#include <linux/debugfs.h>
45	#include <linux/sysctl.h>
46	#include <linux/kdebug.h>
47	#include <linux/memory.h>
48	#include <linux/ftrace.h>
49	#include <linux/cpu.h>
50	#include <linux/jump_label.h>
51
52	#include <asm-generic/sections.h>
53	#include <asm/cacheflush.h>
54	#include <asm/errno.h>
55	#include <asm/uaccess.h>
56
57	#define KPROBE_HASH_BITS 6
58	#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
60
61	/*
62	* Some oddball architectures like 64bit powerpc have function descriptors
63	* so this must be overridable.
64	*/
65	#ifndef kprobe_lookup_name
66	#define kprobe_lookup_name(name, addr) \
67	addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
68	#endif
69
70	static int kprobes_initialized;
71	static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
72	static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
73
74	/* NOTE: change this value only with kprobe_mutex held */
75	static bool kprobes_all_disarmed;
76
77	/* This protects kprobe_table and optimizing_list */
78	static DEFINE_MUTEX(kprobe_mutex);
79	static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
80	static struct {
81	spinlock_t lock ____cacheline_aligned_in_smp;
82	} kretprobe_table_locks[KPROBE_TABLE_SIZE];
83
84	static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
85	{
86	return &(kretprobe_table_locks[hash].lock);
87	}
88
89	/*
90	* Normally, functions that we'd want to prohibit kprobes in, are marked
91	* __kprobes. But, there are cases where such functions already belong to
92	* a different section (__sched for preempt_schedule)
93	*
94	* For such cases, we now have a blacklist
95	*/
96	static struct kprobe_blackpoint kprobe_blacklist[] = {
97	{"preempt_schedule",},
98	{"native_get_debugreg",},
99	{"irq_entries_start",},
100	{"common_interrupt",},
101	{"mcount",}, /* mcount can be called from everywhere */
102	{NULL} /* Terminator */
103	};
104
105	#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
106	/*
107	* kprobe->ainsn.insn points to the copy of the instruction to be
108	* single-stepped. x86_64, POWER4 and above have no-exec support and
109	* stepping on the instruction on a vmalloced/kmalloced/data page
110	* is a recipe for disaster
111	*/
112	struct kprobe_insn_page {
113	struct list_head list;
114	kprobe_opcode_t insns; / Page of instruction slots */
115	int nused;
116	int ngarbage;
117	char slot_used[];
118	};
119
120	#define KPROBE_INSN_PAGE_SIZE(slots) \
121	(offsetof(struct kprobe_insn_page, slot_used) + \
122	(sizeof(char) * (slots)))
123
124	struct kprobe_insn_cache {
125	struct list_head pages; /* list of kprobe_insn_page */
126	size_t insn_size; /* size of instruction slot */
127	int nr_garbage;
128	};
129
130	static int slots_per_page(struct kprobe_insn_cache *c)
131	{
132	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
133	}
134
135	enum kprobe_slot_state {
136	SLOT_CLEAN = 0,
137	SLOT_DIRTY = 1,
138	SLOT_USED = 2,
139	};
140
141	static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
142	static struct kprobe_insn_cache kprobe_insn_slots = {
143	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
144	.insn_size = MAX_INSN_SIZE,
145	.nr_garbage = 0,
146	};
147	static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
148
149	/**
150	* __get_insn_slot() - Find a slot on an executable page for an instruction.
151	* We allocate an executable page if there's no room on existing ones.
152	*/
153	static kprobe_opcode_t __kprobes __get_insn_slot(struct kprobe_insn_cache c)
154	{
155	struct kprobe_insn_page *kip;
156
157	retry:
158	list_for_each_entry(kip, &c->pages, list) {
159	if (kip->nused < slots_per_page(c)) {
160	int i;
161	for (i = 0; i < slots_per_page(c); i++) {
162	if (kip->slot_used[i] == SLOT_CLEAN) {
163	kip->slot_used[i] = SLOT_USED;
164	kip->nused++;
165	return kip->insns + (i * c->insn_size);
166	}
167	}
168	/* kip->nused is broken. Fix it. */
169	kip->nused = slots_per_page(c);
170	WARN_ON(1);
171	}
172	}
173
174	/* If there are any garbage slots, collect it and try again. */
175	if (c->nr_garbage && collect_garbage_slots(c) == 0)
176	goto retry;
177
178	/* All out of space. Need to allocate a new page. */
179	kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
180	if (!kip)
181	return NULL;
182
183	/*
184	* Use module_alloc so this page is within +/- 2GB of where the
185	* kernel image and loaded module images reside. This is required
186	* so x86_64 can correctly handle the %rip-relative fixups.
187	*/
188	kip->insns = module_alloc(PAGE_SIZE);
189	if (!kip->insns) {
190	kfree(kip);
191	return NULL;
192	}
193	INIT_LIST_HEAD(&kip->list);
194	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
195	kip->slot_used[0] = SLOT_USED;
196	kip->nused = 1;
197	kip->ngarbage = 0;
198	list_add(&kip->list, &c->pages);
199	return kip->insns;
200	}
201
202
203	kprobe_opcode_t __kprobes *get_insn_slot(void)
204	{
205	kprobe_opcode_t *ret = NULL;
206
207	mutex_lock(&kprobe_insn_mutex);
208	ret = __get_insn_slot(&kprobe_insn_slots);
209	mutex_unlock(&kprobe_insn_mutex);
210
211	return ret;
212	}
213
214	/* Return 1 if all garbages are collected, otherwise 0. */
215	static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
216	{
217	kip->slot_used[idx] = SLOT_CLEAN;
218	kip->nused--;
219	if (kip->nused == 0) {
220	/*
221	* Page is no longer in use. Free it unless
222	* it's the last one. We keep the last one
223	* so as not to have to set it up again the
224	* next time somebody inserts a probe.
225	*/
226	if (!list_is_singular(&kip->list)) {
227	list_del(&kip->list);
228	module_free(NULL, kip->insns);
229	kfree(kip);
230	}
231	return 1;
232	}
233	return 0;
234	}
235
236	static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
237	{
238	struct kprobe_insn_page kip, next;
239
240	/* Ensure no-one is interrupted on the garbages */
241	synchronize_sched();
242
243	list_for_each_entry_safe(kip, next, &c->pages, list) {
244	int i;
245	if (kip->ngarbage == 0)
246	continue;
247	kip->ngarbage = 0; /* we will collect all garbages */
248	for (i = 0; i < slots_per_page(c); i++) {
249	if (kip->slot_used[i] == SLOT_DIRTY &&
250	collect_one_slot(kip, i))
251	break;
252	}
253	}
254	c->nr_garbage = 0;
255	return 0;
256	}
257
258	static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
259	kprobe_opcode_t *slot, int dirty)
260	{
261	struct kprobe_insn_page *kip;
262
263	list_for_each_entry(kip, &c->pages, list) {
264	long idx = ((long)slot - (long)kip->insns) /
265	(c->insn_size * sizeof(kprobe_opcode_t));
266	if (idx >= 0 && idx < slots_per_page(c)) {
267	WARN_ON(kip->slot_used[idx] != SLOT_USED);
268	if (dirty) {
269	kip->slot_used[idx] = SLOT_DIRTY;
270	kip->ngarbage++;
271	if (++c->nr_garbage > slots_per_page(c))
272	collect_garbage_slots(c);
273	} else
274	collect_one_slot(kip, idx);
275	return;
276	}
277	}
278	/* Could not free this slot. */
279	WARN_ON(1);
280	}
281
282	void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
283	{
284	mutex_lock(&kprobe_insn_mutex);
285	__free_insn_slot(&kprobe_insn_slots, slot, dirty);
286	mutex_unlock(&kprobe_insn_mutex);
287	}
288	#ifdef CONFIG_OPTPROBES
289	/* For optimized_kprobe buffer */
290	static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
291	static struct kprobe_insn_cache kprobe_optinsn_slots = {
292	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
293	/* .insn_size is initialized later */
294	.nr_garbage = 0,
295	};
296	/* Get a slot for optimized_kprobe buffer */
297	kprobe_opcode_t __kprobes *get_optinsn_slot(void)
298	{
299	kprobe_opcode_t *ret = NULL;
300
301	mutex_lock(&kprobe_optinsn_mutex);
302	ret = __get_insn_slot(&kprobe_optinsn_slots);
303	mutex_unlock(&kprobe_optinsn_mutex);
304
305	return ret;
306	}
307
308	void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
309	{
310	mutex_lock(&kprobe_optinsn_mutex);
311	__free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
312	mutex_unlock(&kprobe_optinsn_mutex);
313	}
314	#endif
315	#endif
316
317	/* We have preemption disabled.. so it is safe to use __ versions */
318	static inline void set_kprobe_instance(struct kprobe *kp)
319	{
320	__this_cpu_write(kprobe_instance, kp);
321	}
322
323	static inline void reset_kprobe_instance(void)
324	{
325	__this_cpu_write(kprobe_instance, NULL);
326	}
327
328	/*
329	* This routine is called either:
330	* - under the kprobe_mutex - during kprobe_[un]register()
331	* OR
332	* - with preemption disabled - from arch/xxx/kernel/kprobes.c
333	*/
334	struct kprobe __kprobes get_kprobe(void addr)
335	{
336	struct hlist_head *head;
337	struct hlist_node *node;
338	struct kprobe *p;
339
340	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
341	hlist_for_each_entry_rcu(p, node, head, hlist) {
342	if (p->addr == addr)
343	return p;
344	}
345
346	return NULL;
347	}
348
349	static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs);
350
351	/* Return true if the kprobe is an aggregator */
352	static inline int kprobe_aggrprobe(struct kprobe *p)
353	{
354	return p->pre_handler == aggr_pre_handler;
355	}
356
357	/* Return true(!0) if the kprobe is unused */
358	static inline int kprobe_unused(struct kprobe *p)
359	{
360	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
361	list_empty(&p->list);
362	}
363
364	/*
365	* Keep all fields in the kprobe consistent
366	*/
367	static inline void copy_kprobe(struct kprobe ap, struct kprobe p)
368	{
369	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
370	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
371	}
372
373	#ifdef CONFIG_OPTPROBES
374	/* NOTE: change this value only with kprobe_mutex held */
375	static bool kprobes_allow_optimization;
376
377	/*
378	* Call all pre_handler on the list, but ignores its return value.
379	* This must be called from arch-dep optimized caller.
380	*/
381	void __kprobes opt_pre_handler(struct kprobe p, struct pt_regs regs)
382	{
383	struct kprobe *kp;
384
385	list_for_each_entry_rcu(kp, &p->list, list) {
386	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
387	set_kprobe_instance(kp);
388	kp->pre_handler(kp, regs);
389	}
390	reset_kprobe_instance();
391	}
392	}
393
394	/* Free optimized instructions and optimized_kprobe */
395	static __kprobes void free_aggr_kprobe(struct kprobe *p)
396	{
397	struct optimized_kprobe *op;
398
399	op = container_of(p, struct optimized_kprobe, kp);
400	arch_remove_optimized_kprobe(op);
401	arch_remove_kprobe(p);
402	kfree(op);
403	}
404
405	/* Return true(!0) if the kprobe is ready for optimization. */
406	static inline int kprobe_optready(struct kprobe *p)
407	{
408	struct optimized_kprobe *op;
409
410	if (kprobe_aggrprobe(p)) {
411	op = container_of(p, struct optimized_kprobe, kp);
412	return arch_prepared_optinsn(&op->optinsn);
413	}
414
415	return 0;
416	}
417
418	/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
419	static inline int kprobe_disarmed(struct kprobe *p)
420	{
421	struct optimized_kprobe *op;
422
423	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */
424	if (!kprobe_aggrprobe(p))
425	return kprobe_disabled(p);
426
427	op = container_of(p, struct optimized_kprobe, kp);
428
429	return kprobe_disabled(p) && list_empty(&op->list);
430	}
431
432	/* Return true(!0) if the probe is queued on (un)optimizing lists */
433	static int __kprobes kprobe_queued(struct kprobe *p)
434	{
435	struct optimized_kprobe *op;
436
437	if (kprobe_aggrprobe(p)) {
438	op = container_of(p, struct optimized_kprobe, kp);
439	if (!list_empty(&op->list))
440	return 1;
441	}
442	return 0;
443	}
444
445	/*
446	* Return an optimized kprobe whose optimizing code replaces
447	* instructions including addr (exclude breakpoint).
448	*/
449	static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
450	{
451	int i;
452	struct kprobe *p = NULL;
453	struct optimized_kprobe *op;
454
455	/* Don't check i == 0, since that is a breakpoint case. */
456	for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
457	p = get_kprobe((void *)(addr - i));
458
459	if (p && kprobe_optready(p)) {
460	op = container_of(p, struct optimized_kprobe, kp);
461	if (arch_within_optimized_kprobe(op, addr))
462	return p;
463	}
464
465	return NULL;
466	}
467
468	/* Optimization staging list, protected by kprobe_mutex */
469	static LIST_HEAD(optimizing_list);
470	static LIST_HEAD(unoptimizing_list);
471
472	static void kprobe_optimizer(struct work_struct *work);
473	static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
474	static DECLARE_COMPLETION(optimizer_comp);
475	#define OPTIMIZE_DELAY 5
476
477	/*
478	* Optimize (replace a breakpoint with a jump) kprobes listed on
479	* optimizing_list.
480	*/
481	static __kprobes void do_optimize_kprobes(void)
482	{
483	/* Optimization never be done when disarmed */
484	if (kprobes_all_disarmed \|\| !kprobes_allow_optimization \|\|
485	list_empty(&optimizing_list))
486	return;
487
488	/*
489	* The optimization/unoptimization refers online_cpus via
490	* stop_machine() and cpu-hotplug modifies online_cpus.
491	* And same time, text_mutex will be held in cpu-hotplug and here.
492	* This combination can cause a deadlock (cpu-hotplug try to lock
493	* text_mutex but stop_machine can not be done because online_cpus
494	* has been changed)
495	* To avoid this deadlock, we need to call get_online_cpus()
496	* for preventing cpu-hotplug outside of text_mutex locking.
497	*/
498	get_online_cpus();
499	mutex_lock(&text_mutex);
500	arch_optimize_kprobes(&optimizing_list);
501	mutex_unlock(&text_mutex);
502	put_online_cpus();
503	}
504
505	/*
506	* Unoptimize (replace a jump with a breakpoint and remove the breakpoint
507	* if need) kprobes listed on unoptimizing_list.
508	*/
509	static __kprobes void do_unoptimize_kprobes(struct list_head *free_list)
510	{
511	struct optimized_kprobe op, tmp;
512
513	/* Unoptimization must be done anytime */
514	if (list_empty(&unoptimizing_list))
515	return;
516
517	/* Ditto to do_optimize_kprobes */
518	get_online_cpus();
519	mutex_lock(&text_mutex);
520	arch_unoptimize_kprobes(&unoptimizing_list, free_list);
521	/* Loop free_list for disarming */
522	list_for_each_entry_safe(op, tmp, free_list, list) {
523	/* Disarm probes if marked disabled */
524	if (kprobe_disabled(&op->kp))
525	arch_disarm_kprobe(&op->kp);
526	if (kprobe_unused(&op->kp)) {
527	/*
528	* Remove unused probes from hash list. After waiting
529	* for synchronization, these probes are reclaimed.
530	* (reclaiming is done by do_free_cleaned_kprobes.)
531	*/
532	hlist_del_rcu(&op->kp.hlist);
533	} else
534	list_del_init(&op->list);
535	}
536	mutex_unlock(&text_mutex);
537	put_online_cpus();
538	}
539
540	/* Reclaim all kprobes on the free_list */
541	static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list)
542	{
543	struct optimized_kprobe op, tmp;
544
545	list_for_each_entry_safe(op, tmp, free_list, list) {
546	BUG_ON(!kprobe_unused(&op->kp));
547	list_del_init(&op->list);
548	free_aggr_kprobe(&op->kp);
549	}
550	}
551
552	/* Start optimizer after OPTIMIZE_DELAY passed */
553	static __kprobes void kick_kprobe_optimizer(void)
554	{
555	if (!delayed_work_pending(&optimizing_work))
556	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
557	}
558
559	/* Kprobe jump optimizer */
560	static __kprobes void kprobe_optimizer(struct work_struct *work)
561	{
562	LIST_HEAD(free_list);
563
564	/* Lock modules while optimizing kprobes */
565	mutex_lock(&module_mutex);
566	mutex_lock(&kprobe_mutex);
567
568	/*
569	* Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
570	* kprobes before waiting for quiesence period.
571	*/
572	do_unoptimize_kprobes(&free_list);
573
574	/*
575	* Step 2: Wait for quiesence period to ensure all running interrupts
576	* are done. Because optprobe may modify multiple instructions
577	* there is a chance that Nth instruction is interrupted. In that
578	* case, running interrupt can return to 2nd-Nth byte of jump
579	* instruction. This wait is for avoiding it.
580	*/
581	synchronize_sched();
582
583	/* Step 3: Optimize kprobes after quiesence period */
584	do_optimize_kprobes();
585
586	/* Step 4: Free cleaned kprobes after quiesence period */
587	do_free_cleaned_kprobes(&free_list);
588
589	mutex_unlock(&kprobe_mutex);
590	mutex_unlock(&module_mutex);
591
592	/* Step 5: Kick optimizer again if needed */
593	if (!list_empty(&optimizing_list) \|\| !list_empty(&unoptimizing_list))
594	kick_kprobe_optimizer();
595	else
596	/* Wake up all waiters */
597	complete_all(&optimizer_comp);
598	}
599
600	/* Wait for completing optimization and unoptimization */
601	static __kprobes void wait_for_kprobe_optimizer(void)
602	{
603	if (delayed_work_pending(&optimizing_work))
604	wait_for_completion(&optimizer_comp);
605	}
606
607	/* Optimize kprobe if p is ready to be optimized */
608	static __kprobes void optimize_kprobe(struct kprobe *p)
609	{
610	struct optimized_kprobe *op;
611
612	/* Check if the kprobe is disabled or not ready for optimization. */
613	if (!kprobe_optready(p) \|\| !kprobes_allow_optimization \|\|
614	(kprobe_disabled(p) \|\| kprobes_all_disarmed))
615	return;
616
617	/* Both of break_handler and post_handler are not supported. */
618	if (p->break_handler \|\| p->post_handler)
619	return;
620
621	op = container_of(p, struct optimized_kprobe, kp);
622
623	/* Check there is no other kprobes at the optimized instructions */
624	if (arch_check_optimized_kprobe(op) < 0)
625	return;
626
627	/* Check if it is already optimized. */
628	if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
629	return;
630	op->kp.flags \|= KPROBE_FLAG_OPTIMIZED;
631
632	if (!list_empty(&op->list))
633	/* This is under unoptimizing. Just dequeue the probe */
634	list_del_init(&op->list);
635	else {
636	list_add(&op->list, &optimizing_list);
637	kick_kprobe_optimizer();
638	}
639	}
640
641	/* Short cut to direct unoptimizing */
642	static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)
643	{
644	get_online_cpus();
645	arch_unoptimize_kprobe(op);
646	put_online_cpus();
647	if (kprobe_disabled(&op->kp))
648	arch_disarm_kprobe(&op->kp);
649	}
650
651	/* Unoptimize a kprobe if p is optimized */
652	static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)
653	{
654	struct optimized_kprobe *op;
655
656	if (!kprobe_aggrprobe(p) \|\| kprobe_disarmed(p))
657	return; /* This is not an optprobe nor optimized */
658
659	op = container_of(p, struct optimized_kprobe, kp);
660	if (!kprobe_optimized(p)) {
661	/* Unoptimized or unoptimizing case */
662	if (force && !list_empty(&op->list)) {
663	/*
664	* Only if this is unoptimizing kprobe and forced,
665	* forcibly unoptimize it. (No need to unoptimize
666	* unoptimized kprobe again :)
667	*/
668	list_del_init(&op->list);
669	force_unoptimize_kprobe(op);
670	}
671	return;
672	}
673
674	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
675	if (!list_empty(&op->list)) {
676	/* Dequeue from the optimization queue */
677	list_del_init(&op->list);
678	return;
679	}
680	/* Optimized kprobe case */
681	if (force)
682	/* Forcibly update the code: this is a special case */
683	force_unoptimize_kprobe(op);
684	else {
685	list_add(&op->list, &unoptimizing_list);
686	kick_kprobe_optimizer();
687	}
688	}
689
690	/* Cancel unoptimizing for reusing */
691	static void reuse_unused_kprobe(struct kprobe *ap)
692	{
693	struct optimized_kprobe *op;
694
695	BUG_ON(!kprobe_unused(ap));
696	/*
697	* Unused kprobe MUST be on the way of delayed unoptimizing (means
698	* there is still a relative jump) and disabled.
699	*/
700	op = container_of(ap, struct optimized_kprobe, kp);
701	if (unlikely(list_empty(&op->list)))
702	printk(KERN_WARNING "Warning: found a stray unused "
703	"aggrprobe@%p\n", ap->addr);
704	/* Enable the probe again */
705	ap->flags &= ~KPROBE_FLAG_DISABLED;
706	/* Optimize it again (remove from op->list) */
707	BUG_ON(!kprobe_optready(ap));
708	optimize_kprobe(ap);
709	}
710
711	/* Remove optimized instructions */
712	static void __kprobes kill_optimized_kprobe(struct kprobe *p)
713	{
714	struct optimized_kprobe *op;
715
716	op = container_of(p, struct optimized_kprobe, kp);
717	if (!list_empty(&op->list))
718	/* Dequeue from the (un)optimization queue */
719	list_del_init(&op->list);
720
721	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
722	/* Don't touch the code, because it is already freed. */
723	arch_remove_optimized_kprobe(op);
724	}
725
726	/* Try to prepare optimized instructions */
727	static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
728	{
729	struct optimized_kprobe *op;
730
731	op = container_of(p, struct optimized_kprobe, kp);
732	arch_prepare_optimized_kprobe(op);
733	}
734
735	/* Allocate new optimized_kprobe and try to prepare optimized instructions */
736	static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)
737	{
738	struct optimized_kprobe *op;
739
740	op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
741	if (!op)
742	return NULL;
743
744	INIT_LIST_HEAD(&op->list);
745	op->kp.addr = p->addr;
746	arch_prepare_optimized_kprobe(op);
747
748	return &op->kp;
749	}
750
751	static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p);
752
753	/*
754	* Prepare an optimized_kprobe and optimize it
755	* NOTE: p must be a normal registered kprobe
756	*/
757	static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
758	{
759	struct kprobe *ap;
760	struct optimized_kprobe *op;
761
762	ap = alloc_aggr_kprobe(p);
763	if (!ap)
764	return;
765
766	op = container_of(ap, struct optimized_kprobe, kp);
767	if (!arch_prepared_optinsn(&op->optinsn)) {
768	/* If failed to setup optimizing, fallback to kprobe */
769	arch_remove_optimized_kprobe(op);
770	kfree(op);
771	return;
772	}
773
774	init_aggr_kprobe(ap, p);
775	optimize_kprobe(ap);
776	}
777
778	#ifdef CONFIG_SYSCTL
779	/* This should be called with kprobe_mutex locked */
780	static void __kprobes optimize_all_kprobes(void)
781	{
782	struct hlist_head *head;
783	struct hlist_node *node;
784	struct kprobe *p;
785	unsigned int i;
786
787	/* If optimization is already allowed, just return */
788	if (kprobes_allow_optimization)
789	return;
790
791	kprobes_allow_optimization = true;
792	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
793	head = &kprobe_table[i];
794	hlist_for_each_entry_rcu(p, node, head, hlist)
795	if (!kprobe_disabled(p))
796	optimize_kprobe(p);
797	}
798	printk(KERN_INFO "Kprobes globally optimized\n");
799	}
800
801	/* This should be called with kprobe_mutex locked */
802	static void __kprobes unoptimize_all_kprobes(void)
803	{
804	struct hlist_head *head;
805	struct hlist_node *node;
806	struct kprobe *p;
807	unsigned int i;
808
809	/* If optimization is already prohibited, just return */
810	if (!kprobes_allow_optimization)
811	return;
812
813	kprobes_allow_optimization = false;
814	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
815	head = &kprobe_table[i];
816	hlist_for_each_entry_rcu(p, node, head, hlist) {
817	if (!kprobe_disabled(p))
818	unoptimize_kprobe(p, false);
819	}
820	}
821	/* Wait for unoptimizing completion */
822	wait_for_kprobe_optimizer();
823	printk(KERN_INFO "Kprobes globally unoptimized\n");
824	}
825
826	int sysctl_kprobes_optimization;
827	int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
828	void __user buffer, size_t length,
829	loff_t *ppos)
830	{
831	int ret;
832
833	mutex_lock(&kprobe_mutex);
834	sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
835	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
836
837	if (sysctl_kprobes_optimization)
838	optimize_all_kprobes();
839	else
840	unoptimize_all_kprobes();
841	mutex_unlock(&kprobe_mutex);
842
843	return ret;
844	}
845	#endif /* CONFIG_SYSCTL */
846
847	/* Put a breakpoint for a probe. Must be called with text_mutex locked */
848	static void __kprobes __arm_kprobe(struct kprobe *p)
849	{
850	struct kprobe *_p;
851
852	/* Check collision with other optimized kprobes */
853	_p = get_optimized_kprobe((unsigned long)p->addr);
854	if (unlikely(_p))
855	/* Fallback to unoptimized kprobe */
856	unoptimize_kprobe(_p, true);
857
858	arch_arm_kprobe(p);
859	optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
860	}
861
862	/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
863	static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)
864	{
865	struct kprobe *_p;
866
867	unoptimize_kprobe(p, false); /* Try to unoptimize */
868
869	if (!kprobe_queued(p)) {
870	arch_disarm_kprobe(p);
871	/* If another kprobe was blocked, optimize it. */
872	_p = get_optimized_kprobe((unsigned long)p->addr);
873	if (unlikely(_p) && reopt)
874	optimize_kprobe(_p);
875	}
876	/* TODO: reoptimize others after unoptimized this probe */
877	}
878
879	#else /* !CONFIG_OPTPROBES */
880
881	#define optimize_kprobe(p) do {} while (0)
882	#define unoptimize_kprobe(p, f) do {} while (0)
883	#define kill_optimized_kprobe(p) do {} while (0)
884	#define prepare_optimized_kprobe(p) do {} while (0)
885	#define try_to_optimize_kprobe(p) do {} while (0)
886	#define __arm_kprobe(p) arch_arm_kprobe(p)
887	#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
888	#define kprobe_disarmed(p) kprobe_disabled(p)
889	#define wait_for_kprobe_optimizer() do {} while (0)
890
891	/* There should be no unused kprobes can be reused without optimization */
892	static void reuse_unused_kprobe(struct kprobe *ap)
893	{
894	printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
895	BUG_ON(kprobe_unused(ap));
896	}
897
898	static __kprobes void free_aggr_kprobe(struct kprobe *p)
899	{
900	arch_remove_kprobe(p);
901	kfree(p);
902	}
903
904	static __kprobes struct kprobe alloc_aggr_kprobe(struct kprobe p)
905	{
906	return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
907	}
908	#endif /* CONFIG_OPTPROBES */
909
910	/* Arm a kprobe with text_mutex */
911	static void __kprobes arm_kprobe(struct kprobe *kp)
912	{
913	/*
914	* Here, since __arm_kprobe() doesn't use stop_machine(),
915	* this doesn't cause deadlock on text_mutex. So, we don't
916	* need get_online_cpus().
917	*/
918	mutex_lock(&text_mutex);
919	__arm_kprobe(kp);
920	mutex_unlock(&text_mutex);
921	}
922
923	/* Disarm a kprobe with text_mutex */
924	static void __kprobes disarm_kprobe(struct kprobe *kp)
925	{
926	/* Ditto */
927	mutex_lock(&text_mutex);
928	__disarm_kprobe(kp, true);
929	mutex_unlock(&text_mutex);
930	}
931
932	/*
933	* Aggregate handlers for multiple kprobes support - these handlers
934	* take care of invoking the individual kprobe handlers on p->list
935	*/
936	static int __kprobes aggr_pre_handler(struct kprobe p, struct pt_regs regs)
937	{
938	struct kprobe *kp;
939
940	list_for_each_entry_rcu(kp, &p->list, list) {
941	if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
942	set_kprobe_instance(kp);
943	if (kp->pre_handler(kp, regs))
944	return 1;
945	}
946	reset_kprobe_instance();
947	}
948	return 0;
949	}
950
951	static void __kprobes aggr_post_handler(struct kprobe p, struct pt_regs regs,
952	unsigned long flags)
953	{
954	struct kprobe *kp;
955
956	list_for_each_entry_rcu(kp, &p->list, list) {
957	if (kp->post_handler && likely(!kprobe_disabled(kp))) {
958	set_kprobe_instance(kp);
959	kp->post_handler(kp, regs, flags);
960	reset_kprobe_instance();
961	}
962	}
963	}
964
965	static int __kprobes aggr_fault_handler(struct kprobe p, struct pt_regs regs,
966	int trapnr)
967	{
968	struct kprobe *cur = __this_cpu_read(kprobe_instance);
969
970	/*
971	* if we faulted "during" the execution of a user specified
972	* probe handler, invoke just that probe's fault handler
973	*/
974	if (cur && cur->fault_handler) {
975	if (cur->fault_handler(cur, regs, trapnr))
976	return 1;
977	}
978	return 0;
979	}
980
981	static int __kprobes aggr_break_handler(struct kprobe p, struct pt_regs regs)
982	{
983	struct kprobe *cur = __this_cpu_read(kprobe_instance);
984	int ret = 0;
985
986	if (cur && cur->break_handler) {
987	if (cur->break_handler(cur, regs))
988	ret = 1;
989	}
990	reset_kprobe_instance();
991	return ret;
992	}
993
994	/* Walks the list and increments nmissed count for multiprobe case */
995	void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
996	{
997	struct kprobe *kp;
998	if (!kprobe_aggrprobe(p)) {
999	p->nmissed++;
1000	} else {
1001	list_for_each_entry_rcu(kp, &p->list, list)
1002	kp->nmissed++;
1003	}
1004	return;
1005	}
1006
1007	void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
1008	struct hlist_head *head)
1009	{
1010	struct kretprobe *rp = ri->rp;
1011
1012	/* remove rp inst off the rprobe_inst_table */
1013	hlist_del(&ri->hlist);
1014	INIT_HLIST_NODE(&ri->hlist);
1015	if (likely(rp)) {
1016	spin_lock(&rp->lock);
1017	hlist_add_head(&ri->hlist, &rp->free_instances);
1018	spin_unlock(&rp->lock);
1019	} else
1020	/* Unregistering */
1021	hlist_add_head(&ri->hlist, head);
1022	}
1023
1024	void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
1025	struct hlist_head *head, unsigned long flags)
1026	__acquires(hlist_lock)
1027	{
1028	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1029	spinlock_t *hlist_lock;
1030
1031	*head = &kretprobe_inst_table[hash];
1032	hlist_lock = kretprobe_table_lock_ptr(hash);
1033	spin_lock_irqsave(hlist_lock, *flags);
1034	}
1035
1036	static void __kprobes kretprobe_table_lock(unsigned long hash,
1037	unsigned long *flags)
1038	__acquires(hlist_lock)
1039	{
1040	spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1041	spin_lock_irqsave(hlist_lock, *flags);
1042	}
1043
1044	void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
1045	unsigned long *flags)
1046	__releases(hlist_lock)
1047	{
1048	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1049	spinlock_t *hlist_lock;
1050
1051	hlist_lock = kretprobe_table_lock_ptr(hash);
1052	spin_unlock_irqrestore(hlist_lock, *flags);
1053	}
1054
1055	static void __kprobes kretprobe_table_unlock(unsigned long hash,
1056	unsigned long *flags)
1057	__releases(hlist_lock)
1058	{
1059	spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1060	spin_unlock_irqrestore(hlist_lock, *flags);
1061	}
1062
1063	/*
1064	* This function is called from finish_task_switch when task tk becomes dead,
1065	* so that we can recycle any function-return probe instances associated
1066	* with this task. These left over instances represent probed functions
1067	* that have been called but will never return.
1068	*/
1069	void __kprobes kprobe_flush_task(struct task_struct *tk)
1070	{
1071	struct kretprobe_instance *ri;
1072	struct hlist_head *head, empty_rp;
1073	struct hlist_node node, tmp;
1074	unsigned long hash, flags = 0;
1075
1076	if (unlikely(!kprobes_initialized))
1077	/* Early boot. kretprobe_table_locks not yet initialized. */
1078	return;
1079
1080	hash = hash_ptr(tk, KPROBE_HASH_BITS);
1081	head = &kretprobe_inst_table[hash];
1082	kretprobe_table_lock(hash, &flags);
1083	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
1084	if (ri->task == tk)
1085	recycle_rp_inst(ri, &empty_rp);
1086	}
1087	kretprobe_table_unlock(hash, &flags);
1088	INIT_HLIST_HEAD(&empty_rp);
1089	hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
1090	hlist_del(&ri->hlist);
1091	kfree(ri);
1092	}
1093	}
1094
1095	static inline void free_rp_inst(struct kretprobe *rp)
1096	{
1097	struct kretprobe_instance *ri;
1098	struct hlist_node pos, next;
1099
1100	hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
1101	hlist_del(&ri->hlist);
1102	kfree(ri);
1103	}
1104	}
1105
1106	static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
1107	{
1108	unsigned long flags, hash;
1109	struct kretprobe_instance *ri;
1110	struct hlist_node pos, next;
1111	struct hlist_head *head;
1112
1113	/* No race here */
1114	for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1115	kretprobe_table_lock(hash, &flags);
1116	head = &kretprobe_inst_table[hash];
1117	hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
1118	if (ri->rp == rp)
1119	ri->rp = NULL;
1120	}
1121	kretprobe_table_unlock(hash, &flags);
1122	}
1123	free_rp_inst(rp);
1124	}
1125
1126	/*
1127	* Add the new probe to ap->list. Fail if this is the
1128	* second jprobe at the address - two jprobes can't coexist
1129	*/
1130	static int __kprobes add_new_kprobe(struct kprobe ap, struct kprobe p)
1131	{
1132	BUG_ON(kprobe_gone(ap) \|\| kprobe_gone(p));
1133
1134	if (p->break_handler \|\| p->post_handler)
1135	unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
1136
1137	if (p->break_handler) {
1138	if (ap->break_handler)
1139	return -EEXIST;
1140	list_add_tail_rcu(&p->list, &ap->list);
1141	ap->break_handler = aggr_break_handler;
1142	} else
1143	list_add_rcu(&p->list, &ap->list);
1144	if (p->post_handler && !ap->post_handler)
1145	ap->post_handler = aggr_post_handler;
1146
1147	if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
1148	ap->flags &= ~KPROBE_FLAG_DISABLED;
1149	if (!kprobes_all_disarmed)
1150	/* Arm the breakpoint again. */
1151	__arm_kprobe(ap);
1152	}
1153	return 0;
1154	}
1155
1156	/*
1157	* Fill in the required fields of the "manager kprobe". Replace the
1158	* earlier kprobe in the hlist with the manager kprobe
1159	*/
1160	static void __kprobes init_aggr_kprobe(struct kprobe ap, struct kprobe p)
1161	{
1162	/* Copy p's insn slot to ap */
1163	copy_kprobe(p, ap);
1164	flush_insn_slot(ap);
1165	ap->addr = p->addr;
1166	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1167	ap->pre_handler = aggr_pre_handler;
1168	ap->fault_handler = aggr_fault_handler;
1169	/* We don't care the kprobe which has gone. */
1170	if (p->post_handler && !kprobe_gone(p))
1171	ap->post_handler = aggr_post_handler;
1172	if (p->break_handler && !kprobe_gone(p))
1173	ap->break_handler = aggr_break_handler;
1174
1175	INIT_LIST_HEAD(&ap->list);
1176	INIT_HLIST_NODE(&ap->hlist);
1177
1178	list_add_rcu(&p->list, &ap->list);
1179	hlist_replace_rcu(&p->hlist, &ap->hlist);
1180	}
1181
1182	/*
1183	* This is the second or subsequent kprobe at the address - handle
1184	* the intricacies
1185	*/
1186	static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,
1187	struct kprobe *p)
1188	{
1189	int ret = 0;
1190	struct kprobe *ap = orig_p;
1191
1192	if (!kprobe_aggrprobe(orig_p)) {
1193	/* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1194	ap = alloc_aggr_kprobe(orig_p);
1195	if (!ap)
1196	return -ENOMEM;
1197	init_aggr_kprobe(ap, orig_p);
1198	} else if (kprobe_unused(ap))
1199	/* This probe is going to die. Rescue it */
1200	reuse_unused_kprobe(ap);
1201
1202	if (kprobe_gone(ap)) {
1203	/*
1204	* Attempting to insert new probe at the same location that
1205	* had a probe in the module vaddr area which already
1206	* freed. So, the instruction slot has already been
1207	* released. We need a new slot for the new probe.
1208	*/
1209	ret = arch_prepare_kprobe(ap);
1210	if (ret)
1211	/*
1212	* Even if fail to allocate new slot, don't need to
1213	* free aggr_probe. It will be used next time, or
1214	* freed by unregister_kprobe.
1215	*/
1216	return ret;
1217
1218	/* Prepare optimized instructions if possible. */
1219	prepare_optimized_kprobe(ap);
1220
1221	/*
1222	* Clear gone flag to prevent allocating new slot again, and
1223	* set disabled flag because it is not armed yet.
1224	*/
1225	ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1226	\| KPROBE_FLAG_DISABLED;
1227	}
1228
1229	/* Copy ap's insn slot to p */
1230	copy_kprobe(ap, p);
1231	return add_new_kprobe(ap, p);
1232	}
1233
1234	static int __kprobes in_kprobes_functions(unsigned long addr)
1235	{
1236	struct kprobe_blackpoint *kb;
1237
1238	if (addr >= (unsigned long)__kprobes_text_start &&
1239	addr < (unsigned long)__kprobes_text_end)
1240	return -EINVAL;
1241	/*
1242	* If there exists a kprobe_blacklist, verify and
1243	* fail any probe registration in the prohibited area
1244	*/
1245	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1246	if (kb->start_addr) {
1247	if (addr >= kb->start_addr &&
1248	addr < (kb->start_addr + kb->range))
1249	return -EINVAL;
1250	}
1251	}
1252	return 0;
1253	}
1254
1255	/*
1256	* If we have a symbol_name argument, look it up and add the offset field
1257	* to it. This way, we can specify a relative address to a symbol.
1258	*/
1259	static kprobe_opcode_t __kprobes kprobe_addr(struct kprobe p)
1260	{
1261	kprobe_opcode_t *addr = p->addr;
1262	if (p->symbol_name) {
1263	if (addr)
1264	return NULL;
1265	kprobe_lookup_name(p->symbol_name, addr);
1266	}
1267
1268	if (!addr)
1269	return NULL;
1270	return (kprobe_opcode_t )(((char )addr) + p->offset);
1271	}
1272
1273	/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1274	static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1275	{
1276	struct kprobe ap, list_p;
1277
1278	ap = get_kprobe(p->addr);
1279	if (unlikely(!ap))
1280	return NULL;
1281
1282	if (p != ap) {
1283	list_for_each_entry_rcu(list_p, &ap->list, list)
1284	if (list_p == p)
1285	/* kprobe p is a valid probe */
1286	goto valid;
1287	return NULL;
1288	}
1289	valid:
1290	return ap;
1291	}
1292
1293	/* Return error if the kprobe is being re-registered */
1294	static inline int check_kprobe_rereg(struct kprobe *p)
1295	{
1296	int ret = 0;
1297
1298	mutex_lock(&kprobe_mutex);
1299	if (__get_valid_kprobe(p))
1300	ret = -EINVAL;
1301	mutex_unlock(&kprobe_mutex);
1302
1303	return ret;
1304	}
1305
1306	int __kprobes register_kprobe(struct kprobe *p)
1307	{
1308	int ret = 0;
1309	struct kprobe *old_p;
1310	struct module *probed_mod;
1311	kprobe_opcode_t *addr;
1312
1313	addr = kprobe_addr(p);
1314	if (!addr)
1315	return -EINVAL;
1316	p->addr = addr;
1317
1318	ret = check_kprobe_rereg(p);
1319	if (ret)
1320	return ret;
1321
1322	jump_label_lock();
1323	preempt_disable();
1324	if (!kernel_text_address((unsigned long) p->addr) \|\|
1325	in_kprobes_functions((unsigned long) p->addr) \|\|
1326	ftrace_text_reserved(p->addr, p->addr) \|\|
1327	jump_label_text_reserved(p->addr, p->addr))
1328	goto fail_with_jump_label;
1329
1330	/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1331	p->flags &= KPROBE_FLAG_DISABLED;
1332
1333	/*
1334	* Check if are we probing a module.
1335	*/
1336	probed_mod = __module_text_address((unsigned long) p->addr);
1337	if (probed_mod) {
1338	/*
1339	* We must hold a refcount of the probed module while updating
1340	* its code to prohibit unexpected unloading.
1341	*/
1342	if (unlikely(!try_module_get(probed_mod)))
1343	goto fail_with_jump_label;
1344
1345	/*
1346	* If the module freed .init.text, we couldn't insert
1347	* kprobes in there.
1348	*/
1349	if (within_module_init((unsigned long)p->addr, probed_mod) &&
1350	probed_mod->state != MODULE_STATE_COMING) {
1351	module_put(probed_mod);
1352	goto fail_with_jump_label;
1353	}
1354	}
1355	preempt_enable();
1356	jump_label_unlock();
1357
1358	p->nmissed = 0;
1359	INIT_LIST_HEAD(&p->list);
1360	mutex_lock(&kprobe_mutex);
1361
1362	jump_label_lock(); /* needed to call jump_label_text_reserved() */
1363
1364	get_online_cpus(); /* For avoiding text_mutex deadlock. */
1365	mutex_lock(&text_mutex);
1366
1367	old_p = get_kprobe(p->addr);
1368	if (old_p) {
1369	/* Since this may unoptimize old_p, locking text_mutex. */
1370	ret = register_aggr_kprobe(old_p, p);
1371	goto out;
1372	}
1373
1374	ret = arch_prepare_kprobe(p);
1375	if (ret)
1376	goto out;
1377
1378	INIT_HLIST_NODE(&p->hlist);
1379	hlist_add_head_rcu(&p->hlist,
1380	&kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1381
1382	if (!kprobes_all_disarmed && !kprobe_disabled(p))
1383	__arm_kprobe(p);
1384
1385	/* Try to optimize kprobe */
1386	try_to_optimize_kprobe(p);
1387
1388	out:
1389	mutex_unlock(&text_mutex);
1390	put_online_cpus();
1391	jump_label_unlock();
1392	mutex_unlock(&kprobe_mutex);
1393
1394	if (probed_mod)
1395	module_put(probed_mod);
1396
1397	return ret;
1398
1399	fail_with_jump_label:
1400	preempt_enable();
1401	jump_label_unlock();
1402	return -EINVAL;
1403	}
1404	EXPORT_SYMBOL_GPL(register_kprobe);
1405
1406	/* Check if all probes on the aggrprobe are disabled */
1407	static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)
1408	{
1409	struct kprobe *kp;
1410
1411	list_for_each_entry_rcu(kp, &ap->list, list)
1412	if (!kprobe_disabled(kp))
1413	/*
1414	* There is an active probe on the list.
1415	* We can't disable this ap.
1416	*/
1417	return 0;
1418
1419	return 1;
1420	}
1421
1422	/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1423	static struct kprobe __kprobes __disable_kprobe(struct kprobe p)
1424	{
1425	struct kprobe *orig_p;
1426
1427	/* Get an original kprobe for return */
1428	orig_p = __get_valid_kprobe(p);
1429	if (unlikely(orig_p == NULL))
1430	return NULL;
1431
1432	if (!kprobe_disabled(p)) {
1433	/* Disable probe if it is a child probe */
1434	if (p != orig_p)
1435	p->flags \|= KPROBE_FLAG_DISABLED;
1436
1437	/* Try to disarm and disable this/parent probe */
1438	if (p == orig_p \|\| aggr_kprobe_disabled(orig_p)) {
1439	disarm_kprobe(orig_p);
1440	orig_p->flags \|= KPROBE_FLAG_DISABLED;
1441	}
1442	}
1443
1444	return orig_p;
1445	}
1446
1447	/*
1448	* Unregister a kprobe without a scheduler synchronization.
1449	*/
1450	static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1451	{
1452	struct kprobe ap, list_p;
1453
1454	/* Disable kprobe. This will disarm it if needed. */
1455	ap = __disable_kprobe(p);
1456	if (ap == NULL)
1457	return -EINVAL;
1458
1459	if (ap == p)
1460	/*
1461	* This probe is an independent(and non-optimized) kprobe
1462	* (not an aggrprobe). Remove from the hash list.
1463	*/
1464	goto disarmed;
1465
1466	/* Following process expects this probe is an aggrprobe */
1467	WARN_ON(!kprobe_aggrprobe(ap));
1468
1469	if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1470	/*
1471	* !disarmed could be happen if the probe is under delayed
1472	* unoptimizing.
1473	*/
1474	goto disarmed;
1475	else {
1476	/* If disabling probe has special handlers, update aggrprobe */
1477	if (p->break_handler && !kprobe_gone(p))
1478	ap->break_handler = NULL;
1479	if (p->post_handler && !kprobe_gone(p)) {
1480	list_for_each_entry_rcu(list_p, &ap->list, list) {
1481	if ((list_p != p) && (list_p->post_handler))
1482	goto noclean;
1483	}
1484	ap->post_handler = NULL;
1485	}
1486	noclean:
1487	/*
1488	* Remove from the aggrprobe: this path will do nothing in
1489	* __unregister_kprobe_bottom().
1490	*/
1491	list_del_rcu(&p->list);
1492	if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1493	/*
1494	* Try to optimize this probe again, because post
1495	* handler may have been changed.
1496	*/
1497	optimize_kprobe(ap);
1498	}
1499	return 0;
1500
1501	disarmed:
1502	BUG_ON(!kprobe_disarmed(ap));
1503	hlist_del_rcu(&ap->hlist);
1504	return 0;
1505	}
1506
1507	static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1508	{
1509	struct kprobe *ap;
1510
1511	if (list_empty(&p->list))
1512	/* This is an independent kprobe */
1513	arch_remove_kprobe(p);
1514	else if (list_is_singular(&p->list)) {
1515	/* This is the last child of an aggrprobe */
1516	ap = list_entry(p->list.next, struct kprobe, list);
1517	list_del(&p->list);
1518	free_aggr_kprobe(ap);
1519	}
1520	/* Otherwise, do nothing. */
1521	}
1522
1523	int __kprobes register_kprobes(struct kprobe **kps, int num)
1524	{
1525	int i, ret = 0;
1526
1527	if (num <= 0)
1528	return -EINVAL;
1529	for (i = 0; i < num; i++) {
1530	ret = register_kprobe(kps[i]);
1531	if (ret < 0) {
1532	if (i > 0)
1533	unregister_kprobes(kps, i);
1534	break;
1535	}
1536	}
1537	return ret;
1538	}
1539	EXPORT_SYMBOL_GPL(register_kprobes);
1540
1541	void __kprobes unregister_kprobe(struct kprobe *p)
1542	{
1543	unregister_kprobes(&p, 1);
1544	}
1545	EXPORT_SYMBOL_GPL(unregister_kprobe);
1546
1547	void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1548	{
1549	int i;
1550
1551	if (num <= 0)
1552	return;
1553	mutex_lock(&kprobe_mutex);
1554	for (i = 0; i < num; i++)
1555	if (__unregister_kprobe_top(kps[i]) < 0)
1556	kps[i]->addr = NULL;
1557	mutex_unlock(&kprobe_mutex);
1558
1559	synchronize_sched();
1560	for (i = 0; i < num; i++)
1561	if (kps[i]->addr)
1562	__unregister_kprobe_bottom(kps[i]);
1563	}
1564	EXPORT_SYMBOL_GPL(unregister_kprobes);
1565
1566	static struct notifier_block kprobe_exceptions_nb = {
1567	.notifier_call = kprobe_exceptions_notify,
1568	.priority = 0x7fffffff /* we need to be notified first */
1569	};
1570
1571	unsigned long __weak arch_deref_entry_point(void *entry)
1572	{
1573	return (unsigned long)entry;
1574	}
1575
1576	int __kprobes register_jprobes(struct jprobe **jps, int num)
1577	{
1578	struct jprobe *jp;
1579	int ret = 0, i;
1580
1581	if (num <= 0)
1582	return -EINVAL;
1583	for (i = 0; i < num; i++) {
1584	unsigned long addr, offset;
1585	jp = jps[i];
1586	addr = arch_deref_entry_point(jp->entry);
1587
1588	/* Verify probepoint is a function entry point */
1589	if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1590	offset == 0) {
1591	jp->kp.pre_handler = setjmp_pre_handler;
1592	jp->kp.break_handler = longjmp_break_handler;
1593	ret = register_kprobe(&jp->kp);
1594	} else
1595	ret = -EINVAL;
1596
1597	if (ret < 0) {
1598	if (i > 0)
1599	unregister_jprobes(jps, i);
1600	break;
1601	}
1602	}
1603	return ret;
1604	}
1605	EXPORT_SYMBOL_GPL(register_jprobes);
1606
1607	int __kprobes register_jprobe(struct jprobe *jp)
1608	{
1609	return register_jprobes(&jp, 1);
1610	}
1611	EXPORT_SYMBOL_GPL(register_jprobe);
1612
1613	void __kprobes unregister_jprobe(struct jprobe *jp)
1614	{
1615	unregister_jprobes(&jp, 1);
1616	}
1617	EXPORT_SYMBOL_GPL(unregister_jprobe);
1618
1619	void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1620	{
1621	int i;
1622
1623	if (num <= 0)
1624	return;
1625	mutex_lock(&kprobe_mutex);
1626	for (i = 0; i < num; i++)
1627	if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1628	jps[i]->kp.addr = NULL;
1629	mutex_unlock(&kprobe_mutex);
1630
1631	synchronize_sched();
1632	for (i = 0; i < num; i++) {
1633	if (jps[i]->kp.addr)
1634	__unregister_kprobe_bottom(&jps[i]->kp);
1635	}
1636	}
1637	EXPORT_SYMBOL_GPL(unregister_jprobes);
1638
1639	#ifdef CONFIG_KRETPROBES
1640	/*
1641	* This kprobe pre_handler is registered with every kretprobe. When probe
1642	* hits it will set up the return probe.
1643	*/
1644	static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1645	struct pt_regs *regs)
1646	{
1647	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1648	unsigned long hash, flags = 0;
1649	struct kretprobe_instance *ri;
1650
1651	/TODO: consider to only swap the RA after the last pre_handler fired /
1652	hash = hash_ptr(current, KPROBE_HASH_BITS);
1653	spin_lock_irqsave(&rp->lock, flags);
1654	if (!hlist_empty(&rp->free_instances)) {
1655	ri = hlist_entry(rp->free_instances.first,
1656	struct kretprobe_instance, hlist);
1657	hlist_del(&ri->hlist);
1658	spin_unlock_irqrestore(&rp->lock, flags);
1659
1660	ri->rp = rp;
1661	ri->task = current;
1662
1663	if (rp->entry_handler && rp->entry_handler(ri, regs))
1664	return 0;
1665
1666	arch_prepare_kretprobe(ri, regs);
1667
1668	/* XXX(hch): why is there no hlist_move_head? */
1669	INIT_HLIST_NODE(&ri->hlist);
1670	kretprobe_table_lock(hash, &flags);
1671	hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1672	kretprobe_table_unlock(hash, &flags);
1673	} else {
1674	rp->nmissed++;
1675	spin_unlock_irqrestore(&rp->lock, flags);
1676	}
1677	return 0;
1678	}
1679
1680	int __kprobes register_kretprobe(struct kretprobe *rp)
1681	{
1682	int ret = 0;
1683	struct kretprobe_instance *inst;
1684	int i;
1685	void *addr;
1686
1687	if (kretprobe_blacklist_size) {
1688	addr = kprobe_addr(&rp->kp);
1689	if (!addr)
1690	return -EINVAL;
1691
1692	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1693	if (kretprobe_blacklist[i].addr == addr)
1694	return -EINVAL;
1695	}
1696	}
1697
1698	rp->kp.pre_handler = pre_handler_kretprobe;
1699	rp->kp.post_handler = NULL;
1700	rp->kp.fault_handler = NULL;
1701	rp->kp.break_handler = NULL;
1702
1703	/* Pre-allocate memory for max kretprobe instances */
1704	if (rp->maxactive <= 0) {
1705	#ifdef CONFIG_PREEMPT
1706	rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1707	#else
1708	rp->maxactive = num_possible_cpus();
1709	#endif
1710	}
1711	spin_lock_init(&rp->lock);
1712	INIT_HLIST_HEAD(&rp->free_instances);
1713	for (i = 0; i < rp->maxactive; i++) {
1714	inst = kmalloc(sizeof(struct kretprobe_instance) +
1715	rp->data_size, GFP_KERNEL);
1716	if (inst == NULL) {
1717	free_rp_inst(rp);
1718	return -ENOMEM;
1719	}
1720	INIT_HLIST_NODE(&inst->hlist);
1721	hlist_add_head(&inst->hlist, &rp->free_instances);
1722	}
1723
1724	rp->nmissed = 0;
1725	/* Establish function entry probe point */
1726	ret = register_kprobe(&rp->kp);
1727	if (ret != 0)
1728	free_rp_inst(rp);
1729	return ret;
1730	}
1731	EXPORT_SYMBOL_GPL(register_kretprobe);
1732
1733	int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1734	{
1735	int ret = 0, i;
1736
1737	if (num <= 0)
1738	return -EINVAL;
1739	for (i = 0; i < num; i++) {
1740	ret = register_kretprobe(rps[i]);
1741	if (ret < 0) {
1742	if (i > 0)
1743	unregister_kretprobes(rps, i);
1744	break;
1745	}
1746	}
1747	return ret;
1748	}
1749	EXPORT_SYMBOL_GPL(register_kretprobes);
1750
1751	void __kprobes unregister_kretprobe(struct kretprobe *rp)
1752	{
1753	unregister_kretprobes(&rp, 1);
1754	}
1755	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1756
1757	void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1758	{
1759	int i;
1760
1761	if (num <= 0)
1762	return;
1763	mutex_lock(&kprobe_mutex);
1764	for (i = 0; i < num; i++)
1765	if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1766	rps[i]->kp.addr = NULL;
1767	mutex_unlock(&kprobe_mutex);
1768
1769	synchronize_sched();
1770	for (i = 0; i < num; i++) {
1771	if (rps[i]->kp.addr) {
1772	__unregister_kprobe_bottom(&rps[i]->kp);
1773	cleanup_rp_inst(rps[i]);
1774	}
1775	}
1776	}
1777	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1778
1779	#else /* CONFIG_KRETPROBES */
1780	int __kprobes register_kretprobe(struct kretprobe *rp)
1781	{
1782	return -ENOSYS;
1783	}
1784	EXPORT_SYMBOL_GPL(register_kretprobe);
1785
1786	int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1787	{
1788	return -ENOSYS;
1789	}
1790	EXPORT_SYMBOL_GPL(register_kretprobes);
1791
1792	void __kprobes unregister_kretprobe(struct kretprobe *rp)
1793	{
1794	}
1795	EXPORT_SYMBOL_GPL(unregister_kretprobe);
1796
1797	void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1798	{
1799	}
1800	EXPORT_SYMBOL_GPL(unregister_kretprobes);
1801
1802	static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1803	struct pt_regs *regs)
1804	{
1805	return 0;
1806	}
1807
1808	#endif /* CONFIG_KRETPROBES */
1809
1810	/* Set the kprobe gone and remove its instruction buffer. */
1811	static void __kprobes kill_kprobe(struct kprobe *p)
1812	{
1813	struct kprobe *kp;
1814
1815	p->flags \|= KPROBE_FLAG_GONE;
1816	if (kprobe_aggrprobe(p)) {
1817	/*
1818	* If this is an aggr_kprobe, we have to list all the
1819	* chained probes and mark them GONE.
1820	*/
1821	list_for_each_entry_rcu(kp, &p->list, list)
1822	kp->flags \|= KPROBE_FLAG_GONE;
1823	p->post_handler = NULL;
1824	p->break_handler = NULL;
1825	kill_optimized_kprobe(p);
1826	}
1827	/*
1828	* Here, we can remove insn_slot safely, because no thread calls
1829	* the original probed function (which will be freed soon) any more.
1830	*/
1831	arch_remove_kprobe(p);
1832	}
1833
1834	/* Disable one kprobe */
1835	int __kprobes disable_kprobe(struct kprobe *kp)
1836	{
1837	int ret = 0;
1838
1839	mutex_lock(&kprobe_mutex);
1840
1841	/* Disable this kprobe */
1842	if (__disable_kprobe(kp) == NULL)
1843	ret = -EINVAL;
1844
1845	mutex_unlock(&kprobe_mutex);
1846	return ret;
1847	}
1848	EXPORT_SYMBOL_GPL(disable_kprobe);
1849
1850	/* Enable one kprobe */
1851	int __kprobes enable_kprobe(struct kprobe *kp)
1852	{
1853	int ret = 0;
1854	struct kprobe *p;
1855
1856	mutex_lock(&kprobe_mutex);
1857
1858	/* Check whether specified probe is valid. */
1859	p = __get_valid_kprobe(kp);
1860	if (unlikely(p == NULL)) {
1861	ret = -EINVAL;
1862	goto out;
1863	}
1864
1865	if (kprobe_gone(kp)) {
1866	/* This kprobe has gone, we couldn't enable it. */
1867	ret = -EINVAL;
1868	goto out;
1869	}
1870
1871	if (p != kp)
1872	kp->flags &= ~KPROBE_FLAG_DISABLED;
1873
1874	if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1875	p->flags &= ~KPROBE_FLAG_DISABLED;
1876	arm_kprobe(p);
1877	}
1878	out:
1879	mutex_unlock(&kprobe_mutex);
1880	return ret;
1881	}
1882	EXPORT_SYMBOL_GPL(enable_kprobe);
1883
1884	void __kprobes dump_kprobe(struct kprobe *kp)
1885	{
1886	printk(KERN_WARNING "Dumping kprobe:\n");
1887	printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1888	kp->symbol_name, kp->addr, kp->offset);
1889	}
1890
1891	/* Module notifier call back, checking kprobes on the module */
1892	static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1893	unsigned long val, void *data)
1894	{
1895	struct module *mod = data;
1896	struct hlist_head *head;
1897	struct hlist_node *node;
1898	struct kprobe *p;
1899	unsigned int i;
1900	int checkcore = (val == MODULE_STATE_GOING);
1901
1902	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1903	return NOTIFY_DONE;
1904
1905	/*
1906	* When MODULE_STATE_GOING was notified, both of module .text and
1907	* .init.text sections would be freed. When MODULE_STATE_LIVE was
1908	* notified, only .init.text section would be freed. We need to
1909	* disable kprobes which have been inserted in the sections.
1910	*/
1911	mutex_lock(&kprobe_mutex);
1912	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1913	head = &kprobe_table[i];
1914	hlist_for_each_entry_rcu(p, node, head, hlist)
1915	if (within_module_init((unsigned long)p->addr, mod) \|\|
1916	(checkcore &&
1917	within_module_core((unsigned long)p->addr, mod))) {
1918	/*
1919	* The vaddr this probe is installed will soon
1920	* be vfreed buy not synced to disk. Hence,
1921	* disarming the breakpoint isn't needed.
1922	*/
1923	kill_kprobe(p);
1924	}
1925	}
1926	mutex_unlock(&kprobe_mutex);
1927	return NOTIFY_DONE;
1928	}
1929
1930	static struct notifier_block kprobe_module_nb = {
1931	.notifier_call = kprobes_module_callback,
1932	.priority = 0
1933	};
1934
1935	static int __init init_kprobes(void)
1936	{
1937	int i, err = 0;
1938	unsigned long offset = 0, size = 0;
1939	char *modname, namebuf[128];
1940	const char *symbol_name;
1941	void *addr;
1942	struct kprobe_blackpoint *kb;
1943
1944	/* FIXME allocate the probe table, currently defined statically */
1945	/* initialize all list heads */
1946	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1947	INIT_HLIST_HEAD(&kprobe_table[i]);
1948	INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1949	spin_lock_init(&(kretprobe_table_locks[i].lock));
1950	}
1951
1952	/*
1953	* Lookup and populate the kprobe_blacklist.
1954	*
1955	* Unlike the kretprobe blacklist, we'll need to determine
1956	* the range of addresses that belong to the said functions,
1957	* since a kprobe need not necessarily be at the beginning
1958	* of a function.
1959	*/
1960	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1961	kprobe_lookup_name(kb->name, addr);
1962	if (!addr)
1963	continue;
1964
1965	kb->start_addr = (unsigned long)addr;
1966	symbol_name = kallsyms_lookup(kb->start_addr,
1967	&size, &offset, &modname, namebuf);
1968	if (!symbol_name)
1969	kb->range = 0;
1970	else
1971	kb->range = size;
1972	}
1973
1974	if (kretprobe_blacklist_size) {
1975	/* lookup the function address from its name */
1976	for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1977	kprobe_lookup_name(kretprobe_blacklist[i].name,
1978	kretprobe_blacklist[i].addr);
1979	if (!kretprobe_blacklist[i].addr)
1980	printk("kretprobe: lookup failed: %s\n",
1981	kretprobe_blacklist[i].name);
1982	}
1983	}
1984
1985	#if defined(CONFIG_OPTPROBES)
1986	#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1987	/* Init kprobe_optinsn_slots */
1988	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1989	#endif
1990	/* By default, kprobes can be optimized */
1991	kprobes_allow_optimization = true;
1992	#endif
1993
1994	/* By default, kprobes are armed */
1995	kprobes_all_disarmed = false;
1996
1997	err = arch_init_kprobes();
1998	if (!err)
1999	err = register_die_notifier(&kprobe_exceptions_nb);
2000	if (!err)
2001	err = register_module_notifier(&kprobe_module_nb);
2002
2003	kprobes_initialized = (err == 0);
2004
2005	if (!err)
2006	init_test_probes();
2007	return err;
2008	}
2009
2010	#ifdef CONFIG_DEBUG_FS
2011	static void __kprobes report_probe(struct seq_file pi, struct kprobe p,
2012	const char sym, int offset, char modname, struct kprobe *pp)
2013	{
2014	char *kprobe_type;
2015
2016	if (p->pre_handler == pre_handler_kretprobe)
2017	kprobe_type = "r";
2018	else if (p->pre_handler == setjmp_pre_handler)
2019	kprobe_type = "j";
2020	else
2021	kprobe_type = "k";
2022
2023	if (sym)
2024	seq_printf(pi, "%p %s %s+0x%x %s ",
2025	p->addr, kprobe_type, sym, offset,
2026	(modname ? modname : " "));
2027	else
2028	seq_printf(pi, "%p %s %p ",
2029	p->addr, kprobe_type, p->addr);
2030
2031	if (!pp)
2032	pp = p;
2033	seq_printf(pi, "%s%s%s\n",
2034	(kprobe_gone(p) ? "[GONE]" : ""),
2035	((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2036	(kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
2037	}
2038
2039	static void __kprobes kprobe_seq_start(struct seq_file f, loff_t *pos)
2040	{
2041	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2042	}
2043
2044	static void __kprobes kprobe_seq_next(struct seq_file f, void v, loff_t pos)
2045	{
2046	(*pos)++;
2047	if (*pos >= KPROBE_TABLE_SIZE)
2048	return NULL;
2049	return pos;
2050	}
2051
2052	static void __kprobes kprobe_seq_stop(struct seq_file f, void v)
2053	{
2054	/* Nothing to do */
2055	}
2056
2057	static int __kprobes show_kprobe_addr(struct seq_file pi, void v)
2058	{
2059	struct hlist_head *head;
2060	struct hlist_node *node;
2061	struct kprobe p, kp;
2062	const char *sym = NULL;
2063	unsigned int i = (loff_t ) v;
2064	unsigned long offset = 0;
2065	char *modname, namebuf[128];
2066
2067	head = &kprobe_table[i];
2068	preempt_disable();
2069	hlist_for_each_entry_rcu(p, node, head, hlist) {
2070	sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2071	&offset, &modname, namebuf);
2072	if (kprobe_aggrprobe(p)) {
2073	list_for_each_entry_rcu(kp, &p->list, list)
2074	report_probe(pi, kp, sym, offset, modname, p);
2075	} else
2076	report_probe(pi, p, sym, offset, modname, NULL);
2077	}
2078	preempt_enable();
2079	return 0;
2080	}
2081
2082	static const struct seq_operations kprobes_seq_ops = {
2083	.start = kprobe_seq_start,
2084	.next = kprobe_seq_next,
2085	.stop = kprobe_seq_stop,
2086	.show = show_kprobe_addr
2087	};
2088
2089	static int __kprobes kprobes_open(struct inode inode, struct file filp)
2090	{
2091	return seq_open(filp, &kprobes_seq_ops);
2092	}
2093
2094	static const struct file_operations debugfs_kprobes_operations = {
2095	.open = kprobes_open,
2096	.read = seq_read,
2097	.llseek = seq_lseek,
2098	.release = seq_release,
2099	};
2100
2101	static void __kprobes arm_all_kprobes(void)
2102	{
2103	struct hlist_head *head;
2104	struct hlist_node *node;
2105	struct kprobe *p;
2106	unsigned int i;
2107
2108	mutex_lock(&kprobe_mutex);
2109
2110	/* If kprobes are armed, just return */
2111	if (!kprobes_all_disarmed)
2112	goto already_enabled;
2113
2114	/* Arming kprobes doesn't optimize kprobe itself */
2115	mutex_lock(&text_mutex);
2116	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2117	head = &kprobe_table[i];
2118	hlist_for_each_entry_rcu(p, node, head, hlist)
2119	if (!kprobe_disabled(p))
2120	__arm_kprobe(p);
2121	}
2122	mutex_unlock(&text_mutex);
2123
2124	kprobes_all_disarmed = false;
2125	printk(KERN_INFO "Kprobes globally enabled\n");
2126
2127	already_enabled:
2128	mutex_unlock(&kprobe_mutex);
2129	return;
2130	}
2131
2132	static void __kprobes disarm_all_kprobes(void)
2133	{
2134	struct hlist_head *head;
2135	struct hlist_node *node;
2136	struct kprobe *p;
2137	unsigned int i;
2138
2139	mutex_lock(&kprobe_mutex);
2140
2141	/* If kprobes are already disarmed, just return */
2142	if (kprobes_all_disarmed) {
2143	mutex_unlock(&kprobe_mutex);
2144	return;
2145	}
2146
2147	kprobes_all_disarmed = true;
2148	printk(KERN_INFO "Kprobes globally disabled\n");
2149
2150	mutex_lock(&text_mutex);
2151	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2152	head = &kprobe_table[i];
2153	hlist_for_each_entry_rcu(p, node, head, hlist) {
2154	if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
2155	__disarm_kprobe(p, false);
2156	}
2157	}
2158	mutex_unlock(&text_mutex);
2159	mutex_unlock(&kprobe_mutex);
2160
2161	/* Wait for disarming all kprobes by optimizer */
2162	wait_for_kprobe_optimizer();
2163	}
2164
2165	/*
2166	* XXX: The debugfs bool file interface doesn't allow for callbacks
2167	* when the bool state is switched. We can reuse that facility when
2168	* available
2169	*/
2170	static ssize_t read_enabled_file_bool(struct file *file,
2171	char __user user_buf, size_t count, loff_t ppos)
2172	{
2173	char buf[3];
2174
2175	if (!kprobes_all_disarmed)
2176	buf[0] = '1';
2177	else
2178	buf[0] = '0';
2179	buf[1] = '\n';
2180	buf[2] = 0x00;
2181	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2182	}
2183
2184	static ssize_t write_enabled_file_bool(struct file *file,
2185	const char __user user_buf, size_t count, loff_t ppos)
2186	{
2187	char buf[32];
2188	int buf_size;
2189
2190	buf_size = min(count, (sizeof(buf)-1));
2191	if (copy_from_user(buf, user_buf, buf_size))
2192	return -EFAULT;
2193
2194	switch (buf[0]) {
2195	case 'y':
2196	case 'Y':
2197	case '1':
2198	arm_all_kprobes();
2199	break;
2200	case 'n':
2201	case 'N':
2202	case '0':
2203	disarm_all_kprobes();
2204	break;
2205	}
2206
2207	return count;
2208	}
2209
2210	static const struct file_operations fops_kp = {
2211	.read = read_enabled_file_bool,
2212	.write = write_enabled_file_bool,
2213	.llseek = default_llseek,
2214	};
2215
2216	static int __kprobes debugfs_kprobe_init(void)
2217	{
2218	struct dentry dir, file;
2219	unsigned int value = 1;
2220
2221	dir = debugfs_create_dir("kprobes", NULL);
2222	if (!dir)
2223	return -ENOMEM;
2224
2225	file = debugfs_create_file("list", 0444, dir, NULL,
2226	&debugfs_kprobes_operations);
2227	if (!file) {
2228	debugfs_remove(dir);
2229	return -ENOMEM;
2230	}
2231
2232	file = debugfs_create_file("enabled", 0600, dir,
2233	&value, &fops_kp);
2234	if (!file) {
2235	debugfs_remove(dir);
2236	return -ENOMEM;
2237	}
2238
2239	return 0;
2240	}
2241
2242	late_initcall(debugfs_kprobe_init);
2243	#endif /* CONFIG_DEBUG_FS */
2244
2245	module_init(init_kprobes);
2246
2247	/* defined in arch/.../kernel/kprobes.c */
2248	EXPORT_SYMBOL_GPL(jprobe_return);
2249

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/kernel/kprobes.c

interactive