Root/kernel/kprobes.c

1/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
33 */
34#include <linux/kprobes.h>
35#include <linux/hash.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/stddef.h>
39#include <linux/export.h>
40#include <linux/moduleloader.h>
41#include <linux/kallsyms.h>
42#include <linux/freezer.h>
43#include <linux/seq_file.h>
44#include <linux/debugfs.h>
45#include <linux/sysctl.h>
46#include <linux/kdebug.h>
47#include <linux/memory.h>
48#include <linux/ftrace.h>
49#include <linux/cpu.h>
50#include <linux/jump_label.h>
51
52#include <asm-generic/sections.h>
53#include <asm/cacheflush.h>
54#include <asm/errno.h>
55#include <asm/uaccess.h>
56
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
60
61/*
62 * Some oddball architectures like 64bit powerpc have function descriptors
63 * so this must be overridable.
64 */
65#ifndef kprobe_lookup_name
66#define kprobe_lookup_name(name, addr) \
67    addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
68#endif
69
70static int kprobes_initialized;
71static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
72static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
73
74/* NOTE: change this value only with kprobe_mutex held */
75static bool kprobes_all_disarmed;
76
77/* This protects kprobe_table and optimizing_list */
78static DEFINE_MUTEX(kprobe_mutex);
79static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
80static struct {
81    raw_spinlock_t lock ____cacheline_aligned_in_smp;
82} kretprobe_table_locks[KPROBE_TABLE_SIZE];
83
84static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
85{
86    return &(kretprobe_table_locks[hash].lock);
87}
88
89/*
90 * Normally, functions that we'd want to prohibit kprobes in, are marked
91 * __kprobes. But, there are cases where such functions already belong to
92 * a different section (__sched for preempt_schedule)
93 *
94 * For such cases, we now have a blacklist
95 */
96static struct kprobe_blackpoint kprobe_blacklist[] = {
97    {"preempt_schedule",},
98    {"native_get_debugreg",},
99    {"irq_entries_start",},
100    {"common_interrupt",},
101    {"mcount",}, /* mcount can be called from everywhere */
102    {NULL} /* Terminator */
103};
104
105#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
106/*
107 * kprobe->ainsn.insn points to the copy of the instruction to be
108 * single-stepped. x86_64, POWER4 and above have no-exec support and
109 * stepping on the instruction on a vmalloced/kmalloced/data page
110 * is a recipe for disaster
111 */
112struct kprobe_insn_page {
113    struct list_head list;
114    kprobe_opcode_t *insns; /* Page of instruction slots */
115    int nused;
116    int ngarbage;
117    char slot_used[];
118};
119
120#define KPROBE_INSN_PAGE_SIZE(slots) \
121    (offsetof(struct kprobe_insn_page, slot_used) + \
122     (sizeof(char) * (slots)))
123
124struct kprobe_insn_cache {
125    struct list_head pages; /* list of kprobe_insn_page */
126    size_t insn_size; /* size of instruction slot */
127    int nr_garbage;
128};
129
130static int slots_per_page(struct kprobe_insn_cache *c)
131{
132    return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
133}
134
135enum kprobe_slot_state {
136    SLOT_CLEAN = 0,
137    SLOT_DIRTY = 1,
138    SLOT_USED = 2,
139};
140
141static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
142static struct kprobe_insn_cache kprobe_insn_slots = {
143    .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
144    .insn_size = MAX_INSN_SIZE,
145    .nr_garbage = 0,
146};
147static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
148
149/**
150 * __get_insn_slot() - Find a slot on an executable page for an instruction.
151 * We allocate an executable page if there's no room on existing ones.
152 */
153static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
154{
155    struct kprobe_insn_page *kip;
156
157 retry:
158    list_for_each_entry(kip, &c->pages, list) {
159        if (kip->nused < slots_per_page(c)) {
160            int i;
161            for (i = 0; i < slots_per_page(c); i++) {
162                if (kip->slot_used[i] == SLOT_CLEAN) {
163                    kip->slot_used[i] = SLOT_USED;
164                    kip->nused++;
165                    return kip->insns + (i * c->insn_size);
166                }
167            }
168            /* kip->nused is broken. Fix it. */
169            kip->nused = slots_per_page(c);
170            WARN_ON(1);
171        }
172    }
173
174    /* If there are any garbage slots, collect it and try again. */
175    if (c->nr_garbage && collect_garbage_slots(c) == 0)
176        goto retry;
177
178    /* All out of space. Need to allocate a new page. */
179    kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
180    if (!kip)
181        return NULL;
182
183    /*
184     * Use module_alloc so this page is within +/- 2GB of where the
185     * kernel image and loaded module images reside. This is required
186     * so x86_64 can correctly handle the %rip-relative fixups.
187     */
188    kip->insns = module_alloc(PAGE_SIZE);
189    if (!kip->insns) {
190        kfree(kip);
191        return NULL;
192    }
193    INIT_LIST_HEAD(&kip->list);
194    memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
195    kip->slot_used[0] = SLOT_USED;
196    kip->nused = 1;
197    kip->ngarbage = 0;
198    list_add(&kip->list, &c->pages);
199    return kip->insns;
200}
201
202
203kprobe_opcode_t __kprobes *get_insn_slot(void)
204{
205    kprobe_opcode_t *ret = NULL;
206
207    mutex_lock(&kprobe_insn_mutex);
208    ret = __get_insn_slot(&kprobe_insn_slots);
209    mutex_unlock(&kprobe_insn_mutex);
210
211    return ret;
212}
213
214/* Return 1 if all garbages are collected, otherwise 0. */
215static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
216{
217    kip->slot_used[idx] = SLOT_CLEAN;
218    kip->nused--;
219    if (kip->nused == 0) {
220        /*
221         * Page is no longer in use. Free it unless
222         * it's the last one. We keep the last one
223         * so as not to have to set it up again the
224         * next time somebody inserts a probe.
225         */
226        if (!list_is_singular(&kip->list)) {
227            list_del(&kip->list);
228            module_free(NULL, kip->insns);
229            kfree(kip);
230        }
231        return 1;
232    }
233    return 0;
234}
235
236static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
237{
238    struct kprobe_insn_page *kip, *next;
239
240    /* Ensure no-one is interrupted on the garbages */
241    synchronize_sched();
242
243    list_for_each_entry_safe(kip, next, &c->pages, list) {
244        int i;
245        if (kip->ngarbage == 0)
246            continue;
247        kip->ngarbage = 0; /* we will collect all garbages */
248        for (i = 0; i < slots_per_page(c); i++) {
249            if (kip->slot_used[i] == SLOT_DIRTY &&
250                collect_one_slot(kip, i))
251                break;
252        }
253    }
254    c->nr_garbage = 0;
255    return 0;
256}
257
258static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
259                       kprobe_opcode_t *slot, int dirty)
260{
261    struct kprobe_insn_page *kip;
262
263    list_for_each_entry(kip, &c->pages, list) {
264        long idx = ((long)slot - (long)kip->insns) /
265                (c->insn_size * sizeof(kprobe_opcode_t));
266        if (idx >= 0 && idx < slots_per_page(c)) {
267            WARN_ON(kip->slot_used[idx] != SLOT_USED);
268            if (dirty) {
269                kip->slot_used[idx] = SLOT_DIRTY;
270                kip->ngarbage++;
271                if (++c->nr_garbage > slots_per_page(c))
272                    collect_garbage_slots(c);
273            } else
274                collect_one_slot(kip, idx);
275            return;
276        }
277    }
278    /* Could not free this slot. */
279    WARN_ON(1);
280}
281
282void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
283{
284    mutex_lock(&kprobe_insn_mutex);
285    __free_insn_slot(&kprobe_insn_slots, slot, dirty);
286    mutex_unlock(&kprobe_insn_mutex);
287}
288#ifdef CONFIG_OPTPROBES
289/* For optimized_kprobe buffer */
290static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
291static struct kprobe_insn_cache kprobe_optinsn_slots = {
292    .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
293    /* .insn_size is initialized later */
294    .nr_garbage = 0,
295};
296/* Get a slot for optimized_kprobe buffer */
297kprobe_opcode_t __kprobes *get_optinsn_slot(void)
298{
299    kprobe_opcode_t *ret = NULL;
300
301    mutex_lock(&kprobe_optinsn_mutex);
302    ret = __get_insn_slot(&kprobe_optinsn_slots);
303    mutex_unlock(&kprobe_optinsn_mutex);
304
305    return ret;
306}
307
308void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
309{
310    mutex_lock(&kprobe_optinsn_mutex);
311    __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
312    mutex_unlock(&kprobe_optinsn_mutex);
313}
314#endif
315#endif
316
317/* We have preemption disabled.. so it is safe to use __ versions */
318static inline void set_kprobe_instance(struct kprobe *kp)
319{
320    __this_cpu_write(kprobe_instance, kp);
321}
322
323static inline void reset_kprobe_instance(void)
324{
325    __this_cpu_write(kprobe_instance, NULL);
326}
327
328/*
329 * This routine is called either:
330 * - under the kprobe_mutex - during kprobe_[un]register()
331 * OR
332 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
333 */
334struct kprobe __kprobes *get_kprobe(void *addr)
335{
336    struct hlist_head *head;
337    struct hlist_node *node;
338    struct kprobe *p;
339
340    head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
341    hlist_for_each_entry_rcu(p, node, head, hlist) {
342        if (p->addr == addr)
343            return p;
344    }
345
346    return NULL;
347}
348
349static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
350
351/* Return true if the kprobe is an aggregator */
352static inline int kprobe_aggrprobe(struct kprobe *p)
353{
354    return p->pre_handler == aggr_pre_handler;
355}
356
357/* Return true(!0) if the kprobe is unused */
358static inline int kprobe_unused(struct kprobe *p)
359{
360    return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
361           list_empty(&p->list);
362}
363
364/*
365 * Keep all fields in the kprobe consistent
366 */
367static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
368{
369    memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
370    memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
371}
372
373#ifdef CONFIG_OPTPROBES
374/* NOTE: change this value only with kprobe_mutex held */
375static bool kprobes_allow_optimization;
376
377/*
378 * Call all pre_handler on the list, but ignores its return value.
379 * This must be called from arch-dep optimized caller.
380 */
381void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
382{
383    struct kprobe *kp;
384
385    list_for_each_entry_rcu(kp, &p->list, list) {
386        if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
387            set_kprobe_instance(kp);
388            kp->pre_handler(kp, regs);
389        }
390        reset_kprobe_instance();
391    }
392}
393
394/* Free optimized instructions and optimized_kprobe */
395static __kprobes void free_aggr_kprobe(struct kprobe *p)
396{
397    struct optimized_kprobe *op;
398
399    op = container_of(p, struct optimized_kprobe, kp);
400    arch_remove_optimized_kprobe(op);
401    arch_remove_kprobe(p);
402    kfree(op);
403}
404
405/* Return true(!0) if the kprobe is ready for optimization. */
406static inline int kprobe_optready(struct kprobe *p)
407{
408    struct optimized_kprobe *op;
409
410    if (kprobe_aggrprobe(p)) {
411        op = container_of(p, struct optimized_kprobe, kp);
412        return arch_prepared_optinsn(&op->optinsn);
413    }
414
415    return 0;
416}
417
418/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
419static inline int kprobe_disarmed(struct kprobe *p)
420{
421    struct optimized_kprobe *op;
422
423    /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
424    if (!kprobe_aggrprobe(p))
425        return kprobe_disabled(p);
426
427    op = container_of(p, struct optimized_kprobe, kp);
428
429    return kprobe_disabled(p) && list_empty(&op->list);
430}
431
432/* Return true(!0) if the probe is queued on (un)optimizing lists */
433static int __kprobes kprobe_queued(struct kprobe *p)
434{
435    struct optimized_kprobe *op;
436
437    if (kprobe_aggrprobe(p)) {
438        op = container_of(p, struct optimized_kprobe, kp);
439        if (!list_empty(&op->list))
440            return 1;
441    }
442    return 0;
443}
444
445/*
446 * Return an optimized kprobe whose optimizing code replaces
447 * instructions including addr (exclude breakpoint).
448 */
449static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
450{
451    int i;
452    struct kprobe *p = NULL;
453    struct optimized_kprobe *op;
454
455    /* Don't check i == 0, since that is a breakpoint case. */
456    for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
457        p = get_kprobe((void *)(addr - i));
458
459    if (p && kprobe_optready(p)) {
460        op = container_of(p, struct optimized_kprobe, kp);
461        if (arch_within_optimized_kprobe(op, addr))
462            return p;
463    }
464
465    return NULL;
466}
467
468/* Optimization staging list, protected by kprobe_mutex */
469static LIST_HEAD(optimizing_list);
470static LIST_HEAD(unoptimizing_list);
471
472static void kprobe_optimizer(struct work_struct *work);
473static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
474static DECLARE_COMPLETION(optimizer_comp);
475#define OPTIMIZE_DELAY 5
476
477/*
478 * Optimize (replace a breakpoint with a jump) kprobes listed on
479 * optimizing_list.
480 */
481static __kprobes void do_optimize_kprobes(void)
482{
483    /* Optimization never be done when disarmed */
484    if (kprobes_all_disarmed || !kprobes_allow_optimization ||
485        list_empty(&optimizing_list))
486        return;
487
488    /*
489     * The optimization/unoptimization refers online_cpus via
490     * stop_machine() and cpu-hotplug modifies online_cpus.
491     * And same time, text_mutex will be held in cpu-hotplug and here.
492     * This combination can cause a deadlock (cpu-hotplug try to lock
493     * text_mutex but stop_machine can not be done because online_cpus
494     * has been changed)
495     * To avoid this deadlock, we need to call get_online_cpus()
496     * for preventing cpu-hotplug outside of text_mutex locking.
497     */
498    get_online_cpus();
499    mutex_lock(&text_mutex);
500    arch_optimize_kprobes(&optimizing_list);
501    mutex_unlock(&text_mutex);
502    put_online_cpus();
503}
504
505/*
506 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
507 * if need) kprobes listed on unoptimizing_list.
508 */
509static __kprobes void do_unoptimize_kprobes(struct list_head *free_list)
510{
511    struct optimized_kprobe *op, *tmp;
512
513    /* Unoptimization must be done anytime */
514    if (list_empty(&unoptimizing_list))
515        return;
516
517    /* Ditto to do_optimize_kprobes */
518    get_online_cpus();
519    mutex_lock(&text_mutex);
520    arch_unoptimize_kprobes(&unoptimizing_list, free_list);
521    /* Loop free_list for disarming */
522    list_for_each_entry_safe(op, tmp, free_list, list) {
523        /* Disarm probes if marked disabled */
524        if (kprobe_disabled(&op->kp))
525            arch_disarm_kprobe(&op->kp);
526        if (kprobe_unused(&op->kp)) {
527            /*
528             * Remove unused probes from hash list. After waiting
529             * for synchronization, these probes are reclaimed.
530             * (reclaiming is done by do_free_cleaned_kprobes.)
531             */
532            hlist_del_rcu(&op->kp.hlist);
533        } else
534            list_del_init(&op->list);
535    }
536    mutex_unlock(&text_mutex);
537    put_online_cpus();
538}
539
540/* Reclaim all kprobes on the free_list */
541static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list)
542{
543    struct optimized_kprobe *op, *tmp;
544
545    list_for_each_entry_safe(op, tmp, free_list, list) {
546        BUG_ON(!kprobe_unused(&op->kp));
547        list_del_init(&op->list);
548        free_aggr_kprobe(&op->kp);
549    }
550}
551
552/* Start optimizer after OPTIMIZE_DELAY passed */
553static __kprobes void kick_kprobe_optimizer(void)
554{
555    if (!delayed_work_pending(&optimizing_work))
556        schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
557}
558
559/* Kprobe jump optimizer */
560static __kprobes void kprobe_optimizer(struct work_struct *work)
561{
562    LIST_HEAD(free_list);
563
564    /* Lock modules while optimizing kprobes */
565    mutex_lock(&module_mutex);
566    mutex_lock(&kprobe_mutex);
567
568    /*
569     * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
570     * kprobes before waiting for quiesence period.
571     */
572    do_unoptimize_kprobes(&free_list);
573
574    /*
575     * Step 2: Wait for quiesence period to ensure all running interrupts
576     * are done. Because optprobe may modify multiple instructions
577     * there is a chance that Nth instruction is interrupted. In that
578     * case, running interrupt can return to 2nd-Nth byte of jump
579     * instruction. This wait is for avoiding it.
580     */
581    synchronize_sched();
582
583    /* Step 3: Optimize kprobes after quiesence period */
584    do_optimize_kprobes();
585
586    /* Step 4: Free cleaned kprobes after quiesence period */
587    do_free_cleaned_kprobes(&free_list);
588
589    mutex_unlock(&kprobe_mutex);
590    mutex_unlock(&module_mutex);
591
592    /* Step 5: Kick optimizer again if needed */
593    if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
594        kick_kprobe_optimizer();
595    else
596        /* Wake up all waiters */
597        complete_all(&optimizer_comp);
598}
599
600/* Wait for completing optimization and unoptimization */
601static __kprobes void wait_for_kprobe_optimizer(void)
602{
603    if (delayed_work_pending(&optimizing_work))
604        wait_for_completion(&optimizer_comp);
605}
606
607/* Optimize kprobe if p is ready to be optimized */
608static __kprobes void optimize_kprobe(struct kprobe *p)
609{
610    struct optimized_kprobe *op;
611
612    /* Check if the kprobe is disabled or not ready for optimization. */
613    if (!kprobe_optready(p) || !kprobes_allow_optimization ||
614        (kprobe_disabled(p) || kprobes_all_disarmed))
615        return;
616
617    /* Both of break_handler and post_handler are not supported. */
618    if (p->break_handler || p->post_handler)
619        return;
620
621    op = container_of(p, struct optimized_kprobe, kp);
622
623    /* Check there is no other kprobes at the optimized instructions */
624    if (arch_check_optimized_kprobe(op) < 0)
625        return;
626
627    /* Check if it is already optimized. */
628    if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
629        return;
630    op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
631
632    if (!list_empty(&op->list))
633        /* This is under unoptimizing. Just dequeue the probe */
634        list_del_init(&op->list);
635    else {
636        list_add(&op->list, &optimizing_list);
637        kick_kprobe_optimizer();
638    }
639}
640
641/* Short cut to direct unoptimizing */
642static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)
643{
644    get_online_cpus();
645    arch_unoptimize_kprobe(op);
646    put_online_cpus();
647    if (kprobe_disabled(&op->kp))
648        arch_disarm_kprobe(&op->kp);
649}
650
651/* Unoptimize a kprobe if p is optimized */
652static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)
653{
654    struct optimized_kprobe *op;
655
656    if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
657        return; /* This is not an optprobe nor optimized */
658
659    op = container_of(p, struct optimized_kprobe, kp);
660    if (!kprobe_optimized(p)) {
661        /* Unoptimized or unoptimizing case */
662        if (force && !list_empty(&op->list)) {
663            /*
664             * Only if this is unoptimizing kprobe and forced,
665             * forcibly unoptimize it. (No need to unoptimize
666             * unoptimized kprobe again :)
667             */
668            list_del_init(&op->list);
669            force_unoptimize_kprobe(op);
670        }
671        return;
672    }
673
674    op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
675    if (!list_empty(&op->list)) {
676        /* Dequeue from the optimization queue */
677        list_del_init(&op->list);
678        return;
679    }
680    /* Optimized kprobe case */
681    if (force)
682        /* Forcibly update the code: this is a special case */
683        force_unoptimize_kprobe(op);
684    else {
685        list_add(&op->list, &unoptimizing_list);
686        kick_kprobe_optimizer();
687    }
688}
689
690/* Cancel unoptimizing for reusing */
691static void reuse_unused_kprobe(struct kprobe *ap)
692{
693    struct optimized_kprobe *op;
694
695    BUG_ON(!kprobe_unused(ap));
696    /*
697     * Unused kprobe MUST be on the way of delayed unoptimizing (means
698     * there is still a relative jump) and disabled.
699     */
700    op = container_of(ap, struct optimized_kprobe, kp);
701    if (unlikely(list_empty(&op->list)))
702        printk(KERN_WARNING "Warning: found a stray unused "
703            "aggrprobe@%p\n", ap->addr);
704    /* Enable the probe again */
705    ap->flags &= ~KPROBE_FLAG_DISABLED;
706    /* Optimize it again (remove from op->list) */
707    BUG_ON(!kprobe_optready(ap));
708    optimize_kprobe(ap);
709}
710
711/* Remove optimized instructions */
712static void __kprobes kill_optimized_kprobe(struct kprobe *p)
713{
714    struct optimized_kprobe *op;
715
716    op = container_of(p, struct optimized_kprobe, kp);
717    if (!list_empty(&op->list))
718        /* Dequeue from the (un)optimization queue */
719        list_del_init(&op->list);
720
721    op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
722    /* Don't touch the code, because it is already freed. */
723    arch_remove_optimized_kprobe(op);
724}
725
726/* Try to prepare optimized instructions */
727static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
728{
729    struct optimized_kprobe *op;
730
731    op = container_of(p, struct optimized_kprobe, kp);
732    arch_prepare_optimized_kprobe(op);
733}
734
735/* Allocate new optimized_kprobe and try to prepare optimized instructions */
736static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
737{
738    struct optimized_kprobe *op;
739
740    op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
741    if (!op)
742        return NULL;
743
744    INIT_LIST_HEAD(&op->list);
745    op->kp.addr = p->addr;
746    arch_prepare_optimized_kprobe(op);
747
748    return &op->kp;
749}
750
751static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
752
753/*
754 * Prepare an optimized_kprobe and optimize it
755 * NOTE: p must be a normal registered kprobe
756 */
757static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
758{
759    struct kprobe *ap;
760    struct optimized_kprobe *op;
761
762    ap = alloc_aggr_kprobe(p);
763    if (!ap)
764        return;
765
766    op = container_of(ap, struct optimized_kprobe, kp);
767    if (!arch_prepared_optinsn(&op->optinsn)) {
768        /* If failed to setup optimizing, fallback to kprobe */
769        arch_remove_optimized_kprobe(op);
770        kfree(op);
771        return;
772    }
773
774    init_aggr_kprobe(ap, p);
775    optimize_kprobe(ap);
776}
777
778#ifdef CONFIG_SYSCTL
779/* This should be called with kprobe_mutex locked */
780static void __kprobes optimize_all_kprobes(void)
781{
782    struct hlist_head *head;
783    struct hlist_node *node;
784    struct kprobe *p;
785    unsigned int i;
786
787    /* If optimization is already allowed, just return */
788    if (kprobes_allow_optimization)
789        return;
790
791    kprobes_allow_optimization = true;
792    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
793        head = &kprobe_table[i];
794        hlist_for_each_entry_rcu(p, node, head, hlist)
795            if (!kprobe_disabled(p))
796                optimize_kprobe(p);
797    }
798    printk(KERN_INFO "Kprobes globally optimized\n");
799}
800
801/* This should be called with kprobe_mutex locked */
802static void __kprobes unoptimize_all_kprobes(void)
803{
804    struct hlist_head *head;
805    struct hlist_node *node;
806    struct kprobe *p;
807    unsigned int i;
808
809    /* If optimization is already prohibited, just return */
810    if (!kprobes_allow_optimization)
811        return;
812
813    kprobes_allow_optimization = false;
814    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
815        head = &kprobe_table[i];
816        hlist_for_each_entry_rcu(p, node, head, hlist) {
817            if (!kprobe_disabled(p))
818                unoptimize_kprobe(p, false);
819        }
820    }
821    /* Wait for unoptimizing completion */
822    wait_for_kprobe_optimizer();
823    printk(KERN_INFO "Kprobes globally unoptimized\n");
824}
825
826int sysctl_kprobes_optimization;
827int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
828                      void __user *buffer, size_t *length,
829                      loff_t *ppos)
830{
831    int ret;
832
833    mutex_lock(&kprobe_mutex);
834    sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
835    ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
836
837    if (sysctl_kprobes_optimization)
838        optimize_all_kprobes();
839    else
840        unoptimize_all_kprobes();
841    mutex_unlock(&kprobe_mutex);
842
843    return ret;
844}
845#endif /* CONFIG_SYSCTL */
846
847/* Put a breakpoint for a probe. Must be called with text_mutex locked */
848static void __kprobes __arm_kprobe(struct kprobe *p)
849{
850    struct kprobe *_p;
851
852    /* Check collision with other optimized kprobes */
853    _p = get_optimized_kprobe((unsigned long)p->addr);
854    if (unlikely(_p))
855        /* Fallback to unoptimized kprobe */
856        unoptimize_kprobe(_p, true);
857
858    arch_arm_kprobe(p);
859    optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
860}
861
862/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
863static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)
864{
865    struct kprobe *_p;
866
867    unoptimize_kprobe(p, false); /* Try to unoptimize */
868
869    if (!kprobe_queued(p)) {
870        arch_disarm_kprobe(p);
871        /* If another kprobe was blocked, optimize it. */
872        _p = get_optimized_kprobe((unsigned long)p->addr);
873        if (unlikely(_p) && reopt)
874            optimize_kprobe(_p);
875    }
876    /* TODO: reoptimize others after unoptimized this probe */
877}
878
879#else /* !CONFIG_OPTPROBES */
880
881#define optimize_kprobe(p) do {} while (0)
882#define unoptimize_kprobe(p, f) do {} while (0)
883#define kill_optimized_kprobe(p) do {} while (0)
884#define prepare_optimized_kprobe(p) do {} while (0)
885#define try_to_optimize_kprobe(p) do {} while (0)
886#define __arm_kprobe(p) arch_arm_kprobe(p)
887#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
888#define kprobe_disarmed(p) kprobe_disabled(p)
889#define wait_for_kprobe_optimizer() do {} while (0)
890
891/* There should be no unused kprobes can be reused without optimization */
892static void reuse_unused_kprobe(struct kprobe *ap)
893{
894    printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
895    BUG_ON(kprobe_unused(ap));
896}
897
898static __kprobes void free_aggr_kprobe(struct kprobe *p)
899{
900    arch_remove_kprobe(p);
901    kfree(p);
902}
903
904static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
905{
906    return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
907}
908#endif /* CONFIG_OPTPROBES */
909
910/* Arm a kprobe with text_mutex */
911static void __kprobes arm_kprobe(struct kprobe *kp)
912{
913    /*
914     * Here, since __arm_kprobe() doesn't use stop_machine(),
915     * this doesn't cause deadlock on text_mutex. So, we don't
916     * need get_online_cpus().
917     */
918    mutex_lock(&text_mutex);
919    __arm_kprobe(kp);
920    mutex_unlock(&text_mutex);
921}
922
923/* Disarm a kprobe with text_mutex */
924static void __kprobes disarm_kprobe(struct kprobe *kp)
925{
926    /* Ditto */
927    mutex_lock(&text_mutex);
928    __disarm_kprobe(kp, true);
929    mutex_unlock(&text_mutex);
930}
931
932/*
933 * Aggregate handlers for multiple kprobes support - these handlers
934 * take care of invoking the individual kprobe handlers on p->list
935 */
936static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
937{
938    struct kprobe *kp;
939
940    list_for_each_entry_rcu(kp, &p->list, list) {
941        if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
942            set_kprobe_instance(kp);
943            if (kp->pre_handler(kp, regs))
944                return 1;
945        }
946        reset_kprobe_instance();
947    }
948    return 0;
949}
950
951static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
952                    unsigned long flags)
953{
954    struct kprobe *kp;
955
956    list_for_each_entry_rcu(kp, &p->list, list) {
957        if (kp->post_handler && likely(!kprobe_disabled(kp))) {
958            set_kprobe_instance(kp);
959            kp->post_handler(kp, regs, flags);
960            reset_kprobe_instance();
961        }
962    }
963}
964
965static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
966                    int trapnr)
967{
968    struct kprobe *cur = __this_cpu_read(kprobe_instance);
969
970    /*
971     * if we faulted "during" the execution of a user specified
972     * probe handler, invoke just that probe's fault handler
973     */
974    if (cur && cur->fault_handler) {
975        if (cur->fault_handler(cur, regs, trapnr))
976            return 1;
977    }
978    return 0;
979}
980
981static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
982{
983    struct kprobe *cur = __this_cpu_read(kprobe_instance);
984    int ret = 0;
985
986    if (cur && cur->break_handler) {
987        if (cur->break_handler(cur, regs))
988            ret = 1;
989    }
990    reset_kprobe_instance();
991    return ret;
992}
993
994/* Walks the list and increments nmissed count for multiprobe case */
995void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
996{
997    struct kprobe *kp;
998    if (!kprobe_aggrprobe(p)) {
999        p->nmissed++;
1000    } else {
1001        list_for_each_entry_rcu(kp, &p->list, list)
1002            kp->nmissed++;
1003    }
1004    return;
1005}
1006
1007void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
1008                struct hlist_head *head)
1009{
1010    struct kretprobe *rp = ri->rp;
1011
1012    /* remove rp inst off the rprobe_inst_table */
1013    hlist_del(&ri->hlist);
1014    INIT_HLIST_NODE(&ri->hlist);
1015    if (likely(rp)) {
1016        raw_spin_lock(&rp->lock);
1017        hlist_add_head(&ri->hlist, &rp->free_instances);
1018        raw_spin_unlock(&rp->lock);
1019    } else
1020        /* Unregistering */
1021        hlist_add_head(&ri->hlist, head);
1022}
1023
1024void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
1025             struct hlist_head **head, unsigned long *flags)
1026__acquires(hlist_lock)
1027{
1028    unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1029    raw_spinlock_t *hlist_lock;
1030
1031    *head = &kretprobe_inst_table[hash];
1032    hlist_lock = kretprobe_table_lock_ptr(hash);
1033    raw_spin_lock_irqsave(hlist_lock, *flags);
1034}
1035
1036static void __kprobes kretprobe_table_lock(unsigned long hash,
1037    unsigned long *flags)
1038__acquires(hlist_lock)
1039{
1040    raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1041    raw_spin_lock_irqsave(hlist_lock, *flags);
1042}
1043
1044void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
1045    unsigned long *flags)
1046__releases(hlist_lock)
1047{
1048    unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1049    raw_spinlock_t *hlist_lock;
1050
1051    hlist_lock = kretprobe_table_lock_ptr(hash);
1052    raw_spin_unlock_irqrestore(hlist_lock, *flags);
1053}
1054
1055static void __kprobes kretprobe_table_unlock(unsigned long hash,
1056       unsigned long *flags)
1057__releases(hlist_lock)
1058{
1059    raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1060    raw_spin_unlock_irqrestore(hlist_lock, *flags);
1061}
1062
1063/*
1064 * This function is called from finish_task_switch when task tk becomes dead,
1065 * so that we can recycle any function-return probe instances associated
1066 * with this task. These left over instances represent probed functions
1067 * that have been called but will never return.
1068 */
1069void __kprobes kprobe_flush_task(struct task_struct *tk)
1070{
1071    struct kretprobe_instance *ri;
1072    struct hlist_head *head, empty_rp;
1073    struct hlist_node *node, *tmp;
1074    unsigned long hash, flags = 0;
1075
1076    if (unlikely(!kprobes_initialized))
1077        /* Early boot. kretprobe_table_locks not yet initialized. */
1078        return;
1079
1080    INIT_HLIST_HEAD(&empty_rp);
1081    hash = hash_ptr(tk, KPROBE_HASH_BITS);
1082    head = &kretprobe_inst_table[hash];
1083    kretprobe_table_lock(hash, &flags);
1084    hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
1085        if (ri->task == tk)
1086            recycle_rp_inst(ri, &empty_rp);
1087    }
1088    kretprobe_table_unlock(hash, &flags);
1089    hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
1090        hlist_del(&ri->hlist);
1091        kfree(ri);
1092    }
1093}
1094
1095static inline void free_rp_inst(struct kretprobe *rp)
1096{
1097    struct kretprobe_instance *ri;
1098    struct hlist_node *pos, *next;
1099
1100    hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
1101        hlist_del(&ri->hlist);
1102        kfree(ri);
1103    }
1104}
1105
1106static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
1107{
1108    unsigned long flags, hash;
1109    struct kretprobe_instance *ri;
1110    struct hlist_node *pos, *next;
1111    struct hlist_head *head;
1112
1113    /* No race here */
1114    for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1115        kretprobe_table_lock(hash, &flags);
1116        head = &kretprobe_inst_table[hash];
1117        hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
1118            if (ri->rp == rp)
1119                ri->rp = NULL;
1120        }
1121        kretprobe_table_unlock(hash, &flags);
1122    }
1123    free_rp_inst(rp);
1124}
1125
1126/*
1127* Add the new probe to ap->list. Fail if this is the
1128* second jprobe at the address - two jprobes can't coexist
1129*/
1130static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1131{
1132    BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
1133
1134    if (p->break_handler || p->post_handler)
1135        unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
1136
1137    if (p->break_handler) {
1138        if (ap->break_handler)
1139            return -EEXIST;
1140        list_add_tail_rcu(&p->list, &ap->list);
1141        ap->break_handler = aggr_break_handler;
1142    } else
1143        list_add_rcu(&p->list, &ap->list);
1144    if (p->post_handler && !ap->post_handler)
1145        ap->post_handler = aggr_post_handler;
1146
1147    if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
1148        ap->flags &= ~KPROBE_FLAG_DISABLED;
1149        if (!kprobes_all_disarmed)
1150            /* Arm the breakpoint again. */
1151            __arm_kprobe(ap);
1152    }
1153    return 0;
1154}
1155
1156/*
1157 * Fill in the required fields of the "manager kprobe". Replace the
1158 * earlier kprobe in the hlist with the manager kprobe
1159 */
1160static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1161{
1162    /* Copy p's insn slot to ap */
1163    copy_kprobe(p, ap);
1164    flush_insn_slot(ap);
1165    ap->addr = p->addr;
1166    ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1167    ap->pre_handler = aggr_pre_handler;
1168    ap->fault_handler = aggr_fault_handler;
1169    /* We don't care the kprobe which has gone. */
1170    if (p->post_handler && !kprobe_gone(p))
1171        ap->post_handler = aggr_post_handler;
1172    if (p->break_handler && !kprobe_gone(p))
1173        ap->break_handler = aggr_break_handler;
1174
1175    INIT_LIST_HEAD(&ap->list);
1176    INIT_HLIST_NODE(&ap->hlist);
1177
1178    list_add_rcu(&p->list, &ap->list);
1179    hlist_replace_rcu(&p->hlist, &ap->hlist);
1180}
1181
1182/*
1183 * This is the second or subsequent kprobe at the address - handle
1184 * the intricacies
1185 */
1186static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,
1187                      struct kprobe *p)
1188{
1189    int ret = 0;
1190    struct kprobe *ap = orig_p;
1191
1192    if (!kprobe_aggrprobe(orig_p)) {
1193        /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1194        ap = alloc_aggr_kprobe(orig_p);
1195        if (!ap)
1196            return -ENOMEM;
1197        init_aggr_kprobe(ap, orig_p);
1198    } else if (kprobe_unused(ap))
1199        /* This probe is going to die. Rescue it */
1200        reuse_unused_kprobe(ap);
1201
1202    if (kprobe_gone(ap)) {
1203        /*
1204         * Attempting to insert new probe at the same location that
1205         * had a probe in the module vaddr area which already
1206         * freed. So, the instruction slot has already been
1207         * released. We need a new slot for the new probe.
1208         */
1209        ret = arch_prepare_kprobe(ap);
1210        if (ret)
1211            /*
1212             * Even if fail to allocate new slot, don't need to
1213             * free aggr_probe. It will be used next time, or
1214             * freed by unregister_kprobe.
1215             */
1216            return ret;
1217
1218        /* Prepare optimized instructions if possible. */
1219        prepare_optimized_kprobe(ap);
1220
1221        /*
1222         * Clear gone flag to prevent allocating new slot again, and
1223         * set disabled flag because it is not armed yet.
1224         */
1225        ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1226                | KPROBE_FLAG_DISABLED;
1227    }
1228
1229    /* Copy ap's insn slot to p */
1230    copy_kprobe(ap, p);
1231    return add_new_kprobe(ap, p);
1232}
1233
1234static int __kprobes in_kprobes_functions(unsigned long addr)
1235{
1236    struct kprobe_blackpoint *kb;
1237
1238    if (addr >= (unsigned long)__kprobes_text_start &&
1239        addr < (unsigned long)__kprobes_text_end)
1240        return -EINVAL;
1241    /*
1242     * If there exists a kprobe_blacklist, verify and
1243     * fail any probe registration in the prohibited area
1244     */
1245    for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1246        if (kb->start_addr) {
1247            if (addr >= kb->start_addr &&
1248                addr < (kb->start_addr + kb->range))
1249                return -EINVAL;
1250        }
1251    }
1252    return 0;
1253}
1254
1255/*
1256 * If we have a symbol_name argument, look it up and add the offset field
1257 * to it. This way, we can specify a relative address to a symbol.
1258 * This returns encoded errors if it fails to look up symbol or invalid
1259 * combination of parameters.
1260 */
1261static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1262{
1263    kprobe_opcode_t *addr = p->addr;
1264
1265    if ((p->symbol_name && p->addr) ||
1266        (!p->symbol_name && !p->addr))
1267        goto invalid;
1268
1269    if (p->symbol_name) {
1270        kprobe_lookup_name(p->symbol_name, addr);
1271        if (!addr)
1272            return ERR_PTR(-ENOENT);
1273    }
1274
1275    addr = (kprobe_opcode_t *)(((char *)addr) + p->offset);
1276    if (addr)
1277        return addr;
1278
1279invalid:
1280    return ERR_PTR(-EINVAL);
1281}
1282
1283/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1284static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1285{
1286    struct kprobe *ap, *list_p;
1287
1288    ap = get_kprobe(p->addr);
1289    if (unlikely(!ap))
1290        return NULL;
1291
1292    if (p != ap) {
1293        list_for_each_entry_rcu(list_p, &ap->list, list)
1294            if (list_p == p)
1295            /* kprobe p is a valid probe */
1296                goto valid;
1297        return NULL;
1298    }
1299valid:
1300    return ap;
1301}
1302
1303/* Return error if the kprobe is being re-registered */
1304static inline int check_kprobe_rereg(struct kprobe *p)
1305{
1306    int ret = 0;
1307
1308    mutex_lock(&kprobe_mutex);
1309    if (__get_valid_kprobe(p))
1310        ret = -EINVAL;
1311    mutex_unlock(&kprobe_mutex);
1312
1313    return ret;
1314}
1315
1316int __kprobes register_kprobe(struct kprobe *p)
1317{
1318    int ret = 0;
1319    struct kprobe *old_p;
1320    struct module *probed_mod;
1321    kprobe_opcode_t *addr;
1322
1323    addr = kprobe_addr(p);
1324    if (IS_ERR(addr))
1325        return PTR_ERR(addr);
1326    p->addr = addr;
1327
1328    ret = check_kprobe_rereg(p);
1329    if (ret)
1330        return ret;
1331
1332    jump_label_lock();
1333    preempt_disable();
1334    if (!kernel_text_address((unsigned long) p->addr) ||
1335        in_kprobes_functions((unsigned long) p->addr) ||
1336        ftrace_text_reserved(p->addr, p->addr) ||
1337        jump_label_text_reserved(p->addr, p->addr)) {
1338        ret = -EINVAL;
1339        goto cannot_probe;
1340    }
1341
1342    /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1343    p->flags &= KPROBE_FLAG_DISABLED;
1344
1345    /*
1346     * Check if are we probing a module.
1347     */
1348    probed_mod = __module_text_address((unsigned long) p->addr);
1349    if (probed_mod) {
1350        /* Return -ENOENT if fail. */
1351        ret = -ENOENT;
1352        /*
1353         * We must hold a refcount of the probed module while updating
1354         * its code to prohibit unexpected unloading.
1355         */
1356        if (unlikely(!try_module_get(probed_mod)))
1357            goto cannot_probe;
1358
1359        /*
1360         * If the module freed .init.text, we couldn't insert
1361         * kprobes in there.
1362         */
1363        if (within_module_init((unsigned long)p->addr, probed_mod) &&
1364            probed_mod->state != MODULE_STATE_COMING) {
1365            module_put(probed_mod);
1366            goto cannot_probe;
1367        }
1368        /* ret will be updated by following code */
1369    }
1370    preempt_enable();
1371    jump_label_unlock();
1372
1373    p->nmissed = 0;
1374    INIT_LIST_HEAD(&p->list);
1375    mutex_lock(&kprobe_mutex);
1376
1377    jump_label_lock(); /* needed to call jump_label_text_reserved() */
1378
1379    get_online_cpus(); /* For avoiding text_mutex deadlock. */
1380    mutex_lock(&text_mutex);
1381
1382    old_p = get_kprobe(p->addr);
1383    if (old_p) {
1384        /* Since this may unoptimize old_p, locking text_mutex. */
1385        ret = register_aggr_kprobe(old_p, p);
1386        goto out;
1387    }
1388
1389    ret = arch_prepare_kprobe(p);
1390    if (ret)
1391        goto out;
1392
1393    INIT_HLIST_NODE(&p->hlist);
1394    hlist_add_head_rcu(&p->hlist,
1395               &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1396
1397    if (!kprobes_all_disarmed && !kprobe_disabled(p))
1398        __arm_kprobe(p);
1399
1400    /* Try to optimize kprobe */
1401    try_to_optimize_kprobe(p);
1402
1403out:
1404    mutex_unlock(&text_mutex);
1405    put_online_cpus();
1406    jump_label_unlock();
1407    mutex_unlock(&kprobe_mutex);
1408
1409    if (probed_mod)
1410        module_put(probed_mod);
1411
1412    return ret;
1413
1414cannot_probe:
1415    preempt_enable();
1416    jump_label_unlock();
1417    return ret;
1418}
1419EXPORT_SYMBOL_GPL(register_kprobe);
1420
1421/* Check if all probes on the aggrprobe are disabled */
1422static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)
1423{
1424    struct kprobe *kp;
1425
1426    list_for_each_entry_rcu(kp, &ap->list, list)
1427        if (!kprobe_disabled(kp))
1428            /*
1429             * There is an active probe on the list.
1430             * We can't disable this ap.
1431             */
1432            return 0;
1433
1434    return 1;
1435}
1436
1437/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1438static struct kprobe *__kprobes __disable_kprobe(struct kprobe *p)
1439{
1440    struct kprobe *orig_p;
1441
1442    /* Get an original kprobe for return */
1443    orig_p = __get_valid_kprobe(p);
1444    if (unlikely(orig_p == NULL))
1445        return NULL;
1446
1447    if (!kprobe_disabled(p)) {
1448        /* Disable probe if it is a child probe */
1449        if (p != orig_p)
1450            p->flags |= KPROBE_FLAG_DISABLED;
1451
1452        /* Try to disarm and disable this/parent probe */
1453        if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1454            disarm_kprobe(orig_p);
1455            orig_p->flags |= KPROBE_FLAG_DISABLED;
1456        }
1457    }
1458
1459    return orig_p;
1460}
1461
1462/*
1463 * Unregister a kprobe without a scheduler synchronization.
1464 */
1465static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1466{
1467    struct kprobe *ap, *list_p;
1468
1469    /* Disable kprobe. This will disarm it if needed. */
1470    ap = __disable_kprobe(p);
1471    if (ap == NULL)
1472        return -EINVAL;
1473
1474    if (ap == p)
1475        /*
1476         * This probe is an independent(and non-optimized) kprobe
1477         * (not an aggrprobe). Remove from the hash list.
1478         */
1479        goto disarmed;
1480
1481    /* Following process expects this probe is an aggrprobe */
1482    WARN_ON(!kprobe_aggrprobe(ap));
1483
1484    if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1485        /*
1486         * !disarmed could be happen if the probe is under delayed
1487         * unoptimizing.
1488         */
1489        goto disarmed;
1490    else {
1491        /* If disabling probe has special handlers, update aggrprobe */
1492        if (p->break_handler && !kprobe_gone(p))
1493            ap->break_handler = NULL;
1494        if (p->post_handler && !kprobe_gone(p)) {
1495            list_for_each_entry_rcu(list_p, &ap->list, list) {
1496                if ((list_p != p) && (list_p->post_handler))
1497                    goto noclean;
1498            }
1499            ap->post_handler = NULL;
1500        }
1501noclean:
1502        /*
1503         * Remove from the aggrprobe: this path will do nothing in
1504         * __unregister_kprobe_bottom().
1505         */
1506        list_del_rcu(&p->list);
1507        if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1508            /*
1509             * Try to optimize this probe again, because post
1510             * handler may have been changed.
1511             */
1512            optimize_kprobe(ap);
1513    }
1514    return 0;
1515
1516disarmed:
1517    BUG_ON(!kprobe_disarmed(ap));
1518    hlist_del_rcu(&ap->hlist);
1519    return 0;
1520}
1521
1522static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1523{
1524    struct kprobe *ap;
1525
1526    if (list_empty(&p->list))
1527        /* This is an independent kprobe */
1528        arch_remove_kprobe(p);
1529    else if (list_is_singular(&p->list)) {
1530        /* This is the last child of an aggrprobe */
1531        ap = list_entry(p->list.next, struct kprobe, list);
1532        list_del(&p->list);
1533        free_aggr_kprobe(ap);
1534    }
1535    /* Otherwise, do nothing. */
1536}
1537
1538int __kprobes register_kprobes(struct kprobe **kps, int num)
1539{
1540    int i, ret = 0;
1541
1542    if (num <= 0)
1543        return -EINVAL;
1544    for (i = 0; i < num; i++) {
1545        ret = register_kprobe(kps[i]);
1546        if (ret < 0) {
1547            if (i > 0)
1548                unregister_kprobes(kps, i);
1549            break;
1550        }
1551    }
1552    return ret;
1553}
1554EXPORT_SYMBOL_GPL(register_kprobes);
1555
1556void __kprobes unregister_kprobe(struct kprobe *p)
1557{
1558    unregister_kprobes(&p, 1);
1559}
1560EXPORT_SYMBOL_GPL(unregister_kprobe);
1561
1562void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1563{
1564    int i;
1565
1566    if (num <= 0)
1567        return;
1568    mutex_lock(&kprobe_mutex);
1569    for (i = 0; i < num; i++)
1570        if (__unregister_kprobe_top(kps[i]) < 0)
1571            kps[i]->addr = NULL;
1572    mutex_unlock(&kprobe_mutex);
1573
1574    synchronize_sched();
1575    for (i = 0; i < num; i++)
1576        if (kps[i]->addr)
1577            __unregister_kprobe_bottom(kps[i]);
1578}
1579EXPORT_SYMBOL_GPL(unregister_kprobes);
1580
1581static struct notifier_block kprobe_exceptions_nb = {
1582    .notifier_call = kprobe_exceptions_notify,
1583    .priority = 0x7fffffff /* we need to be notified first */
1584};
1585
1586unsigned long __weak arch_deref_entry_point(void *entry)
1587{
1588    return (unsigned long)entry;
1589}
1590
1591int __kprobes register_jprobes(struct jprobe **jps, int num)
1592{
1593    struct jprobe *jp;
1594    int ret = 0, i;
1595
1596    if (num <= 0)
1597        return -EINVAL;
1598    for (i = 0; i < num; i++) {
1599        unsigned long addr, offset;
1600        jp = jps[i];
1601        addr = arch_deref_entry_point(jp->entry);
1602
1603        /* Verify probepoint is a function entry point */
1604        if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1605            offset == 0) {
1606            jp->kp.pre_handler = setjmp_pre_handler;
1607            jp->kp.break_handler = longjmp_break_handler;
1608            ret = register_kprobe(&jp->kp);
1609        } else
1610            ret = -EINVAL;
1611
1612        if (ret < 0) {
1613            if (i > 0)
1614                unregister_jprobes(jps, i);
1615            break;
1616        }
1617    }
1618    return ret;
1619}
1620EXPORT_SYMBOL_GPL(register_jprobes);
1621
1622int __kprobes register_jprobe(struct jprobe *jp)
1623{
1624    return register_jprobes(&jp, 1);
1625}
1626EXPORT_SYMBOL_GPL(register_jprobe);
1627
1628void __kprobes unregister_jprobe(struct jprobe *jp)
1629{
1630    unregister_jprobes(&jp, 1);
1631}
1632EXPORT_SYMBOL_GPL(unregister_jprobe);
1633
1634void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1635{
1636    int i;
1637
1638    if (num <= 0)
1639        return;
1640    mutex_lock(&kprobe_mutex);
1641    for (i = 0; i < num; i++)
1642        if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1643            jps[i]->kp.addr = NULL;
1644    mutex_unlock(&kprobe_mutex);
1645
1646    synchronize_sched();
1647    for (i = 0; i < num; i++) {
1648        if (jps[i]->kp.addr)
1649            __unregister_kprobe_bottom(&jps[i]->kp);
1650    }
1651}
1652EXPORT_SYMBOL_GPL(unregister_jprobes);
1653
1654#ifdef CONFIG_KRETPROBES
1655/*
1656 * This kprobe pre_handler is registered with every kretprobe. When probe
1657 * hits it will set up the return probe.
1658 */
1659static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1660                       struct pt_regs *regs)
1661{
1662    struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1663    unsigned long hash, flags = 0;
1664    struct kretprobe_instance *ri;
1665
1666    /*TODO: consider to only swap the RA after the last pre_handler fired */
1667    hash = hash_ptr(current, KPROBE_HASH_BITS);
1668    raw_spin_lock_irqsave(&rp->lock, flags);
1669    if (!hlist_empty(&rp->free_instances)) {
1670        ri = hlist_entry(rp->free_instances.first,
1671                struct kretprobe_instance, hlist);
1672        hlist_del(&ri->hlist);
1673        raw_spin_unlock_irqrestore(&rp->lock, flags);
1674
1675        ri->rp = rp;
1676        ri->task = current;
1677
1678        if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1679            raw_spin_lock_irqsave(&rp->lock, flags);
1680            hlist_add_head(&ri->hlist, &rp->free_instances);
1681            raw_spin_unlock_irqrestore(&rp->lock, flags);
1682            return 0;
1683        }
1684
1685        arch_prepare_kretprobe(ri, regs);
1686
1687        /* XXX(hch): why is there no hlist_move_head? */
1688        INIT_HLIST_NODE(&ri->hlist);
1689        kretprobe_table_lock(hash, &flags);
1690        hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1691        kretprobe_table_unlock(hash, &flags);
1692    } else {
1693        rp->nmissed++;
1694        raw_spin_unlock_irqrestore(&rp->lock, flags);
1695    }
1696    return 0;
1697}
1698
1699int __kprobes register_kretprobe(struct kretprobe *rp)
1700{
1701    int ret = 0;
1702    struct kretprobe_instance *inst;
1703    int i;
1704    void *addr;
1705
1706    if (kretprobe_blacklist_size) {
1707        addr = kprobe_addr(&rp->kp);
1708        if (IS_ERR(addr))
1709            return PTR_ERR(addr);
1710
1711        for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1712            if (kretprobe_blacklist[i].addr == addr)
1713                return -EINVAL;
1714        }
1715    }
1716
1717    rp->kp.pre_handler = pre_handler_kretprobe;
1718    rp->kp.post_handler = NULL;
1719    rp->kp.fault_handler = NULL;
1720    rp->kp.break_handler = NULL;
1721
1722    /* Pre-allocate memory for max kretprobe instances */
1723    if (rp->maxactive <= 0) {
1724#ifdef CONFIG_PREEMPT
1725        rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1726#else
1727        rp->maxactive = num_possible_cpus();
1728#endif
1729    }
1730    raw_spin_lock_init(&rp->lock);
1731    INIT_HLIST_HEAD(&rp->free_instances);
1732    for (i = 0; i < rp->maxactive; i++) {
1733        inst = kmalloc(sizeof(struct kretprobe_instance) +
1734                   rp->data_size, GFP_KERNEL);
1735        if (inst == NULL) {
1736            free_rp_inst(rp);
1737            return -ENOMEM;
1738        }
1739        INIT_HLIST_NODE(&inst->hlist);
1740        hlist_add_head(&inst->hlist, &rp->free_instances);
1741    }
1742
1743    rp->nmissed = 0;
1744    /* Establish function entry probe point */
1745    ret = register_kprobe(&rp->kp);
1746    if (ret != 0)
1747        free_rp_inst(rp);
1748    return ret;
1749}
1750EXPORT_SYMBOL_GPL(register_kretprobe);
1751
1752int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1753{
1754    int ret = 0, i;
1755
1756    if (num <= 0)
1757        return -EINVAL;
1758    for (i = 0; i < num; i++) {
1759        ret = register_kretprobe(rps[i]);
1760        if (ret < 0) {
1761            if (i > 0)
1762                unregister_kretprobes(rps, i);
1763            break;
1764        }
1765    }
1766    return ret;
1767}
1768EXPORT_SYMBOL_GPL(register_kretprobes);
1769
1770void __kprobes unregister_kretprobe(struct kretprobe *rp)
1771{
1772    unregister_kretprobes(&rp, 1);
1773}
1774EXPORT_SYMBOL_GPL(unregister_kretprobe);
1775
1776void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1777{
1778    int i;
1779
1780    if (num <= 0)
1781        return;
1782    mutex_lock(&kprobe_mutex);
1783    for (i = 0; i < num; i++)
1784        if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1785            rps[i]->kp.addr = NULL;
1786    mutex_unlock(&kprobe_mutex);
1787
1788    synchronize_sched();
1789    for (i = 0; i < num; i++) {
1790        if (rps[i]->kp.addr) {
1791            __unregister_kprobe_bottom(&rps[i]->kp);
1792            cleanup_rp_inst(rps[i]);
1793        }
1794    }
1795}
1796EXPORT_SYMBOL_GPL(unregister_kretprobes);
1797
1798#else /* CONFIG_KRETPROBES */
1799int __kprobes register_kretprobe(struct kretprobe *rp)
1800{
1801    return -ENOSYS;
1802}
1803EXPORT_SYMBOL_GPL(register_kretprobe);
1804
1805int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1806{
1807    return -ENOSYS;
1808}
1809EXPORT_SYMBOL_GPL(register_kretprobes);
1810
1811void __kprobes unregister_kretprobe(struct kretprobe *rp)
1812{
1813}
1814EXPORT_SYMBOL_GPL(unregister_kretprobe);
1815
1816void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1817{
1818}
1819EXPORT_SYMBOL_GPL(unregister_kretprobes);
1820
1821static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1822                       struct pt_regs *regs)
1823{
1824    return 0;
1825}
1826
1827#endif /* CONFIG_KRETPROBES */
1828
1829/* Set the kprobe gone and remove its instruction buffer. */
1830static void __kprobes kill_kprobe(struct kprobe *p)
1831{
1832    struct kprobe *kp;
1833
1834    p->flags |= KPROBE_FLAG_GONE;
1835    if (kprobe_aggrprobe(p)) {
1836        /*
1837         * If this is an aggr_kprobe, we have to list all the
1838         * chained probes and mark them GONE.
1839         */
1840        list_for_each_entry_rcu(kp, &p->list, list)
1841            kp->flags |= KPROBE_FLAG_GONE;
1842        p->post_handler = NULL;
1843        p->break_handler = NULL;
1844        kill_optimized_kprobe(p);
1845    }
1846    /*
1847     * Here, we can remove insn_slot safely, because no thread calls
1848     * the original probed function (which will be freed soon) any more.
1849     */
1850    arch_remove_kprobe(p);
1851}
1852
1853/* Disable one kprobe */
1854int __kprobes disable_kprobe(struct kprobe *kp)
1855{
1856    int ret = 0;
1857
1858    mutex_lock(&kprobe_mutex);
1859
1860    /* Disable this kprobe */
1861    if (__disable_kprobe(kp) == NULL)
1862        ret = -EINVAL;
1863
1864    mutex_unlock(&kprobe_mutex);
1865    return ret;
1866}
1867EXPORT_SYMBOL_GPL(disable_kprobe);
1868
1869/* Enable one kprobe */
1870int __kprobes enable_kprobe(struct kprobe *kp)
1871{
1872    int ret = 0;
1873    struct kprobe *p;
1874
1875    mutex_lock(&kprobe_mutex);
1876
1877    /* Check whether specified probe is valid. */
1878    p = __get_valid_kprobe(kp);
1879    if (unlikely(p == NULL)) {
1880        ret = -EINVAL;
1881        goto out;
1882    }
1883
1884    if (kprobe_gone(kp)) {
1885        /* This kprobe has gone, we couldn't enable it. */
1886        ret = -EINVAL;
1887        goto out;
1888    }
1889
1890    if (p != kp)
1891        kp->flags &= ~KPROBE_FLAG_DISABLED;
1892
1893    if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1894        p->flags &= ~KPROBE_FLAG_DISABLED;
1895        arm_kprobe(p);
1896    }
1897out:
1898    mutex_unlock(&kprobe_mutex);
1899    return ret;
1900}
1901EXPORT_SYMBOL_GPL(enable_kprobe);
1902
1903void __kprobes dump_kprobe(struct kprobe *kp)
1904{
1905    printk(KERN_WARNING "Dumping kprobe:\n");
1906    printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1907           kp->symbol_name, kp->addr, kp->offset);
1908}
1909
1910/* Module notifier call back, checking kprobes on the module */
1911static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1912                         unsigned long val, void *data)
1913{
1914    struct module *mod = data;
1915    struct hlist_head *head;
1916    struct hlist_node *node;
1917    struct kprobe *p;
1918    unsigned int i;
1919    int checkcore = (val == MODULE_STATE_GOING);
1920
1921    if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1922        return NOTIFY_DONE;
1923
1924    /*
1925     * When MODULE_STATE_GOING was notified, both of module .text and
1926     * .init.text sections would be freed. When MODULE_STATE_LIVE was
1927     * notified, only .init.text section would be freed. We need to
1928     * disable kprobes which have been inserted in the sections.
1929     */
1930    mutex_lock(&kprobe_mutex);
1931    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1932        head = &kprobe_table[i];
1933        hlist_for_each_entry_rcu(p, node, head, hlist)
1934            if (within_module_init((unsigned long)p->addr, mod) ||
1935                (checkcore &&
1936                 within_module_core((unsigned long)p->addr, mod))) {
1937                /*
1938                 * The vaddr this probe is installed will soon
1939                 * be vfreed buy not synced to disk. Hence,
1940                 * disarming the breakpoint isn't needed.
1941                 */
1942                kill_kprobe(p);
1943            }
1944    }
1945    mutex_unlock(&kprobe_mutex);
1946    return NOTIFY_DONE;
1947}
1948
1949static struct notifier_block kprobe_module_nb = {
1950    .notifier_call = kprobes_module_callback,
1951    .priority = 0
1952};
1953
1954static int __init init_kprobes(void)
1955{
1956    int i, err = 0;
1957    unsigned long offset = 0, size = 0;
1958    char *modname, namebuf[128];
1959    const char *symbol_name;
1960    void *addr;
1961    struct kprobe_blackpoint *kb;
1962
1963    /* FIXME allocate the probe table, currently defined statically */
1964    /* initialize all list heads */
1965    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1966        INIT_HLIST_HEAD(&kprobe_table[i]);
1967        INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1968        raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
1969    }
1970
1971    /*
1972     * Lookup and populate the kprobe_blacklist.
1973     *
1974     * Unlike the kretprobe blacklist, we'll need to determine
1975     * the range of addresses that belong to the said functions,
1976     * since a kprobe need not necessarily be at the beginning
1977     * of a function.
1978     */
1979    for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1980        kprobe_lookup_name(kb->name, addr);
1981        if (!addr)
1982            continue;
1983
1984        kb->start_addr = (unsigned long)addr;
1985        symbol_name = kallsyms_lookup(kb->start_addr,
1986                &size, &offset, &modname, namebuf);
1987        if (!symbol_name)
1988            kb->range = 0;
1989        else
1990            kb->range = size;
1991    }
1992
1993    if (kretprobe_blacklist_size) {
1994        /* lookup the function address from its name */
1995        for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1996            kprobe_lookup_name(kretprobe_blacklist[i].name,
1997                       kretprobe_blacklist[i].addr);
1998            if (!kretprobe_blacklist[i].addr)
1999                printk("kretprobe: lookup failed: %s\n",
2000                       kretprobe_blacklist[i].name);
2001        }
2002    }
2003
2004#if defined(CONFIG_OPTPROBES)
2005#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2006    /* Init kprobe_optinsn_slots */
2007    kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2008#endif
2009    /* By default, kprobes can be optimized */
2010    kprobes_allow_optimization = true;
2011#endif
2012
2013    /* By default, kprobes are armed */
2014    kprobes_all_disarmed = false;
2015
2016    err = arch_init_kprobes();
2017    if (!err)
2018        err = register_die_notifier(&kprobe_exceptions_nb);
2019    if (!err)
2020        err = register_module_notifier(&kprobe_module_nb);
2021
2022    kprobes_initialized = (err == 0);
2023
2024    if (!err)
2025        init_test_probes();
2026    return err;
2027}
2028
2029#ifdef CONFIG_DEBUG_FS
2030static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
2031        const char *sym, int offset, char *modname, struct kprobe *pp)
2032{
2033    char *kprobe_type;
2034
2035    if (p->pre_handler == pre_handler_kretprobe)
2036        kprobe_type = "r";
2037    else if (p->pre_handler == setjmp_pre_handler)
2038        kprobe_type = "j";
2039    else
2040        kprobe_type = "k";
2041
2042    if (sym)
2043        seq_printf(pi, "%p %s %s+0x%x %s ",
2044            p->addr, kprobe_type, sym, offset,
2045            (modname ? modname : " "));
2046    else
2047        seq_printf(pi, "%p %s %p ",
2048            p->addr, kprobe_type, p->addr);
2049
2050    if (!pp)
2051        pp = p;
2052    seq_printf(pi, "%s%s%s\n",
2053        (kprobe_gone(p) ? "[GONE]" : ""),
2054        ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2055        (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
2056}
2057
2058static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2059{
2060    return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2061}
2062
2063static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2064{
2065    (*pos)++;
2066    if (*pos >= KPROBE_TABLE_SIZE)
2067        return NULL;
2068    return pos;
2069}
2070
2071static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
2072{
2073    /* Nothing to do */
2074}
2075
2076static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
2077{
2078    struct hlist_head *head;
2079    struct hlist_node *node;
2080    struct kprobe *p, *kp;
2081    const char *sym = NULL;
2082    unsigned int i = *(loff_t *) v;
2083    unsigned long offset = 0;
2084    char *modname, namebuf[128];
2085
2086    head = &kprobe_table[i];
2087    preempt_disable();
2088    hlist_for_each_entry_rcu(p, node, head, hlist) {
2089        sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2090                    &offset, &modname, namebuf);
2091        if (kprobe_aggrprobe(p)) {
2092            list_for_each_entry_rcu(kp, &p->list, list)
2093                report_probe(pi, kp, sym, offset, modname, p);
2094        } else
2095            report_probe(pi, p, sym, offset, modname, NULL);
2096    }
2097    preempt_enable();
2098    return 0;
2099}
2100
2101static const struct seq_operations kprobes_seq_ops = {
2102    .start = kprobe_seq_start,
2103    .next = kprobe_seq_next,
2104    .stop = kprobe_seq_stop,
2105    .show = show_kprobe_addr
2106};
2107
2108static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
2109{
2110    return seq_open(filp, &kprobes_seq_ops);
2111}
2112
2113static const struct file_operations debugfs_kprobes_operations = {
2114    .open = kprobes_open,
2115    .read = seq_read,
2116    .llseek = seq_lseek,
2117    .release = seq_release,
2118};
2119
2120static void __kprobes arm_all_kprobes(void)
2121{
2122    struct hlist_head *head;
2123    struct hlist_node *node;
2124    struct kprobe *p;
2125    unsigned int i;
2126
2127    mutex_lock(&kprobe_mutex);
2128
2129    /* If kprobes are armed, just return */
2130    if (!kprobes_all_disarmed)
2131        goto already_enabled;
2132
2133    /* Arming kprobes doesn't optimize kprobe itself */
2134    mutex_lock(&text_mutex);
2135    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2136        head = &kprobe_table[i];
2137        hlist_for_each_entry_rcu(p, node, head, hlist)
2138            if (!kprobe_disabled(p))
2139                __arm_kprobe(p);
2140    }
2141    mutex_unlock(&text_mutex);
2142
2143    kprobes_all_disarmed = false;
2144    printk(KERN_INFO "Kprobes globally enabled\n");
2145
2146already_enabled:
2147    mutex_unlock(&kprobe_mutex);
2148    return;
2149}
2150
2151static void __kprobes disarm_all_kprobes(void)
2152{
2153    struct hlist_head *head;
2154    struct hlist_node *node;
2155    struct kprobe *p;
2156    unsigned int i;
2157
2158    mutex_lock(&kprobe_mutex);
2159
2160    /* If kprobes are already disarmed, just return */
2161    if (kprobes_all_disarmed) {
2162        mutex_unlock(&kprobe_mutex);
2163        return;
2164    }
2165
2166    kprobes_all_disarmed = true;
2167    printk(KERN_INFO "Kprobes globally disabled\n");
2168
2169    mutex_lock(&text_mutex);
2170    for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2171        head = &kprobe_table[i];
2172        hlist_for_each_entry_rcu(p, node, head, hlist) {
2173            if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
2174                __disarm_kprobe(p, false);
2175        }
2176    }
2177    mutex_unlock(&text_mutex);
2178    mutex_unlock(&kprobe_mutex);
2179
2180    /* Wait for disarming all kprobes by optimizer */
2181    wait_for_kprobe_optimizer();
2182}
2183
2184/*
2185 * XXX: The debugfs bool file interface doesn't allow for callbacks
2186 * when the bool state is switched. We can reuse that facility when
2187 * available
2188 */
2189static ssize_t read_enabled_file_bool(struct file *file,
2190           char __user *user_buf, size_t count, loff_t *ppos)
2191{
2192    char buf[3];
2193
2194    if (!kprobes_all_disarmed)
2195        buf[0] = '1';
2196    else
2197        buf[0] = '0';
2198    buf[1] = '\n';
2199    buf[2] = 0x00;
2200    return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2201}
2202
2203static ssize_t write_enabled_file_bool(struct file *file,
2204           const char __user *user_buf, size_t count, loff_t *ppos)
2205{
2206    char buf[32];
2207    size_t buf_size;
2208
2209    buf_size = min(count, (sizeof(buf)-1));
2210    if (copy_from_user(buf, user_buf, buf_size))
2211        return -EFAULT;
2212
2213    switch (buf[0]) {
2214    case 'y':
2215    case 'Y':
2216    case '1':
2217        arm_all_kprobes();
2218        break;
2219    case 'n':
2220    case 'N':
2221    case '0':
2222        disarm_all_kprobes();
2223        break;
2224    }
2225
2226    return count;
2227}
2228
2229static const struct file_operations fops_kp = {
2230    .read = read_enabled_file_bool,
2231    .write = write_enabled_file_bool,
2232    .llseek = default_llseek,
2233};
2234
2235static int __kprobes debugfs_kprobe_init(void)
2236{
2237    struct dentry *dir, *file;
2238    unsigned int value = 1;
2239
2240    dir = debugfs_create_dir("kprobes", NULL);
2241    if (!dir)
2242        return -ENOMEM;
2243
2244    file = debugfs_create_file("list", 0444, dir, NULL,
2245                &debugfs_kprobes_operations);
2246    if (!file) {
2247        debugfs_remove(dir);
2248        return -ENOMEM;
2249    }
2250
2251    file = debugfs_create_file("enabled", 0600, dir,
2252                    &value, &fops_kp);
2253    if (!file) {
2254        debugfs_remove(dir);
2255        return -ENOMEM;
2256    }
2257
2258    return 0;
2259}
2260
2261late_initcall(debugfs_kprobe_init);
2262#endif /* CONFIG_DEBUG_FS */
2263
2264module_init(init_kprobes);
2265
2266/* defined in arch/.../kernel/kprobes.c */
2267EXPORT_SYMBOL_GPL(jprobe_return);
2268

Archive Download this file



interactive