Root/kernel/trace/trace.c

1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/ring_buffer.h>
15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/notifier.h>
21#include <linux/irqflags.h>
22#include <linux/debugfs.h>
23#include <linux/pagemap.h>
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
31#include <linux/splice.h>
32#include <linux/kdebug.h>
33#include <linux/string.h>
34#include <linux/rwsem.h>
35#include <linux/slab.h>
36#include <linux/ctype.h>
37#include <linux/init.h>
38#include <linux/poll.h>
39#include <linux/fs.h>
40
41#include "trace.h"
42#include "trace_output.h"
43
44/*
45 * On boot up, the ring buffer is set to the minimum size, so that
46 * we do not waste memory on systems that are not using tracing.
47 */
48int ring_buffer_expanded;
49
50/*
51 * We need to change this state when a selftest is running.
52 * A selftest will lurk into the ring-buffer to count the
53 * entries inserted during the selftest although some concurrent
54 * insertions into the ring-buffer such as trace_printk could occurred
55 * at the same time, giving false positive or negative results.
56 */
57static bool __read_mostly tracing_selftest_running;
58
59/*
60 * If a tracer is running, we do not want to run SELFTEST.
61 */
62bool __read_mostly tracing_selftest_disabled;
63
64/* For tracers that don't implement custom flags */
65static struct tracer_opt dummy_tracer_opt[] = {
66    { }
67};
68
69static struct tracer_flags dummy_tracer_flags = {
70    .val = 0,
71    .opts = dummy_tracer_opt
72};
73
74static int dummy_set_flag(u32 old_flags, u32 bit, int set)
75{
76    return 0;
77}
78
79/*
80 * Kill all tracing for good (never come back).
81 * It is initialized to 1 but will turn to zero if the initialization
82 * of the tracer is successful. But that is the only place that sets
83 * this back to zero.
84 */
85static int tracing_disabled = 1;
86
87DEFINE_PER_CPU(int, ftrace_cpu_disabled);
88
89static inline void ftrace_disable_cpu(void)
90{
91    preempt_disable();
92    __this_cpu_inc(ftrace_cpu_disabled);
93}
94
95static inline void ftrace_enable_cpu(void)
96{
97    __this_cpu_dec(ftrace_cpu_disabled);
98    preempt_enable();
99}
100
101cpumask_var_t __read_mostly tracing_buffer_mask;
102
103/*
104 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105 *
106 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
107 * is set, then ftrace_dump is called. This will output the contents
108 * of the ftrace buffers to the console. This is very useful for
109 * capturing traces that lead to crashes and outputing it to a
110 * serial console.
111 *
112 * It is default off, but you can enable it with either specifying
113 * "ftrace_dump_on_oops" in the kernel command line, or setting
114 * /proc/sys/kernel/ftrace_dump_on_oops
115 * Set 1 if you want to dump buffers of all CPUs
116 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 */
118
119enum ftrace_dump_mode ftrace_dump_on_oops;
120
121static int tracing_set_tracer(const char *buf);
122
123#define MAX_TRACER_SIZE 100
124static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125static char *default_bootup_tracer;
126
127static int __init set_cmdline_ftrace(char *str)
128{
129    strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
130    default_bootup_tracer = bootup_tracer_buf;
131    /* We are using ftrace early, expand it */
132    ring_buffer_expanded = 1;
133    return 1;
134}
135__setup("ftrace=", set_cmdline_ftrace);
136
137static int __init set_ftrace_dump_on_oops(char *str)
138{
139    if (*str++ != '=' || !*str) {
140        ftrace_dump_on_oops = DUMP_ALL;
141        return 1;
142    }
143
144    if (!strcmp("orig_cpu", str)) {
145        ftrace_dump_on_oops = DUMP_ORIG;
146                return 1;
147        }
148
149        return 0;
150}
151__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
152
153unsigned long long ns2usecs(cycle_t nsec)
154{
155    nsec += 500;
156    do_div(nsec, 1000);
157    return nsec;
158}
159
160/*
161 * The global_trace is the descriptor that holds the tracing
162 * buffers for the live tracing. For each CPU, it contains
163 * a link list of pages that will store trace entries. The
164 * page descriptor of the pages in the memory is used to hold
165 * the link list by linking the lru item in the page descriptor
166 * to each of the pages in the buffer per CPU.
167 *
168 * For each active CPU there is a data field that holds the
169 * pages for the buffer for that CPU. Each CPU has the same number
170 * of pages allocated for its buffer.
171 */
172static struct trace_array global_trace;
173
174static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
175
176int filter_current_check_discard(struct ring_buffer *buffer,
177                 struct ftrace_event_call *call, void *rec,
178                 struct ring_buffer_event *event)
179{
180    return filter_check_discard(call, rec, buffer, event);
181}
182EXPORT_SYMBOL_GPL(filter_current_check_discard);
183
184cycle_t ftrace_now(int cpu)
185{
186    u64 ts;
187
188    /* Early boot up does not have a buffer yet */
189    if (!global_trace.buffer)
190        return trace_clock_local();
191
192    ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
193    ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
194
195    return ts;
196}
197
198/*
199 * The max_tr is used to snapshot the global_trace when a maximum
200 * latency is reached. Some tracers will use this to store a maximum
201 * trace while it continues examining live traces.
202 *
203 * The buffers for the max_tr are set up the same as the global_trace.
204 * When a snapshot is taken, the link list of the max_tr is swapped
205 * with the link list of the global_trace and the buffers are reset for
206 * the global_trace so the tracing can continue.
207 */
208static struct trace_array max_tr;
209
210static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
211
212/* tracer_enabled is used to toggle activation of a tracer */
213static int tracer_enabled = 1;
214
215/**
216 * tracing_is_enabled - return tracer_enabled status
217 *
218 * This function is used by other tracers to know the status
219 * of the tracer_enabled flag. Tracers may use this function
220 * to know if it should enable their features when starting
221 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
222 */
223int tracing_is_enabled(void)
224{
225    return tracer_enabled;
226}
227
228/*
229 * trace_buf_size is the size in bytes that is allocated
230 * for a buffer. Note, the number of bytes is always rounded
231 * to page size.
232 *
233 * This number is purposely set to a low number of 16384.
234 * If the dump on oops happens, it will be much appreciated
235 * to not have to wait for all that output. Anyway this can be
236 * boot time and run time configurable.
237 */
238#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
239
240static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
241
242/* trace_types holds a link list of available tracers. */
243static struct tracer *trace_types __read_mostly;
244
245/* current_trace points to the tracer that is currently active */
246static struct tracer *current_trace __read_mostly;
247
248/*
249 * trace_types_lock is used to protect the trace_types list.
250 */
251static DEFINE_MUTEX(trace_types_lock);
252
253/*
254 * serialize the access of the ring buffer
255 *
256 * ring buffer serializes readers, but it is low level protection.
257 * The validity of the events (which returns by ring_buffer_peek() ..etc)
258 * are not protected by ring buffer.
259 *
260 * The content of events may become garbage if we allow other process consumes
261 * these events concurrently:
262 * A) the page of the consumed events may become a normal page
263 * (not reader page) in ring buffer, and this page will be rewrited
264 * by events producer.
265 * B) The page of the consumed events may become a page for splice_read,
266 * and this page will be returned to system.
267 *
268 * These primitives allow multi process access to different cpu ring buffer
269 * concurrently.
270 *
271 * These primitives don't distinguish read-only and read-consume access.
272 * Multi read-only access are also serialized.
273 */
274
275#ifdef CONFIG_SMP
276static DECLARE_RWSEM(all_cpu_access_lock);
277static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
278
279static inline void trace_access_lock(int cpu)
280{
281    if (cpu == TRACE_PIPE_ALL_CPU) {
282        /* gain it for accessing the whole ring buffer. */
283        down_write(&all_cpu_access_lock);
284    } else {
285        /* gain it for accessing a cpu ring buffer. */
286
287        /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
288        down_read(&all_cpu_access_lock);
289
290        /* Secondly block other access to this @cpu ring buffer. */
291        mutex_lock(&per_cpu(cpu_access_lock, cpu));
292    }
293}
294
295static inline void trace_access_unlock(int cpu)
296{
297    if (cpu == TRACE_PIPE_ALL_CPU) {
298        up_write(&all_cpu_access_lock);
299    } else {
300        mutex_unlock(&per_cpu(cpu_access_lock, cpu));
301        up_read(&all_cpu_access_lock);
302    }
303}
304
305static inline void trace_access_lock_init(void)
306{
307    int cpu;
308
309    for_each_possible_cpu(cpu)
310        mutex_init(&per_cpu(cpu_access_lock, cpu));
311}
312
313#else
314
315static DEFINE_MUTEX(access_lock);
316
317static inline void trace_access_lock(int cpu)
318{
319    (void)cpu;
320    mutex_lock(&access_lock);
321}
322
323static inline void trace_access_unlock(int cpu)
324{
325    (void)cpu;
326    mutex_unlock(&access_lock);
327}
328
329static inline void trace_access_lock_init(void)
330{
331}
332
333#endif
334
335/* trace_wait is a waitqueue for tasks blocked on trace_poll */
336static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
337
338/* trace_flags holds trace_options default values */
339unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
340    TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
341    TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE;
342
343static int trace_stop_count;
344static DEFINE_SPINLOCK(tracing_start_lock);
345
346/**
347 * trace_wake_up - wake up tasks waiting for trace input
348 *
349 * Simply wakes up any task that is blocked on the trace_wait
350 * queue. These is used with trace_poll for tasks polling the trace.
351 */
352void trace_wake_up(void)
353{
354    int cpu;
355
356    if (trace_flags & TRACE_ITER_BLOCK)
357        return;
358    /*
359     * The runqueue_is_locked() can fail, but this is the best we
360     * have for now:
361     */
362    cpu = get_cpu();
363    if (!runqueue_is_locked(cpu))
364        wake_up(&trace_wait);
365    put_cpu();
366}
367
368static int __init set_buf_size(char *str)
369{
370    unsigned long buf_size;
371
372    if (!str)
373        return 0;
374    buf_size = memparse(str, &str);
375    /* nr_entries can not be zero */
376    if (buf_size == 0)
377        return 0;
378    trace_buf_size = buf_size;
379    return 1;
380}
381__setup("trace_buf_size=", set_buf_size);
382
383static int __init set_tracing_thresh(char *str)
384{
385    unsigned long threshhold;
386    int ret;
387
388    if (!str)
389        return 0;
390    ret = strict_strtoul(str, 0, &threshhold);
391    if (ret < 0)
392        return 0;
393    tracing_thresh = threshhold * 1000;
394    return 1;
395}
396__setup("tracing_thresh=", set_tracing_thresh);
397
398unsigned long nsecs_to_usecs(unsigned long nsecs)
399{
400    return nsecs / 1000;
401}
402
403/* These must match the bit postions in trace_iterator_flags */
404static const char *trace_options[] = {
405    "print-parent",
406    "sym-offset",
407    "sym-addr",
408    "verbose",
409    "raw",
410    "hex",
411    "bin",
412    "block",
413    "stacktrace",
414    "trace_printk",
415    "ftrace_preempt",
416    "branch",
417    "annotate",
418    "userstacktrace",
419    "sym-userobj",
420    "printk-msg-only",
421    "context-info",
422    "latency-format",
423    "sleep-time",
424    "graph-time",
425    "record-cmd",
426    "overwrite",
427    NULL
428};
429
430static struct {
431    u64 (*func)(void);
432    const char *name;
433} trace_clocks[] = {
434    { trace_clock_local, "local" },
435    { trace_clock_global, "global" },
436};
437
438int trace_clock_id;
439
440/*
441 * trace_parser_get_init - gets the buffer for trace parser
442 */
443int trace_parser_get_init(struct trace_parser *parser, int size)
444{
445    memset(parser, 0, sizeof(*parser));
446
447    parser->buffer = kmalloc(size, GFP_KERNEL);
448    if (!parser->buffer)
449        return 1;
450
451    parser->size = size;
452    return 0;
453}
454
455/*
456 * trace_parser_put - frees the buffer for trace parser
457 */
458void trace_parser_put(struct trace_parser *parser)
459{
460    kfree(parser->buffer);
461}
462
463/*
464 * trace_get_user - reads the user input string separated by space
465 * (matched by isspace(ch))
466 *
467 * For each string found the 'struct trace_parser' is updated,
468 * and the function returns.
469 *
470 * Returns number of bytes read.
471 *
472 * See kernel/trace/trace.h for 'struct trace_parser' details.
473 */
474int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
475    size_t cnt, loff_t *ppos)
476{
477    char ch;
478    size_t read = 0;
479    ssize_t ret;
480
481    if (!*ppos)
482        trace_parser_clear(parser);
483
484    ret = get_user(ch, ubuf++);
485    if (ret)
486        goto out;
487
488    read++;
489    cnt--;
490
491    /*
492     * The parser is not finished with the last write,
493     * continue reading the user input without skipping spaces.
494     */
495    if (!parser->cont) {
496        /* skip white space */
497        while (cnt && isspace(ch)) {
498            ret = get_user(ch, ubuf++);
499            if (ret)
500                goto out;
501            read++;
502            cnt--;
503        }
504
505        /* only spaces were written */
506        if (isspace(ch)) {
507            *ppos += read;
508            ret = read;
509            goto out;
510        }
511
512        parser->idx = 0;
513    }
514
515    /* read the non-space input */
516    while (cnt && !isspace(ch)) {
517        if (parser->idx < parser->size - 1)
518            parser->buffer[parser->idx++] = ch;
519        else {
520            ret = -EINVAL;
521            goto out;
522        }
523        ret = get_user(ch, ubuf++);
524        if (ret)
525            goto out;
526        read++;
527        cnt--;
528    }
529
530    /* We either got finished input or we have to wait for another call. */
531    if (isspace(ch)) {
532        parser->buffer[parser->idx] = 0;
533        parser->cont = false;
534    } else {
535        parser->cont = true;
536        parser->buffer[parser->idx++] = ch;
537    }
538
539    *ppos += read;
540    ret = read;
541
542out:
543    return ret;
544}
545
546ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
547{
548    int len;
549    int ret;
550
551    if (!cnt)
552        return 0;
553
554    if (s->len <= s->readpos)
555        return -EBUSY;
556
557    len = s->len - s->readpos;
558    if (cnt > len)
559        cnt = len;
560    ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
561    if (ret == cnt)
562        return -EFAULT;
563
564    cnt -= ret;
565
566    s->readpos += cnt;
567    return cnt;
568}
569
570static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
571{
572    int len;
573    void *ret;
574
575    if (s->len <= s->readpos)
576        return -EBUSY;
577
578    len = s->len - s->readpos;
579    if (cnt > len)
580        cnt = len;
581    ret = memcpy(buf, s->buffer + s->readpos, cnt);
582    if (!ret)
583        return -EFAULT;
584
585    s->readpos += cnt;
586    return cnt;
587}
588
589/*
590 * ftrace_max_lock is used to protect the swapping of buffers
591 * when taking a max snapshot. The buffers themselves are
592 * protected by per_cpu spinlocks. But the action of the swap
593 * needs its own lock.
594 *
595 * This is defined as a arch_spinlock_t in order to help
596 * with performance when lockdep debugging is enabled.
597 *
598 * It is also used in other places outside the update_max_tr
599 * so it needs to be defined outside of the
600 * CONFIG_TRACER_MAX_TRACE.
601 */
602static arch_spinlock_t ftrace_max_lock =
603    (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
604
605unsigned long __read_mostly tracing_thresh;
606
607#ifdef CONFIG_TRACER_MAX_TRACE
608unsigned long __read_mostly tracing_max_latency;
609
610/*
611 * Copy the new maximum trace into the separate maximum-trace
612 * structure. (this way the maximum trace is permanently saved,
613 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
614 */
615static void
616__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
617{
618    struct trace_array_cpu *data = tr->data[cpu];
619    struct trace_array_cpu *max_data;
620
621    max_tr.cpu = cpu;
622    max_tr.time_start = data->preempt_timestamp;
623
624    max_data = max_tr.data[cpu];
625    max_data->saved_latency = tracing_max_latency;
626    max_data->critical_start = data->critical_start;
627    max_data->critical_end = data->critical_end;
628
629    memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
630    max_data->pid = tsk->pid;
631    max_data->uid = task_uid(tsk);
632    max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
633    max_data->policy = tsk->policy;
634    max_data->rt_priority = tsk->rt_priority;
635
636    /* record this tasks comm */
637    tracing_record_cmdline(tsk);
638}
639
640/**
641 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
642 * @tr: tracer
643 * @tsk: the task with the latency
644 * @cpu: The cpu that initiated the trace.
645 *
646 * Flip the buffers between the @tr and the max_tr and record information
647 * about which task was the cause of this latency.
648 */
649void
650update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
651{
652    struct ring_buffer *buf = tr->buffer;
653
654    if (trace_stop_count)
655        return;
656
657    WARN_ON_ONCE(!irqs_disabled());
658    if (!current_trace->use_max_tr) {
659        WARN_ON_ONCE(1);
660        return;
661    }
662    arch_spin_lock(&ftrace_max_lock);
663
664    tr->buffer = max_tr.buffer;
665    max_tr.buffer = buf;
666
667    __update_max_tr(tr, tsk, cpu);
668    arch_spin_unlock(&ftrace_max_lock);
669}
670
671/**
672 * update_max_tr_single - only copy one trace over, and reset the rest
673 * @tr - tracer
674 * @tsk - task with the latency
675 * @cpu - the cpu of the buffer to copy.
676 *
677 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
678 */
679void
680update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
681{
682    int ret;
683
684    if (trace_stop_count)
685        return;
686
687    WARN_ON_ONCE(!irqs_disabled());
688    if (!current_trace->use_max_tr) {
689        WARN_ON_ONCE(1);
690        return;
691    }
692
693    arch_spin_lock(&ftrace_max_lock);
694
695    ftrace_disable_cpu();
696
697    ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
698
699    if (ret == -EBUSY) {
700        /*
701         * We failed to swap the buffer due to a commit taking
702         * place on this CPU. We fail to record, but we reset
703         * the max trace buffer (no one writes directly to it)
704         * and flag that it failed.
705         */
706        trace_array_printk(&max_tr, _THIS_IP_,
707            "Failed to swap buffers due to commit in progress\n");
708    }
709
710    ftrace_enable_cpu();
711
712    WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
713
714    __update_max_tr(tr, tsk, cpu);
715    arch_spin_unlock(&ftrace_max_lock);
716}
717#endif /* CONFIG_TRACER_MAX_TRACE */
718
719/**
720 * register_tracer - register a tracer with the ftrace system.
721 * @type - the plugin for the tracer
722 *
723 * Register a new plugin tracer.
724 */
725int register_tracer(struct tracer *type)
726__releases(kernel_lock)
727__acquires(kernel_lock)
728{
729    struct tracer *t;
730    int ret = 0;
731
732    if (!type->name) {
733        pr_info("Tracer must have a name\n");
734        return -1;
735    }
736
737    if (strlen(type->name) >= MAX_TRACER_SIZE) {
738        pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
739        return -1;
740    }
741
742    mutex_lock(&trace_types_lock);
743
744    tracing_selftest_running = true;
745
746    for (t = trace_types; t; t = t->next) {
747        if (strcmp(type->name, t->name) == 0) {
748            /* already found */
749            pr_info("Tracer %s already registered\n",
750                type->name);
751            ret = -1;
752            goto out;
753        }
754    }
755
756    if (!type->set_flag)
757        type->set_flag = &dummy_set_flag;
758    if (!type->flags)
759        type->flags = &dummy_tracer_flags;
760    else
761        if (!type->flags->opts)
762            type->flags->opts = dummy_tracer_opt;
763    if (!type->wait_pipe)
764        type->wait_pipe = default_wait_pipe;
765
766
767#ifdef CONFIG_FTRACE_STARTUP_TEST
768    if (type->selftest && !tracing_selftest_disabled) {
769        struct tracer *saved_tracer = current_trace;
770        struct trace_array *tr = &global_trace;
771
772        /*
773         * Run a selftest on this tracer.
774         * Here we reset the trace buffer, and set the current
775         * tracer to be this tracer. The tracer can then run some
776         * internal tracing to verify that everything is in order.
777         * If we fail, we do not register this tracer.
778         */
779        tracing_reset_online_cpus(tr);
780
781        current_trace = type;
782
783        /* If we expanded the buffers, make sure the max is expanded too */
784        if (ring_buffer_expanded && type->use_max_tr)
785            ring_buffer_resize(max_tr.buffer, trace_buf_size);
786
787        /* the test is responsible for initializing and enabling */
788        pr_info("Testing tracer %s: ", type->name);
789        ret = type->selftest(type, tr);
790        /* the test is responsible for resetting too */
791        current_trace = saved_tracer;
792        if (ret) {
793            printk(KERN_CONT "FAILED!\n");
794            goto out;
795        }
796        /* Only reset on passing, to avoid touching corrupted buffers */
797        tracing_reset_online_cpus(tr);
798
799        /* Shrink the max buffer again */
800        if (ring_buffer_expanded && type->use_max_tr)
801            ring_buffer_resize(max_tr.buffer, 1);
802
803        printk(KERN_CONT "PASSED\n");
804    }
805#endif
806
807    type->next = trace_types;
808    trace_types = type;
809
810 out:
811    tracing_selftest_running = false;
812    mutex_unlock(&trace_types_lock);
813
814    if (ret || !default_bootup_tracer)
815        goto out_unlock;
816
817    if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
818        goto out_unlock;
819
820    printk(KERN_INFO "Starting tracer '%s'\n", type->name);
821    /* Do we want this tracer to start on bootup? */
822    tracing_set_tracer(type->name);
823    default_bootup_tracer = NULL;
824    /* disable other selftests, since this will break it. */
825    tracing_selftest_disabled = 1;
826#ifdef CONFIG_FTRACE_STARTUP_TEST
827    printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
828           type->name);
829#endif
830
831 out_unlock:
832    return ret;
833}
834
835void unregister_tracer(struct tracer *type)
836{
837    struct tracer **t;
838
839    mutex_lock(&trace_types_lock);
840    for (t = &trace_types; *t; t = &(*t)->next) {
841        if (*t == type)
842            goto found;
843    }
844    pr_info("Tracer %s not registered\n", type->name);
845    goto out;
846
847 found:
848    *t = (*t)->next;
849
850    if (type == current_trace && tracer_enabled) {
851        tracer_enabled = 0;
852        tracing_stop();
853        if (current_trace->stop)
854            current_trace->stop(&global_trace);
855        current_trace = &nop_trace;
856    }
857out:
858    mutex_unlock(&trace_types_lock);
859}
860
861static void __tracing_reset(struct ring_buffer *buffer, int cpu)
862{
863    ftrace_disable_cpu();
864    ring_buffer_reset_cpu(buffer, cpu);
865    ftrace_enable_cpu();
866}
867
868void tracing_reset(struct trace_array *tr, int cpu)
869{
870    struct ring_buffer *buffer = tr->buffer;
871
872    ring_buffer_record_disable(buffer);
873
874    /* Make sure all commits have finished */
875    synchronize_sched();
876    __tracing_reset(buffer, cpu);
877
878    ring_buffer_record_enable(buffer);
879}
880
881void tracing_reset_online_cpus(struct trace_array *tr)
882{
883    struct ring_buffer *buffer = tr->buffer;
884    int cpu;
885
886    ring_buffer_record_disable(buffer);
887
888    /* Make sure all commits have finished */
889    synchronize_sched();
890
891    tr->time_start = ftrace_now(tr->cpu);
892
893    for_each_online_cpu(cpu)
894        __tracing_reset(buffer, cpu);
895
896    ring_buffer_record_enable(buffer);
897}
898
899void tracing_reset_current(int cpu)
900{
901    tracing_reset(&global_trace, cpu);
902}
903
904void tracing_reset_current_online_cpus(void)
905{
906    tracing_reset_online_cpus(&global_trace);
907}
908
909#define SAVED_CMDLINES 128
910#define NO_CMDLINE_MAP UINT_MAX
911static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
912static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
913static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
914static int cmdline_idx;
915static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
916
917/* temporary disable recording */
918static atomic_t trace_record_cmdline_disabled __read_mostly;
919
920static void trace_init_cmdlines(void)
921{
922    memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
923    memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
924    cmdline_idx = 0;
925}
926
927int is_tracing_stopped(void)
928{
929    return trace_stop_count;
930}
931
932/**
933 * ftrace_off_permanent - disable all ftrace code permanently
934 *
935 * This should only be called when a serious anomally has
936 * been detected. This will turn off the function tracing,
937 * ring buffers, and other tracing utilites. It takes no
938 * locks and can be called from any context.
939 */
940void ftrace_off_permanent(void)
941{
942    tracing_disabled = 1;
943    ftrace_stop();
944    tracing_off_permanent();
945}
946
947/**
948 * tracing_start - quick start of the tracer
949 *
950 * If tracing is enabled but was stopped by tracing_stop,
951 * this will start the tracer back up.
952 */
953void tracing_start(void)
954{
955    struct ring_buffer *buffer;
956    unsigned long flags;
957
958    if (tracing_disabled)
959        return;
960
961    spin_lock_irqsave(&tracing_start_lock, flags);
962    if (--trace_stop_count) {
963        if (trace_stop_count < 0) {
964            /* Someone screwed up their debugging */
965            WARN_ON_ONCE(1);
966            trace_stop_count = 0;
967        }
968        goto out;
969    }
970
971    /* Prevent the buffers from switching */
972    arch_spin_lock(&ftrace_max_lock);
973
974    buffer = global_trace.buffer;
975    if (buffer)
976        ring_buffer_record_enable(buffer);
977
978    buffer = max_tr.buffer;
979    if (buffer)
980        ring_buffer_record_enable(buffer);
981
982    arch_spin_unlock(&ftrace_max_lock);
983
984    ftrace_start();
985 out:
986    spin_unlock_irqrestore(&tracing_start_lock, flags);
987}
988
989/**
990 * tracing_stop - quick stop of the tracer
991 *
992 * Light weight way to stop tracing. Use in conjunction with
993 * tracing_start.
994 */
995void tracing_stop(void)
996{
997    struct ring_buffer *buffer;
998    unsigned long flags;
999
1000    ftrace_stop();
1001    spin_lock_irqsave(&tracing_start_lock, flags);
1002    if (trace_stop_count++)
1003        goto out;
1004
1005    /* Prevent the buffers from switching */
1006    arch_spin_lock(&ftrace_max_lock);
1007
1008    buffer = global_trace.buffer;
1009    if (buffer)
1010        ring_buffer_record_disable(buffer);
1011
1012    buffer = max_tr.buffer;
1013    if (buffer)
1014        ring_buffer_record_disable(buffer);
1015
1016    arch_spin_unlock(&ftrace_max_lock);
1017
1018 out:
1019    spin_unlock_irqrestore(&tracing_start_lock, flags);
1020}
1021
1022void trace_stop_cmdline_recording(void);
1023
1024static void trace_save_cmdline(struct task_struct *tsk)
1025{
1026    unsigned pid, idx;
1027
1028    if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1029        return;
1030
1031    /*
1032     * It's not the end of the world if we don't get
1033     * the lock, but we also don't want to spin
1034     * nor do we want to disable interrupts,
1035     * so if we miss here, then better luck next time.
1036     */
1037    if (!arch_spin_trylock(&trace_cmdline_lock))
1038        return;
1039
1040    idx = map_pid_to_cmdline[tsk->pid];
1041    if (idx == NO_CMDLINE_MAP) {
1042        idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1043
1044        /*
1045         * Check whether the cmdline buffer at idx has a pid
1046         * mapped. We are going to overwrite that entry so we
1047         * need to clear the map_pid_to_cmdline. Otherwise we
1048         * would read the new comm for the old pid.
1049         */
1050        pid = map_cmdline_to_pid[idx];
1051        if (pid != NO_CMDLINE_MAP)
1052            map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1053
1054        map_cmdline_to_pid[idx] = tsk->pid;
1055        map_pid_to_cmdline[tsk->pid] = idx;
1056
1057        cmdline_idx = idx;
1058    }
1059
1060    memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1061
1062    arch_spin_unlock(&trace_cmdline_lock);
1063}
1064
1065void trace_find_cmdline(int pid, char comm[])
1066{
1067    unsigned map;
1068
1069    if (!pid) {
1070        strcpy(comm, "<idle>");
1071        return;
1072    }
1073
1074    if (WARN_ON_ONCE(pid < 0)) {
1075        strcpy(comm, "<XXX>");
1076        return;
1077    }
1078
1079    if (pid > PID_MAX_DEFAULT) {
1080        strcpy(comm, "<...>");
1081        return;
1082    }
1083
1084    preempt_disable();
1085    arch_spin_lock(&trace_cmdline_lock);
1086    map = map_pid_to_cmdline[pid];
1087    if (map != NO_CMDLINE_MAP)
1088        strcpy(comm, saved_cmdlines[map]);
1089    else
1090        strcpy(comm, "<...>");
1091
1092    arch_spin_unlock(&trace_cmdline_lock);
1093    preempt_enable();
1094}
1095
1096void tracing_record_cmdline(struct task_struct *tsk)
1097{
1098    if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1099        !tracing_is_on())
1100        return;
1101
1102    trace_save_cmdline(tsk);
1103}
1104
1105void
1106tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1107                 int pc)
1108{
1109    struct task_struct *tsk = current;
1110
1111    entry->preempt_count = pc & 0xff;
1112    entry->pid = (tsk) ? tsk->pid : 0;
1113    entry->padding = 0;
1114    entry->flags =
1115#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1116        (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1117#else
1118        TRACE_FLAG_IRQS_NOSUPPORT |
1119#endif
1120        ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1121        ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1122        (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1123}
1124EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1125
1126struct ring_buffer_event *
1127trace_buffer_lock_reserve(struct ring_buffer *buffer,
1128              int type,
1129              unsigned long len,
1130              unsigned long flags, int pc)
1131{
1132    struct ring_buffer_event *event;
1133
1134    event = ring_buffer_lock_reserve(buffer, len);
1135    if (event != NULL) {
1136        struct trace_entry *ent = ring_buffer_event_data(event);
1137
1138        tracing_generic_entry_update(ent, flags, pc);
1139        ent->type = type;
1140    }
1141
1142    return event;
1143}
1144
1145static inline void
1146__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1147                 struct ring_buffer_event *event,
1148                 unsigned long flags, int pc,
1149                 int wake)
1150{
1151    ring_buffer_unlock_commit(buffer, event);
1152
1153    ftrace_trace_stack(buffer, flags, 6, pc);
1154    ftrace_trace_userstack(buffer, flags, pc);
1155
1156    if (wake)
1157        trace_wake_up();
1158}
1159
1160void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1161                struct ring_buffer_event *event,
1162                unsigned long flags, int pc)
1163{
1164    __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1165}
1166
1167struct ring_buffer_event *
1168trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1169                  int type, unsigned long len,
1170                  unsigned long flags, int pc)
1171{
1172    *current_rb = global_trace.buffer;
1173    return trace_buffer_lock_reserve(*current_rb,
1174                     type, len, flags, pc);
1175}
1176EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1177
1178void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1179                    struct ring_buffer_event *event,
1180                    unsigned long flags, int pc)
1181{
1182    __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1183}
1184EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1185
1186void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1187                       struct ring_buffer_event *event,
1188                       unsigned long flags, int pc)
1189{
1190    __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1191}
1192EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1193
1194void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1195                     struct ring_buffer_event *event)
1196{
1197    ring_buffer_discard_commit(buffer, event);
1198}
1199EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1200
1201void
1202trace_function(struct trace_array *tr,
1203           unsigned long ip, unsigned long parent_ip, unsigned long flags,
1204           int pc)
1205{
1206    struct ftrace_event_call *call = &event_function;
1207    struct ring_buffer *buffer = tr->buffer;
1208    struct ring_buffer_event *event;
1209    struct ftrace_entry *entry;
1210
1211    /* If we are reading the ring buffer, don't trace */
1212    if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1213        return;
1214
1215    event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1216                      flags, pc);
1217    if (!event)
1218        return;
1219    entry = ring_buffer_event_data(event);
1220    entry->ip = ip;
1221    entry->parent_ip = parent_ip;
1222
1223    if (!filter_check_discard(call, entry, buffer, event))
1224        ring_buffer_unlock_commit(buffer, event);
1225}
1226
1227void
1228ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1229       unsigned long ip, unsigned long parent_ip, unsigned long flags,
1230       int pc)
1231{
1232    if (likely(!atomic_read(&data->disabled)))
1233        trace_function(tr, ip, parent_ip, flags, pc);
1234}
1235
1236#ifdef CONFIG_STACKTRACE
1237static void __ftrace_trace_stack(struct ring_buffer *buffer,
1238                 unsigned long flags,
1239                 int skip, int pc)
1240{
1241    struct ftrace_event_call *call = &event_kernel_stack;
1242    struct ring_buffer_event *event;
1243    struct stack_entry *entry;
1244    struct stack_trace trace;
1245
1246    event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1247                      sizeof(*entry), flags, pc);
1248    if (!event)
1249        return;
1250    entry = ring_buffer_event_data(event);
1251    memset(&entry->caller, 0, sizeof(entry->caller));
1252
1253    trace.nr_entries = 0;
1254    trace.max_entries = FTRACE_STACK_ENTRIES;
1255    trace.skip = skip;
1256    trace.entries = entry->caller;
1257
1258    save_stack_trace(&trace);
1259    if (!filter_check_discard(call, entry, buffer, event))
1260        ring_buffer_unlock_commit(buffer, event);
1261}
1262
1263void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1264            int skip, int pc)
1265{
1266    if (!(trace_flags & TRACE_ITER_STACKTRACE))
1267        return;
1268
1269    __ftrace_trace_stack(buffer, flags, skip, pc);
1270}
1271
1272void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1273           int pc)
1274{
1275    __ftrace_trace_stack(tr->buffer, flags, skip, pc);
1276}
1277
1278/**
1279 * trace_dump_stack - record a stack back trace in the trace buffer
1280 */
1281void trace_dump_stack(void)
1282{
1283    unsigned long flags;
1284
1285    if (tracing_disabled || tracing_selftest_running)
1286        return;
1287
1288    local_save_flags(flags);
1289
1290    /* skipping 3 traces, seems to get us at the caller of this function */
1291    __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1292}
1293
1294static DEFINE_PER_CPU(int, user_stack_count);
1295
1296void
1297ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1298{
1299    struct ftrace_event_call *call = &event_user_stack;
1300    struct ring_buffer_event *event;
1301    struct userstack_entry *entry;
1302    struct stack_trace trace;
1303
1304    if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1305        return;
1306
1307    /*
1308     * NMIs can not handle page faults, even with fix ups.
1309     * The save user stack can (and often does) fault.
1310     */
1311    if (unlikely(in_nmi()))
1312        return;
1313
1314    /*
1315     * prevent recursion, since the user stack tracing may
1316     * trigger other kernel events.
1317     */
1318    preempt_disable();
1319    if (__this_cpu_read(user_stack_count))
1320        goto out;
1321
1322    __this_cpu_inc(user_stack_count);
1323
1324    event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1325                      sizeof(*entry), flags, pc);
1326    if (!event)
1327        goto out_drop_count;
1328    entry = ring_buffer_event_data(event);
1329
1330    entry->tgid = current->tgid;
1331    memset(&entry->caller, 0, sizeof(entry->caller));
1332
1333    trace.nr_entries = 0;
1334    trace.max_entries = FTRACE_STACK_ENTRIES;
1335    trace.skip = 0;
1336    trace.entries = entry->caller;
1337
1338    save_stack_trace_user(&trace);
1339    if (!filter_check_discard(call, entry, buffer, event))
1340        ring_buffer_unlock_commit(buffer, event);
1341
1342 out_drop_count:
1343    __this_cpu_dec(user_stack_count);
1344 out:
1345    preempt_enable();
1346}
1347
1348#ifdef UNUSED
1349static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1350{
1351    ftrace_trace_userstack(tr, flags, preempt_count());
1352}
1353#endif /* UNUSED */
1354
1355#endif /* CONFIG_STACKTRACE */
1356
1357/**
1358 * trace_vbprintk - write binary msg to tracing buffer
1359 *
1360 */
1361int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1362{
1363    static arch_spinlock_t trace_buf_lock =
1364        (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1365    static u32 trace_buf[TRACE_BUF_SIZE];
1366
1367    struct ftrace_event_call *call = &event_bprint;
1368    struct ring_buffer_event *event;
1369    struct ring_buffer *buffer;
1370    struct trace_array *tr = &global_trace;
1371    struct trace_array_cpu *data;
1372    struct bprint_entry *entry;
1373    unsigned long flags;
1374    int disable;
1375    int cpu, len = 0, size, pc;
1376
1377    if (unlikely(tracing_selftest_running || tracing_disabled))
1378        return 0;
1379
1380    /* Don't pollute graph traces with trace_vprintk internals */
1381    pause_graph_tracing();
1382
1383    pc = preempt_count();
1384    preempt_disable_notrace();
1385    cpu = raw_smp_processor_id();
1386    data = tr->data[cpu];
1387
1388    disable = atomic_inc_return(&data->disabled);
1389    if (unlikely(disable != 1))
1390        goto out;
1391
1392    /* Lockdep uses trace_printk for lock tracing */
1393    local_irq_save(flags);
1394    arch_spin_lock(&trace_buf_lock);
1395    len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1396
1397    if (len > TRACE_BUF_SIZE || len < 0)
1398        goto out_unlock;
1399
1400    size = sizeof(*entry) + sizeof(u32) * len;
1401    buffer = tr->buffer;
1402    event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1403                      flags, pc);
1404    if (!event)
1405        goto out_unlock;
1406    entry = ring_buffer_event_data(event);
1407    entry->ip = ip;
1408    entry->fmt = fmt;
1409
1410    memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1411    if (!filter_check_discard(call, entry, buffer, event)) {
1412        ring_buffer_unlock_commit(buffer, event);
1413        ftrace_trace_stack(buffer, flags, 6, pc);
1414    }
1415
1416out_unlock:
1417    arch_spin_unlock(&trace_buf_lock);
1418    local_irq_restore(flags);
1419
1420out:
1421    atomic_dec_return(&data->disabled);
1422    preempt_enable_notrace();
1423    unpause_graph_tracing();
1424
1425    return len;
1426}
1427EXPORT_SYMBOL_GPL(trace_vbprintk);
1428
1429int trace_array_printk(struct trace_array *tr,
1430               unsigned long ip, const char *fmt, ...)
1431{
1432    int ret;
1433    va_list ap;
1434
1435    if (!(trace_flags & TRACE_ITER_PRINTK))
1436        return 0;
1437
1438    va_start(ap, fmt);
1439    ret = trace_array_vprintk(tr, ip, fmt, ap);
1440    va_end(ap);
1441    return ret;
1442}
1443
1444int trace_array_vprintk(struct trace_array *tr,
1445            unsigned long ip, const char *fmt, va_list args)
1446{
1447    static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1448    static char trace_buf[TRACE_BUF_SIZE];
1449
1450    struct ftrace_event_call *call = &event_print;
1451    struct ring_buffer_event *event;
1452    struct ring_buffer *buffer;
1453    struct trace_array_cpu *data;
1454    int cpu, len = 0, size, pc;
1455    struct print_entry *entry;
1456    unsigned long irq_flags;
1457    int disable;
1458
1459    if (tracing_disabled || tracing_selftest_running)
1460        return 0;
1461
1462    pc = preempt_count();
1463    preempt_disable_notrace();
1464    cpu = raw_smp_processor_id();
1465    data = tr->data[cpu];
1466
1467    disable = atomic_inc_return(&data->disabled);
1468    if (unlikely(disable != 1))
1469        goto out;
1470
1471    pause_graph_tracing();
1472    raw_local_irq_save(irq_flags);
1473    arch_spin_lock(&trace_buf_lock);
1474    len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1475
1476    size = sizeof(*entry) + len + 1;
1477    buffer = tr->buffer;
1478    event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1479                      irq_flags, pc);
1480    if (!event)
1481        goto out_unlock;
1482    entry = ring_buffer_event_data(event);
1483    entry->ip = ip;
1484
1485    memcpy(&entry->buf, trace_buf, len);
1486    entry->buf[len] = '\0';
1487    if (!filter_check_discard(call, entry, buffer, event)) {
1488        ring_buffer_unlock_commit(buffer, event);
1489        ftrace_trace_stack(buffer, irq_flags, 6, pc);
1490    }
1491
1492 out_unlock:
1493    arch_spin_unlock(&trace_buf_lock);
1494    raw_local_irq_restore(irq_flags);
1495    unpause_graph_tracing();
1496 out:
1497    atomic_dec_return(&data->disabled);
1498    preempt_enable_notrace();
1499
1500    return len;
1501}
1502
1503int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1504{
1505    return trace_array_vprintk(&global_trace, ip, fmt, args);
1506}
1507EXPORT_SYMBOL_GPL(trace_vprintk);
1508
1509static void trace_iterator_increment(struct trace_iterator *iter)
1510{
1511    /* Don't allow ftrace to trace into the ring buffers */
1512    ftrace_disable_cpu();
1513
1514    iter->idx++;
1515    if (iter->buffer_iter[iter->cpu])
1516        ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1517
1518    ftrace_enable_cpu();
1519}
1520
1521static struct trace_entry *
1522peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1523        unsigned long *lost_events)
1524{
1525    struct ring_buffer_event *event;
1526    struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1527
1528    /* Don't allow ftrace to trace into the ring buffers */
1529    ftrace_disable_cpu();
1530
1531    if (buf_iter)
1532        event = ring_buffer_iter_peek(buf_iter, ts);
1533    else
1534        event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1535                     lost_events);
1536
1537    ftrace_enable_cpu();
1538
1539    return event ? ring_buffer_event_data(event) : NULL;
1540}
1541
1542static struct trace_entry *
1543__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1544          unsigned long *missing_events, u64 *ent_ts)
1545{
1546    struct ring_buffer *buffer = iter->tr->buffer;
1547    struct trace_entry *ent, *next = NULL;
1548    unsigned long lost_events = 0, next_lost = 0;
1549    int cpu_file = iter->cpu_file;
1550    u64 next_ts = 0, ts;
1551    int next_cpu = -1;
1552    int cpu;
1553
1554    /*
1555     * If we are in a per_cpu trace file, don't bother by iterating over
1556     * all cpu and peek directly.
1557     */
1558    if (cpu_file > TRACE_PIPE_ALL_CPU) {
1559        if (ring_buffer_empty_cpu(buffer, cpu_file))
1560            return NULL;
1561        ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1562        if (ent_cpu)
1563            *ent_cpu = cpu_file;
1564
1565        return ent;
1566    }
1567
1568    for_each_tracing_cpu(cpu) {
1569
1570        if (ring_buffer_empty_cpu(buffer, cpu))
1571            continue;
1572
1573        ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1574
1575        /*
1576         * Pick the entry with the smallest timestamp:
1577         */
1578        if (ent && (!next || ts < next_ts)) {
1579            next = ent;
1580            next_cpu = cpu;
1581            next_ts = ts;
1582            next_lost = lost_events;
1583        }
1584    }
1585
1586    if (ent_cpu)
1587        *ent_cpu = next_cpu;
1588
1589    if (ent_ts)
1590        *ent_ts = next_ts;
1591
1592    if (missing_events)
1593        *missing_events = next_lost;
1594
1595    return next;
1596}
1597
1598/* Find the next real entry, without updating the iterator itself */
1599struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1600                      int *ent_cpu, u64 *ent_ts)
1601{
1602    return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1603}
1604
1605/* Find the next real entry, and increment the iterator to the next entry */
1606void *trace_find_next_entry_inc(struct trace_iterator *iter)
1607{
1608    iter->ent = __find_next_entry(iter, &iter->cpu,
1609                      &iter->lost_events, &iter->ts);
1610
1611    if (iter->ent)
1612        trace_iterator_increment(iter);
1613
1614    return iter->ent ? iter : NULL;
1615}
1616
1617static void trace_consume(struct trace_iterator *iter)
1618{
1619    /* Don't allow ftrace to trace into the ring buffers */
1620    ftrace_disable_cpu();
1621    ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1622                &iter->lost_events);
1623    ftrace_enable_cpu();
1624}
1625
1626static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1627{
1628    struct trace_iterator *iter = m->private;
1629    int i = (int)*pos;
1630    void *ent;
1631
1632    WARN_ON_ONCE(iter->leftover);
1633
1634    (*pos)++;
1635
1636    /* can't go backwards */
1637    if (iter->idx > i)
1638        return NULL;
1639
1640    if (iter->idx < 0)
1641        ent = trace_find_next_entry_inc(iter);
1642    else
1643        ent = iter;
1644
1645    while (ent && iter->idx < i)
1646        ent = trace_find_next_entry_inc(iter);
1647
1648    iter->pos = *pos;
1649
1650    return ent;
1651}
1652
1653void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1654{
1655    struct trace_array *tr = iter->tr;
1656    struct ring_buffer_event *event;
1657    struct ring_buffer_iter *buf_iter;
1658    unsigned long entries = 0;
1659    u64 ts;
1660
1661    tr->data[cpu]->skipped_entries = 0;
1662
1663    if (!iter->buffer_iter[cpu])
1664        return;
1665
1666    buf_iter = iter->buffer_iter[cpu];
1667    ring_buffer_iter_reset(buf_iter);
1668
1669    /*
1670     * We could have the case with the max latency tracers
1671     * that a reset never took place on a cpu. This is evident
1672     * by the timestamp being before the start of the buffer.
1673     */
1674    while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1675        if (ts >= iter->tr->time_start)
1676            break;
1677        entries++;
1678        ring_buffer_read(buf_iter, NULL);
1679    }
1680
1681    tr->data[cpu]->skipped_entries = entries;
1682}
1683
1684/*
1685 * The current tracer is copied to avoid a global locking
1686 * all around.
1687 */
1688static void *s_start(struct seq_file *m, loff_t *pos)
1689{
1690    struct trace_iterator *iter = m->private;
1691    static struct tracer *old_tracer;
1692    int cpu_file = iter->cpu_file;
1693    void *p = NULL;
1694    loff_t l = 0;
1695    int cpu;
1696
1697    /* copy the tracer to avoid using a global lock all around */
1698    mutex_lock(&trace_types_lock);
1699    if (unlikely(old_tracer != current_trace && current_trace)) {
1700        old_tracer = current_trace;
1701        *iter->trace = *current_trace;
1702    }
1703    mutex_unlock(&trace_types_lock);
1704
1705    atomic_inc(&trace_record_cmdline_disabled);
1706
1707    if (*pos != iter->pos) {
1708        iter->ent = NULL;
1709        iter->cpu = 0;
1710        iter->idx = -1;
1711
1712        ftrace_disable_cpu();
1713
1714        if (cpu_file == TRACE_PIPE_ALL_CPU) {
1715            for_each_tracing_cpu(cpu)
1716                tracing_iter_reset(iter, cpu);
1717        } else
1718            tracing_iter_reset(iter, cpu_file);
1719
1720        ftrace_enable_cpu();
1721
1722        iter->leftover = 0;
1723        for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1724            ;
1725
1726    } else {
1727        /*
1728         * If we overflowed the seq_file before, then we want
1729         * to just reuse the trace_seq buffer again.
1730         */
1731        if (iter->leftover)
1732            p = iter;
1733        else {
1734            l = *pos - 1;
1735            p = s_next(m, p, &l);
1736        }
1737    }
1738
1739    trace_event_read_lock();
1740    trace_access_lock(cpu_file);
1741    return p;
1742}
1743
1744static void s_stop(struct seq_file *m, void *p)
1745{
1746    struct trace_iterator *iter = m->private;
1747
1748    atomic_dec(&trace_record_cmdline_disabled);
1749    trace_access_unlock(iter->cpu_file);
1750    trace_event_read_unlock();
1751}
1752
1753static void print_lat_help_header(struct seq_file *m)
1754{
1755    seq_puts(m, "# _------=> CPU# \n");
1756    seq_puts(m, "# / _-----=> irqs-off \n");
1757    seq_puts(m, "# | / _----=> need-resched \n");
1758    seq_puts(m, "# || / _---=> hardirq/softirq \n");
1759    seq_puts(m, "# ||| / _--=> preempt-depth \n");
1760    seq_puts(m, "# |||| / delay \n");
1761    seq_puts(m, "# cmd pid ||||| time | caller \n");
1762    seq_puts(m, "# \\ / ||||| \\ | / \n");
1763}
1764
1765static void print_func_help_header(struct seq_file *m)
1766{
1767    seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1768    seq_puts(m, "# | | | | |\n");
1769}
1770
1771
1772void
1773print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1774{
1775    unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1776    struct trace_array *tr = iter->tr;
1777    struct trace_array_cpu *data = tr->data[tr->cpu];
1778    struct tracer *type = current_trace;
1779    unsigned long entries = 0;
1780    unsigned long total = 0;
1781    unsigned long count;
1782    const char *name = "preemption";
1783    int cpu;
1784
1785    if (type)
1786        name = type->name;
1787
1788
1789    for_each_tracing_cpu(cpu) {
1790        count = ring_buffer_entries_cpu(tr->buffer, cpu);
1791        /*
1792         * If this buffer has skipped entries, then we hold all
1793         * entries for the trace and we need to ignore the
1794         * ones before the time stamp.
1795         */
1796        if (tr->data[cpu]->skipped_entries) {
1797            count -= tr->data[cpu]->skipped_entries;
1798            /* total is the same as the entries */
1799            total += count;
1800        } else
1801            total += count +
1802                ring_buffer_overrun_cpu(tr->buffer, cpu);
1803        entries += count;
1804    }
1805
1806    seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1807           name, UTS_RELEASE);
1808    seq_puts(m, "# -----------------------------------"
1809         "---------------------------------\n");
1810    seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1811           " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1812           nsecs_to_usecs(data->saved_latency),
1813           entries,
1814           total,
1815           tr->cpu,
1816#if defined(CONFIG_PREEMPT_NONE)
1817           "server",
1818#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1819           "desktop",
1820#elif defined(CONFIG_PREEMPT)
1821           "preempt",
1822#else
1823           "unknown",
1824#endif
1825           /* These are reserved for later use */
1826           0, 0, 0, 0);
1827#ifdef CONFIG_SMP
1828    seq_printf(m, " #P:%d)\n", num_online_cpus());
1829#else
1830    seq_puts(m, ")\n");
1831#endif
1832    seq_puts(m, "# -----------------\n");
1833    seq_printf(m, "# | task: %.16s-%d "
1834           "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1835           data->comm, data->pid, data->uid, data->nice,
1836           data->policy, data->rt_priority);
1837    seq_puts(m, "# -----------------\n");
1838
1839    if (data->critical_start) {
1840        seq_puts(m, "# => started at: ");
1841        seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1842        trace_print_seq(m, &iter->seq);
1843        seq_puts(m, "\n# => ended at: ");
1844        seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1845        trace_print_seq(m, &iter->seq);
1846        seq_puts(m, "\n#\n");
1847    }
1848
1849    seq_puts(m, "#\n");
1850}
1851
1852static void test_cpu_buff_start(struct trace_iterator *iter)
1853{
1854    struct trace_seq *s = &iter->seq;
1855
1856    if (!(trace_flags & TRACE_ITER_ANNOTATE))
1857        return;
1858
1859    if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1860        return;
1861
1862    if (cpumask_test_cpu(iter->cpu, iter->started))
1863        return;
1864
1865    if (iter->tr->data[iter->cpu]->skipped_entries)
1866        return;
1867
1868    cpumask_set_cpu(iter->cpu, iter->started);
1869
1870    /* Don't print started cpu buffer for the first entry of the trace */
1871    if (iter->idx > 1)
1872        trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1873                iter->cpu);
1874}
1875
1876static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1877{
1878    struct trace_seq *s = &iter->seq;
1879    unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1880    struct trace_entry *entry;
1881    struct trace_event *event;
1882
1883    entry = iter->ent;
1884
1885    test_cpu_buff_start(iter);
1886
1887    event = ftrace_find_event(entry->type);
1888
1889    if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1890        if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1891            if (!trace_print_lat_context(iter))
1892                goto partial;
1893        } else {
1894            if (!trace_print_context(iter))
1895                goto partial;
1896        }
1897    }
1898
1899    if (event)
1900        return event->funcs->trace(iter, sym_flags, event);
1901
1902    if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1903        goto partial;
1904
1905    return TRACE_TYPE_HANDLED;
1906partial:
1907    return TRACE_TYPE_PARTIAL_LINE;
1908}
1909
1910static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1911{
1912    struct trace_seq *s = &iter->seq;
1913    struct trace_entry *entry;
1914    struct trace_event *event;
1915
1916    entry = iter->ent;
1917
1918    if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1919        if (!trace_seq_printf(s, "%d %d %llu ",
1920                      entry->pid, iter->cpu, iter->ts))
1921            goto partial;
1922    }
1923
1924    event = ftrace_find_event(entry->type);
1925    if (event)
1926        return event->funcs->raw(iter, 0, event);
1927
1928    if (!trace_seq_printf(s, "%d ?\n", entry->type))
1929        goto partial;
1930
1931    return TRACE_TYPE_HANDLED;
1932partial:
1933    return TRACE_TYPE_PARTIAL_LINE;
1934}
1935
1936static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1937{
1938    struct trace_seq *s = &iter->seq;
1939    unsigned char newline = '\n';
1940    struct trace_entry *entry;
1941    struct trace_event *event;
1942
1943    entry = iter->ent;
1944
1945    if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1946        SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1947        SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1948        SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1949    }
1950
1951    event = ftrace_find_event(entry->type);
1952    if (event) {
1953        enum print_line_t ret = event->funcs->hex(iter, 0, event);
1954        if (ret != TRACE_TYPE_HANDLED)
1955            return ret;
1956    }
1957
1958    SEQ_PUT_FIELD_RET(s, newline);
1959
1960    return TRACE_TYPE_HANDLED;
1961}
1962
1963static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1964{
1965    struct trace_seq *s = &iter->seq;
1966    struct trace_entry *entry;
1967    struct trace_event *event;
1968
1969    entry = iter->ent;
1970
1971    if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1972        SEQ_PUT_FIELD_RET(s, entry->pid);
1973        SEQ_PUT_FIELD_RET(s, iter->cpu);
1974        SEQ_PUT_FIELD_RET(s, iter->ts);
1975    }
1976
1977    event = ftrace_find_event(entry->type);
1978    return event ? event->funcs->binary(iter, 0, event) :
1979        TRACE_TYPE_HANDLED;
1980}
1981
1982int trace_empty(struct trace_iterator *iter)
1983{
1984    int cpu;
1985
1986    /* If we are looking at one CPU buffer, only check that one */
1987    if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1988        cpu = iter->cpu_file;
1989        if (iter->buffer_iter[cpu]) {
1990            if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1991                return 0;
1992        } else {
1993            if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1994                return 0;
1995        }
1996        return 1;
1997    }
1998
1999    for_each_tracing_cpu(cpu) {
2000        if (iter->buffer_iter[cpu]) {
2001            if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2002                return 0;
2003        } else {
2004            if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2005                return 0;
2006        }
2007    }
2008
2009    return 1;
2010}
2011
2012/* Called with trace_event_read_lock() held. */
2013enum print_line_t print_trace_line(struct trace_iterator *iter)
2014{
2015    enum print_line_t ret;
2016
2017    if (iter->lost_events &&
2018        !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2019                 iter->cpu, iter->lost_events))
2020        return TRACE_TYPE_PARTIAL_LINE;
2021
2022    if (iter->trace && iter->trace->print_line) {
2023        ret = iter->trace->print_line(iter);
2024        if (ret != TRACE_TYPE_UNHANDLED)
2025            return ret;
2026    }
2027
2028    if (iter->ent->type == TRACE_BPRINT &&
2029            trace_flags & TRACE_ITER_PRINTK &&
2030            trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2031        return trace_print_bprintk_msg_only(iter);
2032
2033    if (iter->ent->type == TRACE_PRINT &&
2034            trace_flags & TRACE_ITER_PRINTK &&
2035            trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2036        return trace_print_printk_msg_only(iter);
2037
2038    if (trace_flags & TRACE_ITER_BIN)
2039        return print_bin_fmt(iter);
2040
2041    if (trace_flags & TRACE_ITER_HEX)
2042        return print_hex_fmt(iter);
2043
2044    if (trace_flags & TRACE_ITER_RAW)
2045        return print_raw_fmt(iter);
2046
2047    return print_trace_fmt(iter);
2048}
2049
2050void trace_default_header(struct seq_file *m)
2051{
2052    struct trace_iterator *iter = m->private;
2053
2054    if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2055        /* print nothing if the buffers are empty */
2056        if (trace_empty(iter))
2057            return;
2058        print_trace_header(m, iter);
2059        if (!(trace_flags & TRACE_ITER_VERBOSE))
2060            print_lat_help_header(m);
2061    } else {
2062        if (!(trace_flags & TRACE_ITER_VERBOSE))
2063            print_func_help_header(m);
2064    }
2065}
2066
2067static int s_show(struct seq_file *m, void *v)
2068{
2069    struct trace_iterator *iter = v;
2070    int ret;
2071
2072    if (iter->ent == NULL) {
2073        if (iter->tr) {
2074            seq_printf(m, "# tracer: %s\n", iter->trace->name);
2075            seq_puts(m, "#\n");
2076        }
2077        if (iter->trace && iter->trace->print_header)
2078            iter->trace->print_header(m);
2079        else
2080            trace_default_header(m);
2081
2082    } else if (iter->leftover) {
2083        /*
2084         * If we filled the seq_file buffer earlier, we
2085         * want to just show it now.
2086         */
2087        ret = trace_print_seq(m, &iter->seq);
2088
2089        /* ret should this time be zero, but you never know */
2090        iter->leftover = ret;
2091
2092    } else {
2093        print_trace_line(iter);
2094        ret = trace_print_seq(m, &iter->seq);
2095        /*
2096         * If we overflow the seq_file buffer, then it will
2097         * ask us for this data again at start up.
2098         * Use that instead.
2099         * ret is 0 if seq_file write succeeded.
2100         * -1 otherwise.
2101         */
2102        iter->leftover = ret;
2103    }
2104
2105    return 0;
2106}
2107
2108static const struct seq_operations tracer_seq_ops = {
2109    .start = s_start,
2110    .next = s_next,
2111    .stop = s_stop,
2112    .show = s_show,
2113};
2114
2115static struct trace_iterator *
2116__tracing_open(struct inode *inode, struct file *file)
2117{
2118    long cpu_file = (long) inode->i_private;
2119    void *fail_ret = ERR_PTR(-ENOMEM);
2120    struct trace_iterator *iter;
2121    struct seq_file *m;
2122    int cpu, ret;
2123
2124    if (tracing_disabled)
2125        return ERR_PTR(-ENODEV);
2126
2127    iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2128    if (!iter)
2129        return ERR_PTR(-ENOMEM);
2130
2131    /*
2132     * We make a copy of the current tracer to avoid concurrent
2133     * changes on it while we are reading.
2134     */
2135    mutex_lock(&trace_types_lock);
2136    iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2137    if (!iter->trace)
2138        goto fail;
2139
2140    if (current_trace)
2141        *iter->trace = *current_trace;
2142
2143    if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2144        goto fail;
2145
2146    if (current_trace && current_trace->print_max)
2147        iter->tr = &max_tr;
2148    else
2149        iter->tr = &global_trace;
2150    iter->pos = -1;
2151    mutex_init(&iter->mutex);
2152    iter->cpu_file = cpu_file;
2153
2154    /* Notify the tracer early; before we stop tracing. */
2155    if (iter->trace && iter->trace->open)
2156        iter->trace->open(iter);
2157
2158    /* Annotate start of buffers if we had overruns */
2159    if (ring_buffer_overruns(iter->tr->buffer))
2160        iter->iter_flags |= TRACE_FILE_ANNOTATE;
2161
2162    /* stop the trace while dumping */
2163    tracing_stop();
2164
2165    if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2166        for_each_tracing_cpu(cpu) {
2167            iter->buffer_iter[cpu] =
2168                ring_buffer_read_prepare(iter->tr->buffer, cpu);
2169        }
2170        ring_buffer_read_prepare_sync();
2171        for_each_tracing_cpu(cpu) {
2172            ring_buffer_read_start(iter->buffer_iter[cpu]);
2173            tracing_iter_reset(iter, cpu);
2174        }
2175    } else {
2176        cpu = iter->cpu_file;
2177        iter->buffer_iter[cpu] =
2178            ring_buffer_read_prepare(iter->tr->buffer, cpu);
2179        ring_buffer_read_prepare_sync();
2180        ring_buffer_read_start(iter->buffer_iter[cpu]);
2181        tracing_iter_reset(iter, cpu);
2182    }
2183
2184    ret = seq_open(file, &tracer_seq_ops);
2185    if (ret < 0) {
2186        fail_ret = ERR_PTR(ret);
2187        goto fail_buffer;
2188    }
2189
2190    m = file->private_data;
2191    m->private = iter;
2192
2193    mutex_unlock(&trace_types_lock);
2194
2195    return iter;
2196
2197 fail_buffer:
2198    for_each_tracing_cpu(cpu) {
2199        if (iter->buffer_iter[cpu])
2200            ring_buffer_read_finish(iter->buffer_iter[cpu]);
2201    }
2202    free_cpumask_var(iter->started);
2203    tracing_start();
2204 fail:
2205    mutex_unlock(&trace_types_lock);
2206    kfree(iter->trace);
2207    kfree(iter);
2208
2209    return fail_ret;
2210}
2211
2212int tracing_open_generic(struct inode *inode, struct file *filp)
2213{
2214    if (tracing_disabled)
2215        return -ENODEV;
2216
2217    filp->private_data = inode->i_private;
2218    return 0;
2219}
2220
2221static int tracing_release(struct inode *inode, struct file *file)
2222{
2223    struct seq_file *m = file->private_data;
2224    struct trace_iterator *iter;
2225    int cpu;
2226
2227    if (!(file->f_mode & FMODE_READ))
2228        return 0;
2229
2230    iter = m->private;
2231
2232    mutex_lock(&trace_types_lock);
2233    for_each_tracing_cpu(cpu) {
2234        if (iter->buffer_iter[cpu])
2235            ring_buffer_read_finish(iter->buffer_iter[cpu]);
2236    }
2237
2238    if (iter->trace && iter->trace->close)
2239        iter->trace->close(iter);
2240
2241    /* reenable tracing if it was previously enabled */
2242    tracing_start();
2243    mutex_unlock(&trace_types_lock);
2244
2245    seq_release(inode, file);
2246    mutex_destroy(&iter->mutex);
2247    free_cpumask_var(iter->started);
2248    kfree(iter->trace);
2249    kfree(iter);
2250    return 0;
2251}
2252
2253static int tracing_open(struct inode *inode, struct file *file)
2254{
2255    struct trace_iterator *iter;
2256    int ret = 0;
2257
2258    /* If this file was open for write, then erase contents */
2259    if ((file->f_mode & FMODE_WRITE) &&
2260        (file->f_flags & O_TRUNC)) {
2261        long cpu = (long) inode->i_private;
2262
2263        if (cpu == TRACE_PIPE_ALL_CPU)
2264            tracing_reset_online_cpus(&global_trace);
2265        else
2266            tracing_reset(&global_trace, cpu);
2267    }
2268
2269    if (file->f_mode & FMODE_READ) {
2270        iter = __tracing_open(inode, file);
2271        if (IS_ERR(iter))
2272            ret = PTR_ERR(iter);
2273        else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2274            iter->iter_flags |= TRACE_FILE_LAT_FMT;
2275    }
2276    return ret;
2277}
2278
2279static void *
2280t_next(struct seq_file *m, void *v, loff_t *pos)
2281{
2282    struct tracer *t = v;
2283
2284    (*pos)++;
2285
2286    if (t)
2287        t = t->next;
2288
2289    return t;
2290}
2291
2292static void *t_start(struct seq_file *m, loff_t *pos)
2293{
2294    struct tracer *t;
2295    loff_t l = 0;
2296
2297    mutex_lock(&trace_types_lock);
2298    for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2299        ;
2300
2301    return t;
2302}
2303
2304static void t_stop(struct seq_file *m, void *p)
2305{
2306    mutex_unlock(&trace_types_lock);
2307}
2308
2309static int t_show(struct seq_file *m, void *v)
2310{
2311    struct tracer *t = v;
2312
2313    if (!t)
2314        return 0;
2315
2316    seq_printf(m, "%s", t->name);
2317    if (t->next)
2318        seq_putc(m, ' ');
2319    else
2320        seq_putc(m, '\n');
2321
2322    return 0;
2323}
2324
2325static const struct seq_operations show_traces_seq_ops = {
2326    .start = t_start,
2327    .next = t_next,
2328    .stop = t_stop,
2329    .show = t_show,
2330};
2331
2332static int show_traces_open(struct inode *inode, struct file *file)
2333{
2334    if (tracing_disabled)
2335        return -ENODEV;
2336
2337    return seq_open(file, &show_traces_seq_ops);
2338}
2339
2340static ssize_t
2341tracing_write_stub(struct file *filp, const char __user *ubuf,
2342           size_t count, loff_t *ppos)
2343{
2344    return count;
2345}
2346
2347static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2348{
2349    if (file->f_mode & FMODE_READ)
2350        return seq_lseek(file, offset, origin);
2351    else
2352        return 0;
2353}
2354
2355static const struct file_operations tracing_fops = {
2356    .open = tracing_open,
2357    .read = seq_read,
2358    .write = tracing_write_stub,
2359    .llseek = tracing_seek,
2360    .release = tracing_release,
2361};
2362
2363static const struct file_operations show_traces_fops = {
2364    .open = show_traces_open,
2365    .read = seq_read,
2366    .release = seq_release,
2367    .llseek = seq_lseek,
2368};
2369
2370/*
2371 * Only trace on a CPU if the bitmask is set:
2372 */
2373static cpumask_var_t tracing_cpumask;
2374
2375/*
2376 * The tracer itself will not take this lock, but still we want
2377 * to provide a consistent cpumask to user-space:
2378 */
2379static DEFINE_MUTEX(tracing_cpumask_update_lock);
2380
2381/*
2382 * Temporary storage for the character representation of the
2383 * CPU bitmask (and one more byte for the newline):
2384 */
2385static char mask_str[NR_CPUS + 1];
2386
2387static ssize_t
2388tracing_cpumask_read(struct file *filp, char __user *ubuf,
2389             size_t count, loff_t *ppos)
2390{
2391    int len;
2392
2393    mutex_lock(&tracing_cpumask_update_lock);
2394
2395    len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2396    if (count - len < 2) {
2397        count = -EINVAL;
2398        goto out_err;
2399    }
2400    len += sprintf(mask_str + len, "\n");
2401    count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2402
2403out_err:
2404    mutex_unlock(&tracing_cpumask_update_lock);
2405
2406    return count;
2407}
2408
2409static ssize_t
2410tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2411              size_t count, loff_t *ppos)
2412{
2413    int err, cpu;
2414    cpumask_var_t tracing_cpumask_new;
2415
2416    if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2417        return -ENOMEM;
2418
2419    err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2420    if (err)
2421        goto err_unlock;
2422
2423    mutex_lock(&tracing_cpumask_update_lock);
2424
2425    local_irq_disable();
2426    arch_spin_lock(&ftrace_max_lock);
2427    for_each_tracing_cpu(cpu) {
2428        /*
2429         * Increase/decrease the disabled counter if we are
2430         * about to flip a bit in the cpumask:
2431         */
2432        if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2433                !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2434            atomic_inc(&global_trace.data[cpu]->disabled);
2435        }
2436        if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2437                cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2438            atomic_dec(&global_trace.data[cpu]->disabled);
2439        }
2440    }
2441    arch_spin_unlock(&ftrace_max_lock);
2442    local_irq_enable();
2443
2444    cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2445
2446    mutex_unlock(&tracing_cpumask_update_lock);
2447    free_cpumask_var(tracing_cpumask_new);
2448
2449    return count;
2450
2451err_unlock:
2452    free_cpumask_var(tracing_cpumask_new);
2453
2454    return err;
2455}
2456
2457static const struct file_operations tracing_cpumask_fops = {
2458    .open = tracing_open_generic,
2459    .read = tracing_cpumask_read,
2460    .write = tracing_cpumask_write,
2461    .llseek = generic_file_llseek,
2462};
2463
2464static int tracing_trace_options_show(struct seq_file *m, void *v)
2465{
2466    struct tracer_opt *trace_opts;
2467    u32 tracer_flags;
2468    int i;
2469
2470    mutex_lock(&trace_types_lock);
2471    tracer_flags = current_trace->flags->val;
2472    trace_opts = current_trace->flags->opts;
2473
2474    for (i = 0; trace_options[i]; i++) {
2475        if (trace_flags & (1 << i))
2476            seq_printf(m, "%s\n", trace_options[i]);
2477        else
2478            seq_printf(m, "no%s\n", trace_options[i]);
2479    }
2480
2481    for (i = 0; trace_opts[i].name; i++) {
2482        if (tracer_flags & trace_opts[i].bit)
2483            seq_printf(m, "%s\n", trace_opts[i].name);
2484        else
2485            seq_printf(m, "no%s\n", trace_opts[i].name);
2486    }
2487    mutex_unlock(&trace_types_lock);
2488
2489    return 0;
2490}
2491
2492static int __set_tracer_option(struct tracer *trace,
2493                   struct tracer_flags *tracer_flags,
2494                   struct tracer_opt *opts, int neg)
2495{
2496    int ret;
2497
2498    ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2499    if (ret)
2500        return ret;
2501
2502    if (neg)
2503        tracer_flags->val &= ~opts->bit;
2504    else
2505        tracer_flags->val |= opts->bit;
2506    return 0;
2507}
2508
2509/* Try to assign a tracer specific option */
2510static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2511{
2512    struct tracer_flags *tracer_flags = trace->flags;
2513    struct tracer_opt *opts = NULL;
2514    int i;
2515
2516    for (i = 0; tracer_flags->opts[i].name; i++) {
2517        opts = &tracer_flags->opts[i];
2518
2519        if (strcmp(cmp, opts->name) == 0)
2520            return __set_tracer_option(trace, trace->flags,
2521                           opts, neg);
2522    }
2523
2524    return -EINVAL;
2525}
2526
2527static void set_tracer_flags(unsigned int mask, int enabled)
2528{
2529    /* do nothing if flag is already set */
2530    if (!!(trace_flags & mask) == !!enabled)
2531        return;
2532
2533    if (enabled)
2534        trace_flags |= mask;
2535    else
2536        trace_flags &= ~mask;
2537
2538    if (mask == TRACE_ITER_RECORD_CMD)
2539        trace_event_enable_cmd_record(enabled);
2540
2541    if (mask == TRACE_ITER_OVERWRITE)
2542        ring_buffer_change_overwrite(global_trace.buffer, enabled);
2543}
2544
2545static ssize_t
2546tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2547            size_t cnt, loff_t *ppos)
2548{
2549    char buf[64];
2550    char *cmp;
2551    int neg = 0;
2552    int ret;
2553    int i;
2554
2555    if (cnt >= sizeof(buf))
2556        return -EINVAL;
2557
2558    if (copy_from_user(&buf, ubuf, cnt))
2559        return -EFAULT;
2560
2561    buf[cnt] = 0;
2562    cmp = strstrip(buf);
2563
2564    if (strncmp(cmp, "no", 2) == 0) {
2565        neg = 1;
2566        cmp += 2;
2567    }
2568
2569    for (i = 0; trace_options[i]; i++) {
2570        if (strcmp(cmp, trace_options[i]) == 0) {
2571            set_tracer_flags(1 << i, !neg);
2572            break;
2573        }
2574    }
2575
2576    /* If no option could be set, test the specific tracer options */
2577    if (!trace_options[i]) {
2578        mutex_lock(&trace_types_lock);
2579        ret = set_tracer_option(current_trace, cmp, neg);
2580        mutex_unlock(&trace_types_lock);
2581        if (ret)
2582            return ret;
2583    }
2584
2585    *ppos += cnt;
2586
2587    return cnt;
2588}
2589
2590static int tracing_trace_options_open(struct inode *inode, struct file *file)
2591{
2592    if (tracing_disabled)
2593        return -ENODEV;
2594    return single_open(file, tracing_trace_options_show, NULL);
2595}
2596
2597static const struct file_operations tracing_iter_fops = {
2598    .open = tracing_trace_options_open,
2599    .read = seq_read,
2600    .llseek = seq_lseek,
2601    .release = single_release,
2602    .write = tracing_trace_options_write,
2603};
2604
2605static const char readme_msg[] =
2606    "tracing mini-HOWTO:\n\n"
2607    "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2608    "# cat /sys/kernel/debug/tracing/available_tracers\n"
2609    "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2610    "# cat /sys/kernel/debug/tracing/current_tracer\n"
2611    "nop\n"
2612    "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2613    "# cat /sys/kernel/debug/tracing/current_tracer\n"
2614    "sched_switch\n"
2615    "# cat /sys/kernel/debug/tracing/trace_options\n"
2616    "noprint-parent nosym-offset nosym-addr noverbose\n"
2617    "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2618    "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2619    "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2620    "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2621;
2622
2623static ssize_t
2624tracing_readme_read(struct file *filp, char __user *ubuf,
2625               size_t cnt, loff_t *ppos)
2626{
2627    return simple_read_from_buffer(ubuf, cnt, ppos,
2628                    readme_msg, strlen(readme_msg));
2629}
2630
2631static const struct file_operations tracing_readme_fops = {
2632    .open = tracing_open_generic,
2633    .read = tracing_readme_read,
2634    .llseek = generic_file_llseek,
2635};
2636
2637static ssize_t
2638tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2639                size_t cnt, loff_t *ppos)
2640{
2641    char *buf_comm;
2642    char *file_buf;
2643    char *buf;
2644    int len = 0;
2645    int pid;
2646    int i;
2647
2648    file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2649    if (!file_buf)
2650        return -ENOMEM;
2651
2652    buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2653    if (!buf_comm) {
2654        kfree(file_buf);
2655        return -ENOMEM;
2656    }
2657
2658    buf = file_buf;
2659
2660    for (i = 0; i < SAVED_CMDLINES; i++) {
2661        int r;
2662
2663        pid = map_cmdline_to_pid[i];
2664        if (pid == -1 || pid == NO_CMDLINE_MAP)
2665            continue;
2666
2667        trace_find_cmdline(pid, buf_comm);
2668        r = sprintf(buf, "%d %s\n", pid, buf_comm);
2669        buf += r;
2670        len += r;
2671    }
2672
2673    len = simple_read_from_buffer(ubuf, cnt, ppos,
2674                      file_buf, len);
2675
2676    kfree(file_buf);
2677    kfree(buf_comm);
2678
2679    return len;
2680}
2681
2682static const struct file_operations tracing_saved_cmdlines_fops = {
2683    .open = tracing_open_generic,
2684    .read = tracing_saved_cmdlines_read,
2685    .llseek = generic_file_llseek,
2686};
2687
2688static ssize_t
2689tracing_ctrl_read(struct file *filp, char __user *ubuf,
2690          size_t cnt, loff_t *ppos)
2691{
2692    char buf[64];
2693    int r;
2694
2695    r = sprintf(buf, "%u\n", tracer_enabled);
2696    return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2697}
2698
2699static ssize_t
2700tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2701           size_t cnt, loff_t *ppos)
2702{
2703    struct trace_array *tr = filp->private_data;
2704    char buf[64];
2705    unsigned long val;
2706    int ret;
2707
2708    if (cnt >= sizeof(buf))
2709        return -EINVAL;
2710
2711    if (copy_from_user(&buf, ubuf, cnt))
2712        return -EFAULT;
2713
2714    buf[cnt] = 0;
2715
2716    ret = strict_strtoul(buf, 10, &val);
2717    if (ret < 0)
2718        return ret;
2719
2720    val = !!val;
2721
2722    mutex_lock(&trace_types_lock);
2723    if (tracer_enabled ^ val) {
2724
2725        /* Only need to warn if this is used to change the state */
2726        WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2727
2728        if (val) {
2729            tracer_enabled = 1;
2730            if (current_trace->start)
2731                current_trace->start(tr);
2732            tracing_start();
2733        } else {
2734            tracer_enabled = 0;
2735            tracing_stop();
2736            if (current_trace->stop)
2737                current_trace->stop(tr);
2738        }
2739    }
2740    mutex_unlock(&trace_types_lock);
2741
2742    *ppos += cnt;
2743
2744    return cnt;
2745}
2746
2747static ssize_t
2748tracing_set_trace_read(struct file *filp, char __user *ubuf,
2749               size_t cnt, loff_t *ppos)
2750{
2751    char buf[MAX_TRACER_SIZE+2];
2752    int r;
2753
2754    mutex_lock(&trace_types_lock);
2755    if (current_trace)
2756        r = sprintf(buf, "%s\n", current_trace->name);
2757    else
2758        r = sprintf(buf, "\n");
2759    mutex_unlock(&trace_types_lock);
2760
2761    return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2762}
2763
2764int tracer_init(struct tracer *t, struct trace_array *tr)
2765{
2766    tracing_reset_online_cpus(tr);
2767    return t->init(tr);
2768}
2769
2770static int tracing_resize_ring_buffer(unsigned long size)
2771{
2772    int ret;
2773
2774    /*
2775     * If kernel or user changes the size of the ring buffer
2776     * we use the size that was given, and we can forget about
2777     * expanding it later.
2778     */
2779    ring_buffer_expanded = 1;
2780
2781    ret = ring_buffer_resize(global_trace.buffer, size);
2782    if (ret < 0)
2783        return ret;
2784
2785    if (!current_trace->use_max_tr)
2786        goto out;
2787
2788    ret = ring_buffer_resize(max_tr.buffer, size);
2789    if (ret < 0) {
2790        int r;
2791
2792        r = ring_buffer_resize(global_trace.buffer,
2793                       global_trace.entries);
2794        if (r < 0) {
2795            /*
2796             * AARGH! We are left with different
2797             * size max buffer!!!!
2798             * The max buffer is our "snapshot" buffer.
2799             * When a tracer needs a snapshot (one of the
2800             * latency tracers), it swaps the max buffer
2801             * with the saved snap shot. We succeeded to
2802             * update the size of the main buffer, but failed to
2803             * update the size of the max buffer. But when we tried
2804             * to reset the main buffer to the original size, we
2805             * failed there too. This is very unlikely to
2806             * happen, but if it does, warn and kill all
2807             * tracing.
2808             */
2809            WARN_ON(1);
2810            tracing_disabled = 1;
2811        }
2812        return ret;
2813    }
2814
2815    max_tr.entries = size;
2816 out:
2817    global_trace.entries = size;
2818
2819    return ret;
2820}
2821
2822
2823/**
2824 * tracing_update_buffers - used by tracing facility to expand ring buffers
2825 *
2826 * To save on memory when the tracing is never used on a system with it
2827 * configured in. The ring buffers are set to a minimum size. But once
2828 * a user starts to use the tracing facility, then they need to grow
2829 * to their default size.
2830 *
2831 * This function is to be called when a tracer is about to be used.
2832 */
2833int tracing_update_buffers(void)
2834{
2835    int ret = 0;
2836
2837    mutex_lock(&trace_types_lock);
2838    if (!ring_buffer_expanded)
2839        ret = tracing_resize_ring_buffer(trace_buf_size);
2840    mutex_unlock(&trace_types_lock);
2841
2842    return ret;
2843}
2844
2845struct trace_option_dentry;
2846
2847static struct trace_option_dentry *
2848create_trace_option_files(struct tracer *tracer);
2849
2850static void
2851destroy_trace_option_files(struct trace_option_dentry *topts);
2852
2853static int tracing_set_tracer(const char *buf)
2854{
2855    static struct trace_option_dentry *topts;
2856    struct trace_array *tr = &global_trace;
2857    struct tracer *t;
2858    int ret = 0;
2859
2860    mutex_lock(&trace_types_lock);
2861
2862    if (!ring_buffer_expanded) {
2863        ret = tracing_resize_ring_buffer(trace_buf_size);
2864        if (ret < 0)
2865            goto out;
2866        ret = 0;
2867    }
2868
2869    for (t = trace_types; t; t = t->next) {
2870        if (strcmp(t->name, buf) == 0)
2871            break;
2872    }
2873    if (!t) {
2874        ret = -EINVAL;
2875        goto out;
2876    }
2877    if (t == current_trace)
2878        goto out;
2879
2880    trace_branch_disable();
2881    if (current_trace && current_trace->reset)
2882        current_trace->reset(tr);
2883    if (current_trace && current_trace->use_max_tr) {
2884        /*
2885         * We don't free the ring buffer. instead, resize it because
2886         * The max_tr ring buffer has some state (e.g. ring->clock) and
2887         * we want preserve it.
2888         */
2889        ring_buffer_resize(max_tr.buffer, 1);
2890        max_tr.entries = 1;
2891    }
2892    destroy_trace_option_files(topts);
2893
2894    current_trace = t;
2895
2896    topts = create_trace_option_files(current_trace);
2897    if (current_trace->use_max_tr) {
2898        ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
2899        if (ret < 0)
2900            goto out;
2901        max_tr.entries = global_trace.entries;
2902    }
2903
2904    if (t->init) {
2905        ret = tracer_init(t, tr);
2906        if (ret)
2907            goto out;
2908    }
2909
2910    trace_branch_enable(tr);
2911 out:
2912    mutex_unlock(&trace_types_lock);
2913
2914    return ret;
2915}
2916
2917static ssize_t
2918tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2919            size_t cnt, loff_t *ppos)
2920{
2921    char buf[MAX_TRACER_SIZE+1];
2922    int i;
2923    size_t ret;
2924    int err;
2925
2926    ret = cnt;
2927
2928    if (cnt > MAX_TRACER_SIZE)
2929        cnt = MAX_TRACER_SIZE;
2930
2931    if (copy_from_user(&buf, ubuf, cnt))
2932        return -EFAULT;
2933
2934    buf[cnt] = 0;
2935
2936    /* strip ending whitespace. */
2937    for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2938        buf[i] = 0;
2939
2940    err = tracing_set_tracer(buf);
2941    if (err)
2942        return err;
2943
2944    *ppos += ret;
2945
2946    return ret;
2947}
2948
2949static ssize_t
2950tracing_max_lat_read(struct file *filp, char __user *ubuf,
2951             size_t cnt, loff_t *ppos)
2952{
2953    unsigned long *ptr = filp->private_data;
2954    char buf[64];
2955    int r;
2956
2957    r = snprintf(buf, sizeof(buf), "%ld\n",
2958             *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2959    if (r > sizeof(buf))
2960        r = sizeof(buf);
2961    return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2962}
2963
2964static ssize_t
2965tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2966              size_t cnt, loff_t *ppos)
2967{
2968    unsigned long *ptr = filp->private_data;
2969    char buf[64];
2970    unsigned long val;
2971    int ret;
2972
2973    if (cnt >= sizeof(buf))
2974        return -EINVAL;
2975
2976    if (copy_from_user(&buf, ubuf, cnt))
2977        return -EFAULT;
2978
2979    buf[cnt] = 0;
2980
2981    ret = strict_strtoul(buf, 10, &val);
2982    if (ret < 0)
2983        return ret;
2984
2985    *ptr = val * 1000;
2986
2987    return cnt;
2988}
2989
2990static int tracing_open_pipe(struct inode *inode, struct file *filp)
2991{
2992    long cpu_file = (long) inode->i_private;
2993    struct trace_iterator *iter;
2994    int ret = 0;
2995
2996    if (tracing_disabled)
2997        return -ENODEV;
2998
2999    mutex_lock(&trace_types_lock);
3000
3001    /* create a buffer to store the information to pass to userspace */
3002    iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3003    if (!iter) {
3004        ret = -ENOMEM;
3005        goto out;
3006    }
3007
3008    /*
3009     * We make a copy of the current tracer to avoid concurrent
3010     * changes on it while we are reading.
3011     */
3012    iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3013    if (!iter->trace) {
3014        ret = -ENOMEM;
3015        goto fail;
3016    }
3017    if (current_trace)
3018        *iter->trace = *current_trace;
3019
3020    if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3021        ret = -ENOMEM;
3022        goto fail;
3023    }
3024
3025    /* trace pipe does not show start of buffer */
3026    cpumask_setall(iter->started);
3027
3028    if (trace_flags & TRACE_ITER_LATENCY_FMT)
3029        iter->iter_flags |= TRACE_FILE_LAT_FMT;
3030
3031    iter->cpu_file = cpu_file;
3032    iter->tr = &global_trace;
3033    mutex_init(&iter->mutex);
3034    filp->private_data = iter;
3035
3036    if (iter->trace->pipe_open)
3037        iter->trace->pipe_open(iter);
3038
3039    nonseekable_open(inode, filp);
3040out:
3041    mutex_unlock(&trace_types_lock);
3042    return ret;
3043
3044fail:
3045    kfree(iter->trace);
3046    kfree(iter);
3047    mutex_unlock(&trace_types_lock);
3048    return ret;
3049}
3050
3051static int tracing_release_pipe(struct inode *inode, struct file *file)
3052{
3053    struct trace_iterator *iter = file->private_data;
3054
3055    mutex_lock(&trace_types_lock);
3056
3057    if (iter->trace->pipe_close)
3058        iter->trace->pipe_close(iter);
3059
3060    mutex_unlock(&trace_types_lock);
3061
3062    free_cpumask_var(iter->started);
3063    mutex_destroy(&iter->mutex);
3064    kfree(iter->trace);
3065    kfree(iter);
3066
3067    return 0;
3068}
3069
3070static unsigned int
3071tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3072{
3073    struct trace_iterator *iter = filp->private_data;
3074
3075    if (trace_flags & TRACE_ITER_BLOCK) {
3076        /*
3077         * Always select as readable when in blocking mode
3078         */
3079        return POLLIN | POLLRDNORM;
3080    } else {
3081        if (!trace_empty(iter))
3082            return POLLIN | POLLRDNORM;
3083        poll_wait(filp, &trace_wait, poll_table);
3084        if (!trace_empty(iter))
3085            return POLLIN | POLLRDNORM;
3086
3087        return 0;
3088    }
3089}
3090
3091
3092void default_wait_pipe(struct trace_iterator *iter)
3093{
3094    DEFINE_WAIT(wait);
3095
3096    prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3097
3098    if (trace_empty(iter))
3099        schedule();
3100
3101    finish_wait(&trace_wait, &wait);
3102}
3103
3104/*
3105 * This is a make-shift waitqueue.
3106 * A tracer might use this callback on some rare cases:
3107 *
3108 * 1) the current tracer might hold the runqueue lock when it wakes up
3109 * a reader, hence a deadlock (sched, function, and function graph tracers)
3110 * 2) the function tracers, trace all functions, we don't want
3111 * the overhead of calling wake_up and friends
3112 * (and tracing them too)
3113 *
3114 * Anyway, this is really very primitive wakeup.
3115 */
3116void poll_wait_pipe(struct trace_iterator *iter)
3117{
3118    set_current_state(TASK_INTERRUPTIBLE);
3119    /* sleep for 100 msecs, and try again. */
3120    schedule_timeout(HZ / 10);
3121}
3122
3123/* Must be called with trace_types_lock mutex held. */
3124static int tracing_wait_pipe(struct file *filp)
3125{
3126    struct trace_iterator *iter = filp->private_data;
3127
3128    while (trace_empty(iter)) {
3129
3130        if ((filp->f_flags & O_NONBLOCK)) {
3131            return -EAGAIN;
3132        }
3133
3134        mutex_unlock(&iter->mutex);
3135
3136        iter->trace->wait_pipe(iter);
3137
3138        mutex_lock(&iter->mutex);
3139
3140        if (signal_pending(current))
3141            return -EINTR;
3142
3143        /*
3144         * We block until we read something and tracing is disabled.
3145         * We still block if tracing is disabled, but we have never
3146         * read anything. This allows a user to cat this file, and
3147         * then enable tracing. But after we have read something,
3148         * we give an EOF when tracing is again disabled.
3149         *
3150         * iter->pos will be 0 if we haven't read anything.
3151         */
3152        if (!tracer_enabled && iter->pos)
3153            break;
3154    }
3155
3156    return 1;
3157}
3158
3159/*
3160 * Consumer reader.
3161 */
3162static ssize_t
3163tracing_read_pipe(struct file *filp, char __user *ubuf,
3164          size_t cnt, loff_t *ppos)
3165{
3166    struct trace_iterator *iter = filp->private_data;
3167    static struct tracer *old_tracer;
3168    ssize_t sret;
3169
3170    /* return any leftover data */
3171    sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3172    if (sret != -EBUSY)
3173        return sret;
3174
3175    trace_seq_init(&iter->seq);
3176
3177    /* copy the tracer to avoid using a global lock all around */
3178    mutex_lock(&trace_types_lock);
3179    if (unlikely(old_tracer != current_trace && current_trace)) {
3180        old_tracer = current_trace;
3181        *iter->trace = *current_trace;
3182    }
3183    mutex_unlock(&trace_types_lock);
3184
3185    /*
3186     * Avoid more than one consumer on a single file descriptor
3187     * This is just a matter of traces coherency, the ring buffer itself
3188     * is protected.
3189     */
3190    mutex_lock(&iter->mutex);
3191    if (iter->trace->read) {
3192        sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3193        if (sret)
3194            goto out;
3195    }
3196
3197waitagain:
3198    sret = tracing_wait_pipe(filp);
3199    if (sret <= 0)
3200        goto out;
3201
3202    /* stop when tracing is finished */
3203    if (trace_empty(iter)) {
3204        sret = 0;
3205        goto out;
3206    }
3207
3208    if (cnt >= PAGE_SIZE)
3209        cnt = PAGE_SIZE - 1;
3210
3211    /* reset all but tr, trace, and overruns */
3212    memset(&iter->seq, 0,
3213           sizeof(struct trace_iterator) -
3214           offsetof(struct trace_iterator, seq));
3215    iter->pos = -1;
3216
3217    trace_event_read_lock();
3218    trace_access_lock(iter->cpu_file);
3219    while (trace_find_next_entry_inc(iter) != NULL) {
3220        enum print_line_t ret;
3221        int len = iter->seq.len;
3222
3223        ret = print_trace_line(iter);
3224        if (ret == TRACE_TYPE_PARTIAL_LINE) {
3225            /* don't print partial lines */
3226            iter->seq.len = len;
3227            break;
3228        }
3229        if (ret != TRACE_TYPE_NO_CONSUME)
3230            trace_consume(iter);
3231
3232        if (iter->seq.len >= cnt)
3233            break;
3234
3235        /*
3236         * Setting the full flag means we reached the trace_seq buffer
3237         * size and we should leave by partial output condition above.
3238         * One of the trace_seq_* functions is not used properly.
3239         */
3240        WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3241              iter->ent->type);
3242    }
3243    trace_access_unlock(iter->cpu_file);
3244    trace_event_read_unlock();
3245
3246    /* Now copy what we have to the user */
3247    sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3248    if (iter->seq.readpos >= iter->seq.len)
3249        trace_seq_init(&iter->seq);
3250
3251    /*
3252     * If there was nothing to send to user, in spite of consuming trace
3253     * entries, go back to wait for more entries.
3254     */
3255    if (sret == -EBUSY)
3256        goto waitagain;
3257
3258out:
3259    mutex_unlock(&iter->mutex);
3260
3261    return sret;
3262}
3263
3264static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3265                     struct pipe_buffer *buf)
3266{
3267    __free_page(buf->page);
3268}
3269
3270static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3271                     unsigned int idx)
3272{
3273    __free_page(spd->pages[idx]);
3274}
3275
3276static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3277    .can_merge = 0,
3278    .map = generic_pipe_buf_map,
3279    .unmap = generic_pipe_buf_unmap,
3280    .confirm = generic_pipe_buf_confirm,
3281    .release = tracing_pipe_buf_release,
3282    .steal = generic_pipe_buf_steal,
3283    .get = generic_pipe_buf_get,
3284};
3285
3286static size_t
3287tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3288{
3289    size_t count;
3290    int ret;
3291
3292    /* Seq buffer is page-sized, exactly what we need. */
3293    for (;;) {
3294        count = iter->seq.len;
3295        ret = print_trace_line(iter);
3296        count = iter->seq.len - count;
3297        if (rem < count) {
3298            rem = 0;
3299            iter->seq.len -= count;
3300            break;
3301        }
3302        if (ret == TRACE_TYPE_PARTIAL_LINE) {
3303            iter->seq.len -= count;
3304            break;
3305        }
3306
3307        if (ret != TRACE_TYPE_NO_CONSUME)
3308            trace_consume(iter);
3309        rem -= count;
3310        if (!trace_find_next_entry_inc(iter)) {
3311            rem = 0;
3312            iter->ent = NULL;
3313            break;
3314        }
3315    }
3316
3317    return rem;
3318}
3319
3320static ssize_t tracing_splice_read_pipe(struct file *filp,
3321                    loff_t *ppos,
3322                    struct pipe_inode_info *pipe,
3323                    size_t len,
3324                    unsigned int flags)
3325{
3326    struct page *pages_def[PIPE_DEF_BUFFERS];
3327    struct partial_page partial_def[PIPE_DEF_BUFFERS];
3328    struct trace_iterator *iter = filp->private_data;
3329    struct splice_pipe_desc spd = {
3330        .pages = pages_def,
3331        .partial = partial_def,
3332        .nr_pages = 0, /* This gets updated below. */
3333        .flags = flags,
3334        .ops = &tracing_pipe_buf_ops,
3335        .spd_release = tracing_spd_release_pipe,
3336    };
3337    static struct tracer *old_tracer;
3338    ssize_t ret;
3339    size_t rem;
3340    unsigned int i;
3341
3342    if (splice_grow_spd(pipe, &spd))
3343        return -ENOMEM;
3344
3345    /* copy the tracer to avoid using a global lock all around */
3346    mutex_lock(&trace_types_lock);
3347    if (unlikely(old_tracer != current_trace && current_trace)) {
3348        old_tracer = current_trace;
3349        *iter->trace = *current_trace;
3350    }
3351    mutex_unlock(&trace_types_lock);
3352
3353    mutex_lock(&iter->mutex);
3354
3355    if (iter->trace->splice_read) {
3356        ret = iter->trace->splice_read(iter, filp,
3357                           ppos, pipe, len, flags);
3358        if (ret)
3359            goto out_err;
3360    }
3361
3362    ret = tracing_wait_pipe(filp);
3363    if (ret <= 0)
3364        goto out_err;
3365
3366    if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3367        ret = -EFAULT;
3368        goto out_err;
3369    }
3370
3371    trace_event_read_lock();
3372    trace_access_lock(iter->cpu_file);
3373
3374    /* Fill as many pages as possible. */
3375    for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3376        spd.pages[i] = alloc_page(GFP_KERNEL);
3377        if (!spd.pages[i])
3378            break;
3379
3380        rem = tracing_fill_pipe_page(rem, iter);
3381
3382        /* Copy the data into the page, so we can start over. */
3383        ret = trace_seq_to_buffer(&iter->seq,
3384                      page_address(spd.pages[i]),
3385                      iter->seq.len);
3386        if (ret < 0) {
3387            __free_page(spd.pages[i]);
3388            break;
3389        }
3390        spd.partial[i].offset = 0;
3391        spd.partial[i].len = iter->seq.len;
3392
3393        trace_seq_init(&iter->seq);
3394    }
3395
3396    trace_access_unlock(iter->cpu_file);
3397    trace_event_read_unlock();
3398    mutex_unlock(&iter->mutex);
3399
3400    spd.nr_pages = i;
3401
3402    ret = splice_to_pipe(pipe, &spd);
3403out:
3404    splice_shrink_spd(pipe, &spd);
3405    return ret;
3406
3407out_err:
3408    mutex_unlock(&iter->mutex);
3409    goto out;
3410}
3411
3412static ssize_t
3413tracing_entries_read(struct file *filp, char __user *ubuf,
3414             size_t cnt, loff_t *ppos)
3415{
3416    struct trace_array *tr = filp->private_data;
3417    char buf[96];
3418    int r;
3419
3420    mutex_lock(&trace_types_lock);
3421    if (!ring_buffer_expanded)
3422        r = sprintf(buf, "%lu (expanded: %lu)\n",
3423                tr->entries >> 10,
3424                trace_buf_size >> 10);
3425    else
3426        r = sprintf(buf, "%lu\n", tr->entries >> 10);
3427    mutex_unlock(&trace_types_lock);
3428
3429    return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3430}
3431
3432static ssize_t
3433tracing_entries_write(struct file *filp, const char __user *ubuf,
3434              size_t cnt, loff_t *ppos)
3435{
3436    unsigned long val;
3437    char buf[64];
3438    int ret, cpu;
3439
3440    if (cnt >= sizeof(buf))
3441        return -EINVAL;
3442
3443    if (copy_from_user(&buf, ubuf, cnt))
3444        return -EFAULT;
3445
3446    buf[cnt] = 0;
3447
3448    ret = strict_strtoul(buf, 10, &val);
3449    if (ret < 0)
3450        return ret;
3451
3452    /* must have at least 1 entry */
3453    if (!val)
3454        return -EINVAL;
3455
3456    mutex_lock(&trace_types_lock);
3457
3458    tracing_stop();
3459
3460    /* disable all cpu buffers */
3461    for_each_tracing_cpu(cpu) {
3462        if (global_trace.data[cpu])
3463            atomic_inc(&global_trace.data[cpu]->disabled);
3464        if (max_tr.data[cpu])
3465            atomic_inc(&max_tr.data[cpu]->disabled);
3466    }
3467
3468    /* value is in KB */
3469    val <<= 10;
3470
3471    if (val != global_trace.entries) {
3472        ret = tracing_resize_ring_buffer(val);
3473        if (ret < 0) {
3474            cnt = ret;
3475            goto out;
3476        }
3477    }
3478
3479    *ppos += cnt;
3480
3481    /* If check pages failed, return ENOMEM */
3482    if (tracing_disabled)
3483        cnt = -ENOMEM;
3484 out:
3485    for_each_tracing_cpu(cpu) {
3486        if (global_trace.data[cpu])
3487            atomic_dec(&global_trace.data[cpu]->disabled);
3488        if (max_tr.data[cpu])
3489            atomic_dec(&max_tr.data[cpu]->disabled);
3490    }
3491
3492    tracing_start();
3493    mutex_unlock(&trace_types_lock);
3494
3495    return cnt;
3496}
3497
3498static int mark_printk(const char *fmt, ...)
3499{
3500    int ret;
3501    va_list args;
3502    va_start(args, fmt);
3503    ret = trace_vprintk(0, fmt, args);
3504    va_end(args);
3505    return ret;
3506}
3507
3508static ssize_t
3509tracing_mark_write(struct file *filp, const char __user *ubuf,
3510                    size_t cnt, loff_t *fpos)
3511{
3512    char *buf;
3513    size_t written;
3514
3515    if (tracing_disabled)
3516        return -EINVAL;
3517
3518    if (cnt > TRACE_BUF_SIZE)
3519        cnt = TRACE_BUF_SIZE;
3520
3521    buf = kmalloc(cnt + 2, GFP_KERNEL);
3522    if (buf == NULL)
3523        return -ENOMEM;
3524
3525    if (copy_from_user(buf, ubuf, cnt)) {
3526        kfree(buf);
3527        return -EFAULT;
3528    }
3529    if (buf[cnt-1] != '\n') {
3530        buf[cnt] = '\n';
3531        buf[cnt+1] = '\0';
3532    } else
3533        buf[cnt] = '\0';
3534
3535    written = mark_printk("%s", buf);
3536    kfree(buf);
3537    *fpos += written;
3538
3539    /* don't tell userspace we wrote more - it might confuse them */
3540    if (written > cnt)
3541        written = cnt;
3542
3543    return written;
3544}
3545
3546static int tracing_clock_show(struct seq_file *m, void *v)
3547{
3548    int i;
3549
3550    for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3551        seq_printf(m,
3552            "%s%s%s%s", i ? " " : "",
3553            i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3554            i == trace_clock_id ? "]" : "");
3555    seq_putc(m, '\n');
3556
3557    return 0;
3558}
3559
3560static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3561                   size_t cnt, loff_t *fpos)
3562{
3563    char buf[64];
3564    const char *clockstr;
3565    int i;
3566
3567    if (cnt >= sizeof(buf))
3568        return -EINVAL;
3569
3570    if (copy_from_user(&buf, ubuf, cnt))
3571        return -EFAULT;
3572
3573    buf[cnt] = 0;
3574
3575    clockstr = strstrip(buf);
3576
3577    for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3578        if (strcmp(trace_clocks[i].name, clockstr) == 0)
3579            break;
3580    }
3581    if (i == ARRAY_SIZE(trace_clocks))
3582        return -EINVAL;
3583
3584    trace_clock_id = i;
3585
3586    mutex_lock(&trace_types_lock);
3587
3588    ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3589    if (max_tr.buffer)
3590        ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3591
3592    mutex_unlock(&trace_types_lock);
3593
3594    *fpos += cnt;
3595
3596    return cnt;
3597}
3598
3599static int tracing_clock_open(struct inode *inode, struct file *file)
3600{
3601    if (tracing_disabled)
3602        return -ENODEV;
3603    return single_open(file, tracing_clock_show, NULL);
3604}
3605
3606static const struct file_operations tracing_max_lat_fops = {
3607    .open = tracing_open_generic,
3608    .read = tracing_max_lat_read,
3609    .write = tracing_max_lat_write,
3610    .llseek = generic_file_llseek,
3611};
3612
3613static const struct file_operations tracing_ctrl_fops = {
3614    .open = tracing_open_generic,
3615    .read = tracing_ctrl_read,
3616    .write = tracing_ctrl_write,
3617    .llseek = generic_file_llseek,
3618};
3619
3620static const struct file_operations set_tracer_fops = {
3621    .open = tracing_open_generic,
3622    .read = tracing_set_trace_read,
3623    .write = tracing_set_trace_write,
3624    .llseek = generic_file_llseek,
3625};
3626
3627static const struct file_operations tracing_pipe_fops = {
3628    .open = tracing_open_pipe,
3629    .poll = tracing_poll_pipe,
3630    .read = tracing_read_pipe,
3631    .splice_read = tracing_splice_read_pipe,
3632    .release = tracing_release_pipe,
3633    .llseek = no_llseek,
3634};
3635
3636static const struct file_operations tracing_entries_fops = {
3637    .open = tracing_open_generic,
3638    .read = tracing_entries_read,
3639    .write = tracing_entries_write,
3640    .llseek = generic_file_llseek,
3641};
3642
3643static const struct file_operations tracing_mark_fops = {
3644    .open = tracing_open_generic,
3645    .write = tracing_mark_write,
3646    .llseek = generic_file_llseek,
3647};
3648
3649static const struct file_operations trace_clock_fops = {
3650    .open = tracing_clock_open,
3651    .read = seq_read,
3652    .llseek = seq_lseek,
3653    .release = single_release,
3654    .write = tracing_clock_write,
3655};
3656
3657struct ftrace_buffer_info {
3658    struct trace_array *tr;
3659    void *spare;
3660    int cpu;
3661    unsigned int read;
3662};
3663
3664static int tracing_buffers_open(struct inode *inode, struct file *filp)
3665{
3666    int cpu = (int)(long)inode->i_private;
3667    struct ftrace_buffer_info *info;
3668
3669    if (tracing_disabled)
3670        return -ENODEV;
3671
3672    info = kzalloc(sizeof(*info), GFP_KERNEL);
3673    if (!info)
3674        return -ENOMEM;
3675
3676    info->tr = &global_trace;
3677    info->cpu = cpu;
3678    info->spare = NULL;
3679    /* Force reading ring buffer for first read */
3680    info->read = (unsigned int)-1;
3681
3682    filp->private_data = info;
3683
3684    return nonseekable_open(inode, filp);
3685}
3686
3687static ssize_t
3688tracing_buffers_read(struct file *filp, char __user *ubuf,
3689             size_t count, loff_t *ppos)
3690{
3691    struct ftrace_buffer_info *info = filp->private_data;
3692    ssize_t ret;
3693    size_t size;
3694
3695    if (!count)
3696        return 0;
3697
3698    if (!info->spare)
3699        info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3700    if (!info->spare)
3701        return -ENOMEM;
3702
3703    /* Do we have previous read data to read? */
3704    if (info->read < PAGE_SIZE)
3705        goto read;
3706
3707    info->read = 0;
3708
3709    trace_access_lock(info->cpu);
3710    ret = ring_buffer_read_page(info->tr->buffer,
3711                    &info->spare,
3712                    count,
3713                    info->cpu, 0);
3714    trace_access_unlock(info->cpu);
3715    if (ret < 0)
3716        return 0;
3717
3718read:
3719    size = PAGE_SIZE - info->read;
3720    if (size > count)
3721        size = count;
3722
3723    ret = copy_to_user(ubuf, info->spare + info->read, size);
3724    if (ret == size)
3725        return -EFAULT;
3726    size -= ret;
3727
3728    *ppos += size;
3729    info->read += size;
3730
3731    return size;
3732}
3733
3734static int tracing_buffers_release(struct inode *inode, struct file *file)
3735{
3736    struct ftrace_buffer_info *info = file->private_data;
3737
3738    if (info->spare)
3739        ring_buffer_free_read_page(info->tr->buffer, info->spare);
3740    kfree(info);
3741
3742    return 0;
3743}
3744
3745struct buffer_ref {
3746    struct ring_buffer *buffer;
3747    void *page;
3748    int ref;
3749};
3750
3751static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3752                    struct pipe_buffer *buf)
3753{
3754    struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3755
3756    if (--ref->ref)
3757        return;
3758
3759    ring_buffer_free_read_page(ref->buffer, ref->page);
3760    kfree(ref);
3761    buf->private = 0;
3762}
3763
3764static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3765                 struct pipe_buffer *buf)
3766{
3767    return 1;
3768}
3769
3770static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3771                struct pipe_buffer *buf)
3772{
3773    struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3774
3775    ref->ref++;
3776}
3777
3778/* Pipe buffer operations for a buffer. */
3779static const struct pipe_buf_operations buffer_pipe_buf_ops = {
3780    .can_merge = 0,
3781    .map = generic_pipe_buf_map,
3782    .unmap = generic_pipe_buf_unmap,
3783    .confirm = generic_pipe_buf_confirm,
3784    .release = buffer_pipe_buf_release,
3785    .steal = buffer_pipe_buf_steal,
3786    .get = buffer_pipe_buf_get,
3787};
3788
3789/*
3790 * Callback from splice_to_pipe(), if we need to release some pages
3791 * at the end of the spd in case we error'ed out in filling the pipe.
3792 */
3793static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3794{
3795    struct buffer_ref *ref =
3796        (struct buffer_ref *)spd->partial[i].private;
3797
3798    if (--ref->ref)
3799        return;
3800
3801    ring_buffer_free_read_page(ref->buffer, ref->page);
3802    kfree(ref);
3803    spd->partial[i].private = 0;
3804}
3805
3806static ssize_t
3807tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3808                struct pipe_inode_info *pipe, size_t len,
3809                unsigned int flags)
3810{
3811    struct ftrace_buffer_info *info = file->private_data;
3812    struct partial_page partial_def[PIPE_DEF_BUFFERS];
3813    struct page *pages_def[PIPE_DEF_BUFFERS];
3814    struct splice_pipe_desc spd = {
3815        .pages = pages_def,
3816        .partial = partial_def,
3817        .flags = flags,
3818        .ops = &buffer_pipe_buf_ops,
3819        .spd_release = buffer_spd_release,
3820    };
3821    struct buffer_ref *ref;
3822    int entries, size, i;
3823    size_t ret;
3824
3825    if (splice_grow_spd(pipe, &spd))
3826        return -ENOMEM;
3827
3828    if (*ppos & (PAGE_SIZE - 1)) {
3829        WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3830        ret = -EINVAL;
3831        goto out;
3832    }
3833
3834    if (len & (PAGE_SIZE - 1)) {
3835        WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3836        if (len < PAGE_SIZE) {
3837            ret = -EINVAL;
3838            goto out;
3839        }
3840        len &= PAGE_MASK;
3841    }
3842
3843    trace_access_lock(info->cpu);
3844    entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3845
3846    for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
3847        struct page *page;
3848        int r;
3849
3850        ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3851        if (!ref)
3852            break;
3853
3854        ref->ref = 1;
3855        ref->buffer = info->tr->buffer;
3856        ref->page = ring_buffer_alloc_read_page(ref->buffer);
3857        if (!ref->page) {
3858            kfree(ref);
3859            break;
3860        }
3861
3862        r = ring_buffer_read_page(ref->buffer, &ref->page,
3863                      len, info->cpu, 1);
3864        if (r < 0) {
3865            ring_buffer_free_read_page(ref->buffer,
3866                           ref->page);
3867            kfree(ref);
3868            break;
3869        }
3870
3871        /*
3872         * zero out any left over data, this is going to
3873         * user land.
3874         */
3875        size = ring_buffer_page_len(ref->page);
3876        if (size < PAGE_SIZE)
3877            memset(ref->page + size, 0, PAGE_SIZE - size);
3878
3879        page = virt_to_page(ref->page);
3880
3881        spd.pages[i] = page;
3882        spd.partial[i].len = PAGE_SIZE;
3883        spd.partial[i].offset = 0;
3884        spd.partial[i].private = (unsigned long)ref;
3885        spd.nr_pages++;
3886        *ppos += PAGE_SIZE;
3887
3888        entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3889    }
3890
3891    trace_access_unlock(info->cpu);
3892    spd.nr_pages = i;
3893
3894    /* did we read anything? */
3895    if (!spd.nr_pages) {
3896        if (flags & SPLICE_F_NONBLOCK)
3897            ret = -EAGAIN;
3898        else
3899            ret = 0;
3900        /* TODO: block */
3901        goto out;
3902    }
3903
3904    ret = splice_to_pipe(pipe, &spd);
3905    splice_shrink_spd(pipe, &spd);
3906out:
3907    return ret;
3908}
3909
3910static const struct file_operations tracing_buffers_fops = {
3911    .open = tracing_buffers_open,
3912    .read = tracing_buffers_read,
3913    .release = tracing_buffers_release,
3914    .splice_read = tracing_buffers_splice_read,
3915    .llseek = no_llseek,
3916};
3917
3918static ssize_t
3919tracing_stats_read(struct file *filp, char __user *ubuf,
3920           size_t count, loff_t *ppos)
3921{
3922    unsigned long cpu = (unsigned long)filp->private_data;
3923    struct trace_array *tr = &global_trace;
3924    struct trace_seq *s;
3925    unsigned long cnt;
3926
3927    s = kmalloc(sizeof(*s), GFP_KERNEL);
3928    if (!s)
3929        return -ENOMEM;
3930
3931    trace_seq_init(s);
3932
3933    cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3934    trace_seq_printf(s, "entries: %ld\n", cnt);
3935
3936    cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3937    trace_seq_printf(s, "overrun: %ld\n", cnt);
3938
3939    cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3940    trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3941
3942    count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3943
3944    kfree(s);
3945
3946    return count;
3947}
3948
3949static const struct file_operations tracing_stats_fops = {
3950    .open = tracing_open_generic,
3951    .read = tracing_stats_read,
3952    .llseek = generic_file_llseek,
3953};
3954
3955#ifdef CONFIG_DYNAMIC_FTRACE
3956
3957int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3958{
3959    return 0;
3960}
3961
3962static ssize_t
3963tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3964          size_t cnt, loff_t *ppos)
3965{
3966    static char ftrace_dyn_info_buffer[1024];
3967    static DEFINE_MUTEX(dyn_info_mutex);
3968    unsigned long *p = filp->private_data;
3969    char *buf = ftrace_dyn_info_buffer;
3970    int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3971    int r;
3972
3973    mutex_lock(&dyn_info_mutex);
3974    r = sprintf(buf, "%ld ", *p);
3975
3976    r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3977    buf[r++] = '\n';
3978
3979    r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3980
3981    mutex_unlock(&dyn_info_mutex);
3982
3983    return r;
3984}
3985
3986static const struct file_operations tracing_dyn_info_fops = {
3987    .open = tracing_open_generic,
3988    .read = tracing_read_dyn_info,
3989    .llseek = generic_file_llseek,
3990};
3991#endif
3992
3993static struct dentry *d_tracer;
3994
3995struct dentry *tracing_init_dentry(void)
3996{
3997    static int once;
3998
3999    if (d_tracer)
4000        return d_tracer;
4001
4002    if (!debugfs_initialized())
4003        return NULL;
4004
4005    d_tracer = debugfs_create_dir("tracing", NULL);
4006
4007    if (!d_tracer && !once) {
4008        once = 1;
4009        pr_warning("Could not create debugfs directory 'tracing'\n");
4010        return NULL;
4011    }
4012
4013    return d_tracer;
4014}
4015
4016static struct dentry *d_percpu;
4017
4018struct dentry *tracing_dentry_percpu(void)
4019{
4020    static int once;
4021    struct dentry *d_tracer;
4022
4023    if (d_percpu)
4024        return d_percpu;
4025
4026    d_tracer = tracing_init_dentry();
4027
4028    if (!d_tracer)
4029        return NULL;
4030
4031    d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4032
4033    if (!d_percpu && !once) {
4034        once = 1;
4035        pr_warning("Could not create debugfs directory 'per_cpu'\n");
4036        return NULL;
4037    }
4038
4039    return d_percpu;
4040}
4041
4042static void tracing_init_debugfs_percpu(long cpu)
4043{
4044    struct dentry *d_percpu = tracing_dentry_percpu();
4045    struct dentry *d_cpu;
4046    char cpu_dir[30]; /* 30 characters should be more than enough */
4047
4048    snprintf(cpu_dir, 30, "cpu%ld", cpu);
4049    d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4050    if (!d_cpu) {
4051        pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4052        return;
4053    }
4054
4055    /* per cpu trace_pipe */
4056    trace_create_file("trace_pipe", 0444, d_cpu,
4057            (void *) cpu, &tracing_pipe_fops);
4058
4059    /* per cpu trace */
4060    trace_create_file("trace", 0644, d_cpu,
4061            (void *) cpu, &tracing_fops);
4062
4063    trace_create_file("trace_pipe_raw", 0444, d_cpu,
4064            (void *) cpu, &tracing_buffers_fops);
4065
4066    trace_create_file("stats", 0444, d_cpu,
4067            (void *) cpu, &tracing_stats_fops);
4068}
4069
4070#ifdef CONFIG_FTRACE_SELFTEST
4071/* Let selftest have access to static functions in this file */
4072#include "trace_selftest.c"
4073#endif
4074
4075struct trace_option_dentry {
4076    struct tracer_opt *opt;
4077    struct tracer_flags *flags;
4078    struct dentry *entry;
4079};
4080
4081static ssize_t
4082trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4083            loff_t *ppos)
4084{
4085    struct trace_option_dentry *topt = filp->private_data;
4086    char *buf;
4087
4088    if (topt->flags->val & topt->opt->bit)
4089        buf = "1\n";
4090    else
4091        buf = "0\n";
4092
4093    return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4094}
4095
4096static ssize_t
4097trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4098             loff_t *ppos)
4099{
4100    struct trace_option_dentry *topt = filp->private_data;
4101    unsigned long val;
4102    char buf[64];
4103    int ret;
4104
4105    if (cnt >= sizeof(buf))
4106        return -EINVAL;
4107
4108    if (copy_from_user(&buf, ubuf, cnt))
4109        return -EFAULT;
4110
4111    buf[cnt] = 0;
4112
4113    ret = strict_strtoul(buf, 10, &val);
4114    if (ret < 0)
4115        return ret;
4116
4117    if (val != 0 && val != 1)
4118        return -EINVAL;
4119
4120    if (!!(topt->flags->val & topt->opt->bit) != val) {
4121        mutex_lock(&trace_types_lock);
4122        ret = __set_tracer_option(current_trace, topt->flags,
4123                      topt->opt, !val);
4124        mutex_unlock(&trace_types_lock);
4125        if (ret)
4126            return ret;
4127    }
4128
4129    *ppos += cnt;
4130
4131    return cnt;
4132}
4133
4134
4135static const struct file_operations trace_options_fops = {
4136    .open = tracing_open_generic,
4137    .read = trace_options_read,
4138    .write = trace_options_write,
4139    .llseek = generic_file_llseek,
4140};
4141
4142static ssize_t
4143trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4144            loff_t *ppos)
4145{
4146    long index = (long)filp->private_data;
4147    char *buf;
4148
4149    if (trace_flags & (1 << index))
4150        buf = "1\n";
4151    else
4152        buf = "0\n";
4153
4154    return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4155}
4156
4157static ssize_t
4158trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4159             loff_t *ppos)
4160{
4161    long index = (long)filp->private_data;
4162    char buf[64];
4163    unsigned long val;
4164    int ret;
4165
4166    if (cnt >= sizeof(buf))
4167        return -EINVAL;
4168
4169    if (copy_from_user(&buf, ubuf, cnt))
4170        return -EFAULT;
4171
4172    buf[cnt] = 0;
4173
4174    ret = strict_strtoul(buf, 10, &val);
4175    if (ret < 0)
4176        return ret;
4177
4178    if (val != 0 && val != 1)
4179        return -EINVAL;
4180    set_tracer_flags(1 << index, val);
4181
4182    *ppos += cnt;
4183
4184    return cnt;
4185}
4186
4187static const struct file_operations trace_options_core_fops = {
4188    .open = tracing_open_generic,
4189    .read = trace_options_core_read,
4190    .write = trace_options_core_write,
4191    .llseek = generic_file_llseek,
4192};
4193
4194struct dentry *trace_create_file(const char *name,
4195                 mode_t mode,
4196                 struct dentry *parent,
4197                 void *data,
4198                 const struct file_operations *fops)
4199{
4200    struct dentry *ret;
4201
4202    ret = debugfs_create_file(name, mode, parent, data, fops);
4203    if (!ret)
4204        pr_warning("Could not create debugfs '%s' entry\n", name);
4205
4206    return ret;
4207}
4208
4209
4210static struct dentry *trace_options_init_dentry(void)
4211{
4212    struct dentry *d_tracer;
4213    static struct dentry *t_options;
4214
4215    if (t_options)
4216        return t_options;
4217
4218    d_tracer = tracing_init_dentry();
4219    if (!d_tracer)
4220        return NULL;
4221
4222    t_options = debugfs_create_dir("options", d_tracer);
4223    if (!t_options) {
4224        pr_warning("Could not create debugfs directory 'options'\n");
4225        return NULL;
4226    }
4227
4228    return t_options;
4229}
4230
4231static void
4232create_trace_option_file(struct trace_option_dentry *topt,
4233             struct tracer_flags *flags,
4234             struct tracer_opt *opt)
4235{
4236    struct dentry *t_options;
4237
4238    t_options = trace_options_init_dentry();
4239    if (!t_options)
4240        return;
4241
4242    topt->flags = flags;
4243    topt->opt = opt;
4244
4245    topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4246                    &trace_options_fops);
4247
4248}
4249
4250static struct trace_option_dentry *
4251create_trace_option_files(struct tracer *tracer)
4252{
4253    struct trace_option_dentry *topts;
4254    struct tracer_flags *flags;
4255    struct tracer_opt *opts;
4256    int cnt;
4257
4258    if (!tracer)
4259        return NULL;
4260
4261    flags = tracer->flags;
4262
4263    if (!flags || !flags->opts)
4264        return NULL;
4265
4266    opts = flags->opts;
4267
4268    for (cnt = 0; opts[cnt].name; cnt++)
4269        ;
4270
4271    topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4272    if (!topts)
4273        return NULL;
4274
4275    for (cnt = 0; opts[cnt].name; cnt++)
4276        create_trace_option_file(&topts[cnt], flags,
4277                     &opts[cnt]);
4278
4279    return topts;
4280}
4281
4282static void
4283destroy_trace_option_files(struct trace_option_dentry *topts)
4284{
4285    int cnt;
4286
4287    if (!topts)
4288        return;
4289
4290    for (cnt = 0; topts[cnt].opt; cnt++) {
4291        if (topts[cnt].entry)
4292            debugfs_remove(topts[cnt].entry);
4293    }
4294
4295    kfree(topts);
4296}
4297
4298static struct dentry *
4299create_trace_option_core_file(const char *option, long index)
4300{
4301    struct dentry *t_options;
4302
4303    t_options = trace_options_init_dentry();
4304    if (!t_options)
4305        return NULL;
4306
4307    return trace_create_file(option, 0644, t_options, (void *)index,
4308                    &trace_options_core_fops);
4309}
4310
4311static __init void create_trace_options_dir(void)
4312{
4313    struct dentry *t_options;
4314    int i;
4315
4316    t_options = trace_options_init_dentry();
4317    if (!t_options)
4318        return;
4319
4320    for (i = 0; trace_options[i]; i++)
4321        create_trace_option_core_file(trace_options[i], i);
4322}
4323
4324static __init int tracer_init_debugfs(void)
4325{
4326    struct dentry *d_tracer;
4327    int cpu;
4328
4329    trace_access_lock_init();
4330
4331    d_tracer = tracing_init_dentry();
4332
4333    trace_create_file("tracing_enabled", 0644, d_tracer,
4334            &global_trace, &tracing_ctrl_fops);
4335
4336    trace_create_file("trace_options", 0644, d_tracer,
4337            NULL, &tracing_iter_fops);
4338
4339    trace_create_file("tracing_cpumask", 0644, d_tracer,
4340            NULL, &tracing_cpumask_fops);
4341
4342    trace_create_file("trace", 0644, d_tracer,
4343            (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4344
4345    trace_create_file("available_tracers", 0444, d_tracer,
4346            &global_trace, &show_traces_fops);
4347
4348    trace_create_file("current_tracer", 0644, d_tracer,
4349            &global_trace, &set_tracer_fops);
4350
4351#ifdef CONFIG_TRACER_MAX_TRACE
4352    trace_create_file("tracing_max_latency", 0644, d_tracer,
4353            &tracing_max_latency, &tracing_max_lat_fops);
4354#endif
4355
4356    trace_create_file("tracing_thresh", 0644, d_tracer,
4357            &tracing_thresh, &tracing_max_lat_fops);
4358
4359    trace_create_file("README", 0444, d_tracer,
4360            NULL, &tracing_readme_fops);
4361
4362    trace_create_file("trace_pipe", 0444, d_tracer,
4363            (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4364
4365    trace_create_file("buffer_size_kb", 0644, d_tracer,
4366            &global_trace, &tracing_entries_fops);
4367
4368    trace_create_file("trace_marker", 0220, d_tracer,
4369            NULL, &tracing_mark_fops);
4370
4371    trace_create_file("saved_cmdlines", 0444, d_tracer,
4372            NULL, &tracing_saved_cmdlines_fops);
4373
4374    trace_create_file("trace_clock", 0644, d_tracer, NULL,
4375              &trace_clock_fops);
4376
4377#ifdef CONFIG_DYNAMIC_FTRACE
4378    trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4379            &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4380#endif
4381
4382    create_trace_options_dir();
4383
4384    for_each_tracing_cpu(cpu)
4385        tracing_init_debugfs_percpu(cpu);
4386
4387    return 0;
4388}
4389
4390static int trace_panic_handler(struct notifier_block *this,
4391                   unsigned long event, void *unused)
4392{
4393    if (ftrace_dump_on_oops)
4394        ftrace_dump(ftrace_dump_on_oops);
4395    return NOTIFY_OK;
4396}
4397
4398static struct notifier_block trace_panic_notifier = {
4399    .notifier_call = trace_panic_handler,
4400    .next = NULL,
4401    .priority = 150 /* priority: INT_MAX >= x >= 0 */
4402};
4403
4404static int trace_die_handler(struct notifier_block *self,
4405                 unsigned long val,
4406                 void *data)
4407{
4408    switch (val) {
4409    case DIE_OOPS:
4410        if (ftrace_dump_on_oops)
4411            ftrace_dump(ftrace_dump_on_oops);
4412        break;
4413    default:
4414        break;
4415    }
4416    return NOTIFY_OK;
4417}
4418
4419static struct notifier_block trace_die_notifier = {
4420    .notifier_call = trace_die_handler,
4421    .priority = 200
4422};
4423
4424/*
4425 * printk is set to max of 1024, we really don't need it that big.
4426 * Nothing should be printing 1000 characters anyway.
4427 */
4428#define TRACE_MAX_PRINT 1000
4429
4430/*
4431 * Define here KERN_TRACE so that we have one place to modify
4432 * it if we decide to change what log level the ftrace dump
4433 * should be at.
4434 */
4435#define KERN_TRACE KERN_EMERG
4436
4437void
4438trace_printk_seq(struct trace_seq *s)
4439{
4440    /* Probably should print a warning here. */
4441    if (s->len >= 1000)
4442        s->len = 1000;
4443
4444    /* should be zero ended, but we are paranoid. */
4445    s->buffer[s->len] = 0;
4446
4447    printk(KERN_TRACE "%s", s->buffer);
4448
4449    trace_seq_init(s);
4450}
4451
4452void trace_init_global_iter(struct trace_iterator *iter)
4453{
4454    iter->tr = &global_trace;
4455    iter->trace = current_trace;
4456    iter->cpu_file = TRACE_PIPE_ALL_CPU;
4457}
4458
4459static void
4460__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4461{
4462    static arch_spinlock_t ftrace_dump_lock =
4463        (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4464    /* use static because iter can be a bit big for the stack */
4465    static struct trace_iterator iter;
4466    unsigned int old_userobj;
4467    static int dump_ran;
4468    unsigned long flags;
4469    int cnt = 0, cpu;
4470
4471    /* only one dump */
4472    local_irq_save(flags);
4473    arch_spin_lock(&ftrace_dump_lock);
4474    if (dump_ran)
4475        goto out;
4476
4477    dump_ran = 1;
4478
4479    tracing_off();
4480
4481    if (disable_tracing)
4482        ftrace_kill();
4483
4484    trace_init_global_iter(&iter);
4485
4486    for_each_tracing_cpu(cpu) {
4487        atomic_inc(&iter.tr->data[cpu]->disabled);
4488    }
4489
4490    old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4491
4492    /* don't look at user memory in panic mode */
4493    trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4494
4495    /* Simulate the iterator */
4496    iter.tr = &global_trace;
4497    iter.trace = current_trace;
4498
4499    switch (oops_dump_mode) {
4500    case DUMP_ALL:
4501        iter.cpu_file = TRACE_PIPE_ALL_CPU;
4502        break;
4503    case DUMP_ORIG:
4504        iter.cpu_file = raw_smp_processor_id();
4505        break;
4506    case DUMP_NONE:
4507        goto out_enable;
4508    default:
4509        printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4510        iter.cpu_file = TRACE_PIPE_ALL_CPU;
4511    }
4512
4513    printk(KERN_TRACE "Dumping ftrace buffer:\n");
4514
4515    /*
4516     * We need to stop all tracing on all CPUS to read the
4517     * the next buffer. This is a bit expensive, but is
4518     * not done often. We fill all what we can read,
4519     * and then release the locks again.
4520     */
4521
4522    while (!trace_empty(&iter)) {
4523
4524        if (!cnt)
4525            printk(KERN_TRACE "---------------------------------\n");
4526
4527        cnt++;
4528
4529        /* reset all but tr, trace, and overruns */
4530        memset(&iter.seq, 0,
4531               sizeof(struct trace_iterator) -
4532               offsetof(struct trace_iterator, seq));
4533        iter.iter_flags |= TRACE_FILE_LAT_FMT;
4534        iter.pos = -1;
4535
4536        if (trace_find_next_entry_inc(&iter) != NULL) {
4537            int ret;
4538
4539            ret = print_trace_line(&iter);
4540            if (ret != TRACE_TYPE_NO_CONSUME)
4541                trace_consume(&iter);
4542        }
4543
4544        trace_printk_seq(&iter.seq);
4545    }
4546
4547    if (!cnt)
4548        printk(KERN_TRACE " (ftrace buffer empty)\n");
4549    else
4550        printk(KERN_TRACE "---------------------------------\n");
4551
4552 out_enable:
4553    /* Re-enable tracing if requested */
4554    if (!disable_tracing) {
4555        trace_flags |= old_userobj;
4556
4557        for_each_tracing_cpu(cpu) {
4558            atomic_dec(&iter.tr->data[cpu]->disabled);
4559        }
4560        tracing_on();
4561    }
4562
4563 out:
4564    arch_spin_unlock(&ftrace_dump_lock);
4565    local_irq_restore(flags);
4566}
4567
4568/* By default: disable tracing after the dump */
4569void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
4570{
4571    __ftrace_dump(true, oops_dump_mode);
4572}
4573
4574__init static int tracer_alloc_buffers(void)
4575{
4576    int ring_buf_size;
4577    enum ring_buffer_flags rb_flags;
4578    int i;
4579    int ret = -ENOMEM;
4580
4581
4582    if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4583        goto out;
4584
4585    if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4586        goto out_free_buffer_mask;
4587
4588    /* To save memory, keep the ring buffer size to its minimum */
4589    if (ring_buffer_expanded)
4590        ring_buf_size = trace_buf_size;
4591    else
4592        ring_buf_size = 1;
4593
4594    rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
4595
4596    cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4597    cpumask_copy(tracing_cpumask, cpu_all_mask);
4598
4599    /* TODO: make the number of buffers hot pluggable with CPUS */
4600    global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
4601    if (!global_trace.buffer) {
4602        printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4603        WARN_ON(1);
4604        goto out_free_cpumask;
4605    }
4606    global_trace.entries = ring_buffer_size(global_trace.buffer);
4607
4608
4609#ifdef CONFIG_TRACER_MAX_TRACE
4610    max_tr.buffer = ring_buffer_alloc(1, rb_flags);
4611    if (!max_tr.buffer) {
4612        printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4613        WARN_ON(1);
4614        ring_buffer_free(global_trace.buffer);
4615        goto out_free_cpumask;
4616    }
4617    max_tr.entries = 1;
4618#endif
4619
4620    /* Allocate the first page for all buffers */
4621    for_each_tracing_cpu(i) {
4622        global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4623        max_tr.data[i] = &per_cpu(max_tr_data, i);
4624    }
4625
4626    trace_init_cmdlines();
4627
4628    register_tracer(&nop_trace);
4629    current_trace = &nop_trace;
4630    /* All seems OK, enable tracing */
4631    tracing_disabled = 0;
4632
4633    atomic_notifier_chain_register(&panic_notifier_list,
4634                       &trace_panic_notifier);
4635
4636    register_die_notifier(&trace_die_notifier);
4637
4638    return 0;
4639
4640out_free_cpumask:
4641    free_cpumask_var(tracing_cpumask);
4642out_free_buffer_mask:
4643    free_cpumask_var(tracing_buffer_mask);
4644out:
4645    return ret;
4646}
4647
4648__init static int clear_boot_tracer(void)
4649{
4650    /*
4651     * The default tracer at boot buffer is an init section.
4652     * This function is called in lateinit. If we did not
4653     * find the boot tracer, then clear it out, to prevent
4654     * later registration from accessing the buffer that is
4655     * about to be freed.
4656     */
4657    if (!default_bootup_tracer)
4658        return 0;
4659
4660    printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4661           default_bootup_tracer);
4662    default_bootup_tracer = NULL;
4663
4664    return 0;
4665}
4666
4667early_initcall(tracer_alloc_buffers);
4668fs_initcall(tracer_init_debugfs);
4669late_initcall(clear_boot_tracer);
4670

Archive Download this file



interactive