Root/target/linux/ubicom32/files/arch/ubicom32/kernel/ldsr.c

1/*
2 * arch/ubicom32/kernel/ldsr.c
3 * Ubicom32 architecture Linux Device Services Driver Interface
4 *
5 * (C) Copyright 2009, Ubicom, Inc.
6 *
7 * This file is part of the Ubicom32 Linux Kernel Port.
8 *
9 * The Ubicom32 Linux Kernel Port is free software: you can redistribute
10 * it and/or modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation, either version 2 of the
12 * License, or (at your option) any later version.
13 *
14 * The Ubicom32 Linux Kernel Port is distributed in the hope that it
15 * will be useful, but WITHOUT ANY WARRANTY; without even the implied
16 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
17 * the GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with the Ubicom32 Linux Kernel Port. If not,
21 * see <http://www.gnu.org/licenses/>.
22 *
23 * Ubicom32 implementation derived from (with many thanks):
24 * arch/m68knommu
25 * arch/blackfin
26 * arch/parisc
27 *
28 * NOTES:
29 *
30 * The LDSR is a programmable interrupt controller that is written in software.
31 * It emulates the behavior of an pic by fielding the interrupts, choosing a
32 * victim thread to take the interrupt and forcing that thread to take a context
33 * switch to the appropriate interrupt handler.
34 *
35 * Because traps are treated as just a special class of interrupts, the LDSR
36 * also handles the processing of traps.
37 *
38 * Because we compile Linux both UP and SMP, we need the LDSR to use
39 * architectural locking that is not "compiled out" when compiling UP. For now,
40 * we use the single atomic bit lock.
41 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/sched.h>
45#include <linux/interrupt.h>
46#include <linux/irq.h>
47#include <linux/profile.h>
48#include <linux/clocksource.h>
49#include <linux/types.h>
50#include <linux/module.h>
51#include <linux/cpumask.h>
52#include <linux/bug.h>
53#include <linux/delay.h>
54#include <asm/ip5000.h>
55#include <asm/atomic.h>
56#include <asm/machdep.h>
57#include <asm/asm-offsets.h>
58#include <asm/traps.h>
59#include <asm/thread.h>
60#include <asm/range-protect.h>
61
62/*
63 * One can not print from the LDSR so the best we can do is
64 * check a condition and stall all of the threads.
65 */
66
67// #define DEBUG_LDSR 1
68#if defined(DEBUG_LDSR)
69#define DEBUG_ASSERT(cond) \
70    if (!(cond)) { \
71        THREAD_STALL; \
72    }
73#else
74#define DEBUG_ASSERT(cond)
75#endif
76
77/*
78 * Make global so that we can use it in the RFI code in assembly.
79 */
80unsigned int ldsr_soft_irq_mask;
81EXPORT_SYMBOL(ldsr_soft_irq_mask);
82
83static unsigned int ldsr_suspend_mask;
84static unsigned int ldsr_soft_irq;
85static unsigned int ldsr_stack_space[1024];
86
87static struct ldsr_register_bank {
88    volatile unsigned int enabled0;
89    volatile unsigned int enabled1;
90    volatile unsigned int mask0;
91    volatile unsigned int mask1;
92    unsigned int total;
93    unsigned int retry;
94    unsigned int backout;
95} ldsr_interrupt;
96
97/*
98 * Which thread/cpu are we?
99 */
100static int ldsr_tid = -1;
101
102#if defined(CONFIG_IRQSTACKS)
103/*
104 * per-CPU IRQ stacks (thread information and stack)
105 *
106 * NOTE: Do not use DEFINE_PER_CPU() as it makes it harder
107 * to find the location of ctx from assembly language.
108 */
109union irq_ctx {
110    struct thread_info tinfo;
111    u32 stack[THREAD_SIZE/sizeof(u32)];
112};
113static union irq_ctx *percpu_irq_ctxs[NR_CPUS];
114
115/*
116 * Storage for the interrupt stack.
117 */
118#if !defined(CONFIG_IRQSTACKS_USEOCM)
119static char percpu_irq_stacks[(NR_CPUS * THREAD_SIZE) + (THREAD_SIZE - 1)];
120#else
121/*
122 * For OCM, the linker will ensure that space is allocated for the stack
123 * see (vmlinux.lds.S)
124 */
125static char percpu_irq_stacks[];
126#endif
127
128#endif
129
130/*
131 * Save trap IRQ because we need to un-suspend if it gets set.
132 */
133static unsigned int ldsr_trap_irq_mask;
134static unsigned int ldsr_trap_irq;
135
136/*
137 * ret_from_interrupt_to_kernel
138 * Just restore the context and do nothing else.
139 */
140asmlinkage void ret_from_interrupt_to_kernel(void)__attribute__((naked));
141
142/*
143 * ret_from_interrupt_to_user
144 * Call scheduler if needed. Just restore the context.
145 */
146asmlinkage void ret_from_interrupt_to_user(void)__attribute__((naked));
147
148#ifdef DEBUG_LDSR
149u32_t old_sp, old_pc, old_a0, old_a5, old_a3;
150struct pt_regs copy_regs, *copy_save_area;
151#endif
152
153int __user_mode(unsigned long sp)
154{
155
156    u32_t saved_stack_base = sp & ~(ASM_THREAD_SIZE - 1);
157#if defined(CONFIG_IRQSTACKS_USEOCM)
158    if ((union irq_ctx *)saved_stack_base == percpu_irq_ctxs[smp_processor_id()]) {
159        /*
160         * On the interrupt stack.
161         */
162        return 0;
163    }
164#endif
165
166    if (!(u32_t)current) {
167        return 0;
168    }
169    return saved_stack_base != ((u32_t)current->stack);
170}
171
172/*
173 * ldsr_lock_release()
174 * Release the LDSR lock.
175 */
176static void ldsr_lock_release(void)
177{
178    UBICOM32_UNLOCK(LDSR_LOCK_BIT);
179}
180
181/*
182 * ldsr_lock_acquire()
183 * Acquire the LDSR lock, spin if not available.
184 */
185static void ldsr_lock_acquire(void)
186{
187    UBICOM32_LOCK(LDSR_LOCK_BIT);
188}
189
190/*
191 * ldsr_thread_irq_disable()
192 * Disable interrupts for the specified thread.
193 */
194static void ldsr_thread_irq_disable(unsigned int tid)
195{
196    unsigned int mask = (1 << tid);
197
198    asm volatile (
199    " or.4 scratchpad1, scratchpad1, %0 \n\t"
200        :
201        : "d"(mask)
202        : "cc"
203    );
204}
205
206/*
207 * ldsr_thread_get_interrupts()
208 * Get the interrupt state for all threads.
209 */
210static unsigned long ldsr_thread_get_interrupts(void)
211{
212    unsigned long ret = 0;
213    asm volatile (
214    " move.4 %0, scratchpad1 \n\t"
215        : "=r" (ret)
216        :
217    );
218    return ret;
219}
220
221/*
222 * ldsr_emulate_and_run()
223 * Emulate the instruction and then set the thread to run.
224 */
225static void ldsr_emulate_and_run(unsigned int tid)
226{
227    unsigned int thread_mask = (1 << tid);
228    u32_t write_csr = (tid << 15) | (1 << 14);
229
230    /*
231     * Emulate the unaligned access.
232     */
233    unaligned_emulate(tid);
234
235    /*
236     * Get the thread back in a running state.
237     */
238    asm volatile (
239    " setcsr %0 \n\t"
240    " setcsr_flush 0 \n\t"
241    " move.4 trap_cause, #0 \n\t" /* Clear the trap cause
242                               * register */
243    " setcsr #0 \n\t"
244    " setcsr_flush 0 \n\t"
245    " move.4 mt_dbg_active_set, %1 \n\t" /* Activate thread even if
246                               * in dbg/fault state */
247    " move.4 mt_active_set, %1 \n\t" /* Restart target
248                               * thread. */
249        :
250        : "r" (write_csr), "d" (thread_mask)
251        : "cc"
252    );
253    thread_enable_mask(thread_mask);
254}
255
256/*
257 * ldsr_preemptive_context_save()
258 * save thread context from another hardware thread. The other thread must
259 * be stalled.
260 */
261static inline void ldsr_preemptive_context_save(u32_t thread,
262                        struct pt_regs *regs)
263{
264    /*
265     * Save the current state of the specified thread
266     */
267    asm volatile (
268    " move.4 a3, %0 \n\t"
269
270        /* set src1 from the target thread */
271    " move.4 csr, %1 \n\t"
272    " setcsr_flush 0 \n\t"
273    " setcsr_flush 0 \n\t"
274
275        /* copy state from the other thread */
276    " move.4 "D(PT_D0)"(a3), d0 \n\t"
277    " move.4 "D(PT_D1)"(a3), d1 \n\t"
278    " move.4 "D(PT_D2)"(a3), d2 \n\t"
279    " move.4 "D(PT_D3)"(a3), d3 \n\t"
280    " move.4 "D(PT_D4)"(a3), d4 \n\t"
281    " move.4 "D(PT_D5)"(a3), d5 \n\t"
282    " move.4 "D(PT_D6)"(a3), d6 \n\t"
283    " move.4 "D(PT_D7)"(a3), d7 \n\t"
284    " move.4 "D(PT_D8)"(a3), d8 \n\t"
285    " move.4 "D(PT_D9)"(a3), d9 \n\t"
286    " move.4 "D(PT_D10)"(a3), d10 \n\t"
287    " move.4 "D(PT_D11)"(a3), d11 \n\t"
288    " move.4 "D(PT_D12)"(a3), d12 \n\t"
289    " move.4 "D(PT_D13)"(a3), d13 \n\t"
290    " move.4 "D(PT_D14)"(a3), d14 \n\t"
291    " move.4 "D(PT_D15)"(a3), d15 \n\t"
292    " move.4 "D(PT_A0)"(a3), a0 \n\t"
293    " move.4 "D(PT_A1)"(a3), a1 \n\t"
294    " move.4 "D(PT_A2)"(a3), a2 \n\t"
295    " move.4 "D(PT_A3)"(a3), a3 \n\t"
296    " move.4 "D(PT_A4)"(a3), a4 \n\t"
297    " move.4 "D(PT_A5)"(a3), a5 \n\t"
298    " move.4 "D(PT_A6)"(a3), a6 \n\t"
299    " move.4 "D(PT_SP)"(a3), a7 \n\t"
300    " move.4 "D(PT_ACC0HI)"(a3), acc0_hi \n\t"
301    " move.4 "D(PT_ACC0LO)"(a3), acc0_lo \n\t"
302    " move.4 "D(PT_MAC_RC16)"(a3), mac_rc16 \n\t"
303    " move.4 "D(PT_ACC1HI)"(a3), acc1_hi \n\t"
304    " move.4 "D(PT_ACC1LO)"(a3), acc1_lo \n\t"
305    " move.4 "D(PT_SOURCE3)"(a3), source3 \n\t"
306    " move.4 "D(PT_INST_CNT)"(a3), inst_cnt \n\t"
307    " move.4 "D(PT_CSR)"(a3), csr \n\t"
308    " move.4 "D(PT_DUMMY_UNUSED)"(a3), #0 \n\t"
309    " move.4 "D(PT_INT_MASK0)"(a3), int_mask0 \n\t"
310    " move.4 "D(PT_INT_MASK1)"(a3), int_mask1 \n\t"
311    " move.4 "D(PT_TRAP_CAUSE)"(a3), trap_cause \n\t"
312    " move.4 "D(PT_PC)"(a3), pc \n\t"
313    " move.4 "D(PT_PREVIOUS_PC)"(a3), previous_pc \n\t"
314        /* disable csr thread select */
315    " movei csr, #0 \n\t"
316    " setcsr_flush 0 \n\t"
317    :
318    : "r" (regs->dn), "d" ((thread << 9) | (1 << 8))
319    : "a3"
320    );
321}
322
323/*
324 * ldsr_rotate_threads()
325 * Simple round robin algorithm for choosing the next cpu
326 */
327static int ldsr_rotate_threads(unsigned long cpus)
328{
329    static unsigned char ldsr_bits[8] = {
330        3, 0, 1, 0, 2, 0, 1, 0
331    };
332
333    static int nextbit;
334    int thisbit;
335
336    /*
337     * Move the interrupts down so that we consider interrupts from where
338     * we left off, then take the interrupts we would lose and move them
339     * to the top half of the interrupts value.
340     */
341    cpus = (cpus >> nextbit) | (cpus << ((sizeof(cpus) * 8) - nextbit));
342
343    /*
344     * 50% of the time we won't take this at all and then of the cases where
345     * we do about 50% of those we only execute once.
346     */
347    if (!(cpus & 0xffff)) {
348        nextbit += 16;
349        cpus >>= 16;
350    }
351
352    if (!(cpus & 0xff)) {
353        nextbit += 8;
354        cpus >>= 8;
355    }
356
357    if (!(cpus & 0xf)) {
358        nextbit += 4;
359        cpus >>= 4;
360    }
361
362    nextbit += ldsr_bits[cpus & 0x7];
363    thisbit = (nextbit & ((sizeof(cpus) * 8) - 1));
364    nextbit = (thisbit + 1) & ((sizeof(cpus) * 8) - 1);
365    DEBUG_ASSERT(thisbit < THREAD_ARCHITECTURAL_MAX);
366    return thisbit;
367}
368
369/*
370 * ldsr_rotate_interrupts()
371 * Get rotating next set bit value.
372 */
373static int ldsr_rotate_interrupts(unsigned long long interrupts)
374{
375    static unsigned char ldsr_bits[8] = {
376        3, 0, 1, 0, 2, 0, 1, 0
377    };
378
379    static int nextbit;
380    int thisbit;
381
382    /*
383     * Move the interrupts down so that we consider interrupts from where
384     * we left off, then take the interrupts we would lose and move them
385     * to the top half of the interrupts value.
386     */
387    interrupts = (interrupts >> nextbit) |
388        (interrupts << ((sizeof(interrupts) * 8) - nextbit));
389
390    /*
391     * 50% of the time we won't take this at all and then of the cases where
392     * we do about 50% of those we only execute once.
393     */
394    if (!(interrupts & 0xffffffff)) {
395        nextbit += 32;
396        interrupts >>= 32;
397    }
398
399    if (!(interrupts & 0xffff)) {
400        nextbit += 16;
401        interrupts >>= 16;
402    }
403
404    if (!(interrupts & 0xff)) {
405        nextbit += 8;
406        interrupts >>= 8;
407    }
408
409    if (!(interrupts & 0xf)) {
410        nextbit += 4;
411        interrupts >>= 4;
412    }
413
414    nextbit += ldsr_bits[interrupts & 0x7];
415    thisbit = (nextbit & ((sizeof(interrupts) * 8) - 1));
416    nextbit = (thisbit + 1) & ((sizeof(interrupts) * 8) - 1);
417
418    DEBUG_ASSERT(thisbit < (sizeof(interrupts) * 8));
419    return thisbit;
420}
421
422/*
423 * ldsr_backout_or_irq()
424 *
425 * One way or the other this interrupt is not being
426 * processed, make sure that it is reset. We are
427 * not going to call irq_end_vector() so unmask the
428 * interrupt.
429 */
430static void ldsr_backout_of_irq(int vector, unsigned long tid_mask)
431{
432#if defined(CONFIG_SMP)
433    if (unlikely(vector == smp_ipi_irq)) {
434        smp_reset_ipi(tid_mask);
435    }
436#endif
437    ldsr_unmask_vector(vector);
438    ldsr_interrupt.backout++;
439}
440
441#if defined(CONFIG_IRQSTACKS)
442/*
443 * ldsr_choose_savearea_and_returnvec()
444 * Test our current state (user, kernel, interrupt) and set things up.
445 *
446 * This version of the function uses 3 stacks and nests interrupts
447 * on the interrupt stack.
448 */
449static struct pt_regs *ldsr_choose_savearea_and_returnvec(thread_t tid, u32_t linux_sp, u32_t *pvec)
450{
451    struct pt_regs *save_area;
452    u32_t masked_linux_sp = linux_sp & ~(THREAD_SIZE - 1);
453    struct thread_info * ti= (struct thread_info *)sw_ksp[tid];
454
455#if defined(CONFIG_SMP)
456    union irq_ctx *icp = percpu_irq_ctxs[tid];
457#else
458    union irq_ctx *icp = percpu_irq_ctxs[0];
459#endif
460
461    if (masked_linux_sp == (u32_t)icp) {
462        /*
463         * Fault/Interrupt occurred while on the interrupt stack.
464         */
465        save_area = (struct pt_regs *)((char *)linux_sp - sizeof(struct pt_regs) - 8);
466        *pvec = (u32_t)(&ret_from_interrupt_to_kernel);
467    } else {
468        /*
469         * Fault/Interrupt occurred while on user/kernel stack. This is a new
470         * first use of the interrupt stack.
471         */
472        save_area = (struct pt_regs *) ((char *)icp + sizeof(icp->stack) - sizeof(struct pt_regs) - 8);
473        if (masked_linux_sp == (u32_t)ti) {
474            *pvec = (u32_t)(&ret_from_interrupt_to_kernel);
475        } else {
476            *pvec = (u32_t)(&ret_from_interrupt_to_user);
477        }
478
479        /*
480         * Because the softirq code will execute on the "interrupt" stack, we
481         * need to maintain the knowledge of what "task" was executing on the
482         * cpu. This is done by copying the thread_info->task from the cpu
483         * we are about to context switch into the interrupt contexts thread_info
484         * structure.
485         */
486        icp->tinfo.task = ti->task;
487        icp->tinfo.preempt_count =
488                (icp->tinfo.preempt_count & ~SOFTIRQ_MASK) |
489                (ti->preempt_count & SOFTIRQ_MASK);
490        icp->tinfo.interrupt_nesting = 0;
491    }
492    save_area->nesting_level = icp->tinfo.interrupt_nesting;
493    return save_area;
494}
495
496#else
497/*
498 * ldsr_choose_savearea_and_returnvec()
499 * Test our current state (user, kernel, interrupt) and set things up.
500 *
501 * The version of the function uses just the user & kernel stack and
502 * nests interrupts on the existing kernel stack.
503 */
504static struct pt_regs *ldsr_choose_savearea_and_returnvec(thread_t tid, u32_t linux_sp, u32_t *pvec)
505{
506    struct pt_regs *save_area;
507    u32_t masked_linux_sp = linux_sp & ~(THREAD_SIZE - 1);
508    struct thread_info *ti = (struct thread_info *)sw_ksp[tid];
509
510    if (masked_linux_sp == (u32_t)ti) {
511        /*
512         * Fault/Interrupt occurred while on the kernel stack.
513         */
514        save_area = (struct pt_regs *)((char *)linux_sp - sizeof(struct pt_regs) - 8);
515        *pvec = (u32_t) (&ret_from_interrupt_to_kernel);
516    } else {
517        /*
518         * Fault/Interrupt occurred while on user stack.
519         */
520        ti->interrupt_nesting = 0;
521        save_area = (struct pt_regs *)((u32_t)ti + THREAD_SIZE - sizeof(struct pt_regs) - 8);
522        *pvec = (u32_t) (&ret_from_interrupt_to_user);
523    }
524    save_area->nesting_level = ti->interrupt_nesting;
525    return save_area;
526}
527#endif
528
529/*
530 * ldsr_ctxsw_thread()
531 * Context switch a mainline thread to execute do_IRQ() for the specified
532 * vector.
533 */
534static void ldsr_ctxsw_thread(int vector, thread_t tid)
535{
536    u32_t linux_sp;
537    u32_t return_vector;
538    struct pt_regs *save_area, *regs;
539    u32_t thread_mask = (1 << tid);
540    u32_t read_csr = ((tid << 9) | (1 << 8));
541    u32_t write_csr = (tid << 15) | (1 << 14);
542    u32_t interrupt_vector = (u32_t)(&do_IRQ);
543
544    unsigned int frame_type = UBICOM32_FRAME_TYPE_INTERRUPT;
545
546
547    DEBUG_ASSERT(!thread_is_enabled(tid));
548
549    /*
550     * Acquire the necessary global and per thread locks for tid.
551     * As a side effect, we ensure that the thread has not trapped
552     * and return true if it has.
553     */
554    if (unlikely(thread_is_trapped(tid))) {
555        /*
556         * Read the trap cause, the sp and clear the MT_TRAP bits.
557         */
558        unsigned int cause;
559        asm volatile (
560        " setcsr %3 \n\t"
561        " setcsr_flush 0 \n\t"
562        " setcsr_flush 0 \n\t"
563        " move.4 %0, TRAP_CAUSE \n\t"
564        " move.4 %1, SP \n\t"
565        " setcsr #0 \n\t"
566        " setcsr_flush 0 \n\t"
567        " move.4 MT_BREAK_CLR, %2\n\t"
568        " move.4 MT_TRAP_CLR, %2 \n\t"
569            : "=&r" (cause), "=&r" (linux_sp)
570            : "r" (thread_mask), "m" (read_csr)
571        );
572
573        ldsr_backout_of_irq(vector, (1 << tid));
574
575#if !defined(CONFIG_UNALIGNED_ACCESS_DISABLED)
576        /*
577         * See if the unaligned trap handler can deal with this.
578         * If so, emulate the instruction and then just restart
579         * the thread.
580         */
581        if (unaligned_only(cause)) {
582#if defined(CONFIG_UNALIGNED_ACCESS_USERSPACE_ONLY)
583            /*
584             * Check if this is a kernel stack if so we will not
585             * handle the trap
586             */
587            u32_t masked_linux_sp = linux_sp & ~(THREAD_SIZE - 1);
588            if ((masked_linux_sp != (u32_t)sw_ksp[tid]) &&
589                unaligned_only(cause)) {
590                ldsr_emulate_and_run(tid);
591                return;
592            }
593#else
594            ldsr_emulate_and_run(tid);
595            return;
596#endif
597
598        }
599#endif
600
601        interrupt_vector = (u32_t)(&trap_handler);
602        frame_type = UBICOM32_FRAME_TYPE_TRAP;
603    } else {
604        /*
605         * Read the target thread's SP
606         */
607        asm volatile (
608        " setcsr %1 \n\t"
609        " setcsr_flush 0 \n\t"
610        " setcsr_flush 0 \n\t"
611        " move.4 %0, SP \n\t"
612        " setcsr #0 \n\t"
613        " setcsr_flush 0 \n\t"
614            : "=m" (linux_sp)
615            : "m" (read_csr)
616        );
617    }
618
619    /*
620     * We are delivering an interrupt, count it.
621     */
622    ldsr_interrupt.total++;
623
624    /*
625     * At this point, we will definitely force this thread to
626     * a new context, show its interrupts as disabled.
627     */
628    ldsr_thread_irq_disable(tid);
629
630    /*
631     * Test our current state (user, kernel, interrupt). Save the
632     * appropriate data and setup for the return.
633     */
634    save_area = ldsr_choose_savearea_and_returnvec(tid, linux_sp, &return_vector);
635
636    /*
637     * The pt_regs (save_area) contains the type of thread that we are dealing
638     * with (KERNEL/NORMAL) and is copied into each pt_regs area. We get this
639     * from the current tasks kernel pt_regs area that always exists at the
640     * top of the kernel stack.
641     */
642    regs = (struct pt_regs *)((u32_t)sw_ksp[tid] + THREAD_SIZE - sizeof(struct pt_regs) - 8);
643    save_area->thread_type = regs->thread_type;
644
645    /*
646     * Preserve the context of the Linux thread.
647     */
648    ldsr_preemptive_context_save(tid, save_area);
649
650    /*
651     * Load the fram_type into the save_area.
652     */
653    save_area->frame_type = frame_type;
654
655#ifdef CONFIG_STOP_ON_TRAP
656    /*
657     * Before we get backtrace and showing stacks working well, it sometimes
658     * helps to enter the debugger when a trap occurs before we change the
659     * thread to handle the fault. This optional code causes all threads to
660     * stop on every trap frame. One assumes that GDB connected via the
661     * mailbox interface will be used to recover from this state.
662     */
663    if (frame_type == UBICOM32_FRAME_TYPE_TRAP) {
664        THREAD_STALL;
665    }
666#endif
667
668#ifdef DEBUG_LDSR
669    copy_regs = *save_area;
670    copy_save_area = save_area;
671
672    old_a0 = save_area->an[0];
673    old_a3 = save_area->an[3];
674    old_sp = save_area->an[7];
675    old_a5 = save_area->an[5];
676    old_pc = save_area->pc;
677#endif
678
679    /*
680     * Now we have to switch the kernel thread to run do_IRQ function.
681     * Set pc to do_IRQ
682     * Set d0 to vector
683     * Set d1 to save_area.
684     * Set a5 to the proper return vector.
685     */
686    asm volatile (
687    " setcsr %0 \n\t"
688    " setcsr_flush 0 \n\t"
689    " move.4 d0, %5 \n\t" /* d0 = 0 vector # */
690    " move.4 d1, %1 \n\t" /* d1 = save_area */
691    " move.4 sp, %1 \n\t" /* sp = save_area */
692    " move.4 a5, %2 \n\t" /* a5 = return_vector */
693    " move.4 pc, %3 \n\t" /* pc = do_IRQ routine. */
694    " move.4 trap_cause, #0 \n\t" /* Clear the trap cause
695                               * register */
696    " setcsr #0 \n\t"
697    " setcsr_flush 0 \n\t"
698    " enable_kernel_ranges %4 \n\t"
699    " move.4 mt_dbg_active_set, %4 \n\t" /* Activate thread even if
700                               * in dbg/fault state */
701    " move.4 mt_active_set, %4 \n\t" /* Restart target
702                               * thread. */
703        :
704        : "r" (write_csr), "r" (save_area),
705          "r" (return_vector), "r" (interrupt_vector),
706          "d" (thread_mask), "r" (vector)
707        : "cc"
708    );
709    thread_enable_mask(thread_mask);
710}
711
712/*
713 * ldsr_deliver_interrupt()
714 * Deliver the interrupt to one of the threads or all of the threads.
715 */
716static void ldsr_deliver_interrupt(int vector,
717                   unsigned long deliver_to,
718                   int all)
719{
720    unsigned long disabled_threads;
721    unsigned long possible_threads;
722    unsigned long trapped_threads;
723    unsigned long global_locks;
724
725    /*
726     * Disable all of the threads that we might want to send
727     * this interrupt to.
728     */
729retry:
730    DEBUG_ASSERT(deliver_to);
731    thread_disable_mask(deliver_to);
732
733    /*
734     * If any threads are in the trap state, we have to service the
735     * trap for those threads first.
736     */
737    asm volatile (
738        "move.4 %0, MT_TRAP \n\t"
739        : "=r" (trapped_threads)
740        :
741    );
742
743    trapped_threads &= deliver_to;
744    if (unlikely(trapped_threads)) {
745        /*
746         * all traps will be handled, so clear the trap bit before restarting any threads
747         */
748            ubicom32_clear_interrupt(ldsr_trap_irq);
749
750        /*
751         * Let the remaining untrapped threads, continue.
752         */
753        deliver_to &= ~trapped_threads;
754        if (deliver_to) {
755            thread_enable_mask(deliver_to);
756        }
757
758        /*
759         * For the trapped threads force them to handle
760         * a trap.
761         */
762        while (trapped_threads) {
763            unsigned long which = ffz(~trapped_threads);
764            trapped_threads &= ~(1 << which);
765            ldsr_ctxsw_thread(vector, which);
766        }
767        return;
768    }
769
770    /*
771     * Can we deliver an interrupt to any of the threads?
772     */
773    disabled_threads = ldsr_thread_get_interrupts();
774    possible_threads = deliver_to & ~disabled_threads;
775    if (unlikely(!possible_threads)) {
776#if defined(CONFIG_SMP)
777        /*
778         * In the SMP case, we can not wait because 1 cpu might be
779         * sending an IPI to another cpu which is currently blocked.
780         * The only way to ensure IPI delivery is to backout and
781         * keep trying. For SMP, we don't sleep until the interrupts
782         * are delivered.
783         */
784        thread_enable_mask(deliver_to);
785        ldsr_backout_of_irq(vector, deliver_to);
786        return;
787#else
788        /*
789         * In the UP case, we have nothing to do so we should wait.
790         *
791         * Since the INT_MASK0 and INT_MASK1 are "re-loaded" before we
792         * suspend in the outer loop, we do not need to save them here.
793         *
794         * We test that we were awakened for our specific interrupts
795         * because the ldsr mask/unmask operations will force the ldsr
796         * awake even if the interrupt on the mainline thread is not
797         * completed.
798         */
799        unsigned int scratch = 0;
800        thread_enable_mask(deliver_to);
801        asm volatile (
802        " move.4 INT_MASK0, %1 \n\t"
803        " move.4 INT_MASK1, #0 \n\t"
804
805        "1: suspend \n\t"
806        " move.4 %0, INT_STAT0 \n\t"
807        " and.4 %0, %0, %1 \n\t"
808        " jmpeq.f 1b \n\t"
809
810        " move.4 INT_CLR0, %2 \n\t"
811            : "+r" (scratch)
812            : "d" (ldsr_suspend_mask), "r" (ldsr_soft_irq_mask)
813            : "cc"
814        );
815
816        /*
817         * This delay is sized to coincide with the time it takes a
818         * thread to complete the exit (see return_from_interrupt).
819         */
820        ldsr_interrupt.retry++;
821        __delay(10);
822        goto retry;
823#endif
824    }
825
826    /*
827     * If any of the global locks are held, we can not deliver any
828     * interrupts, we spin delay(10) and then try again. If our
829     * spinning becomes a bottle neck, we will need to suspend but for
830     * now lets just spin.
831     */
832    asm volatile (
833        "move.4 %0, scratchpad1 \n\t"
834        : "=r" (global_locks)
835        :
836    );
837    if (unlikely(global_locks & 0xffff0000)) {
838        thread_enable_mask(deliver_to);
839
840        /*
841         * This delay is sized to coincide with the average time it
842         * takes a thread to release a global lock.
843         */
844        ldsr_interrupt.retry++;
845        __delay(10);
846        goto retry;
847    }
848
849    /*
850     * Deliver to one cpu.
851     */
852    if (!all) {
853        /*
854         * Find our victim and then enable everyone else.
855         */
856        unsigned long victim = ldsr_rotate_threads(possible_threads);
857        DEBUG_ASSERT((deliver_to & (1 << victim)));
858        DEBUG_ASSERT((possible_threads & (1 << victim)));
859
860        deliver_to &= ~(1 << victim);
861        if (deliver_to) {
862            thread_enable_mask(deliver_to);
863        }
864        ldsr_ctxsw_thread(vector, victim);
865        return;
866    }
867
868    /*
869     * If we can't deliver to some threads, wake them
870     * back up and reset things to deliver to them.
871     */
872    deliver_to &= ~possible_threads;
873    if (unlikely(deliver_to)) {
874        thread_enable_mask(deliver_to);
875        ldsr_backout_of_irq(vector, deliver_to);
876    }
877
878    /*
879     * Deliver to all possible threads(s).
880     */
881    while (possible_threads) {
882        unsigned long victim = ffz(~possible_threads);
883        possible_threads &= ~(1 << victim);
884        ldsr_ctxsw_thread(vector, victim);
885    }
886}
887
888/*
889 * ldsr_thread()
890 * This thread acts as the interrupt controller for Linux.
891 */
892static void ldsr_thread(void *arg)
893{
894    int stat0;
895    int stat1;
896    int interrupt0;
897    int interrupt1;
898    long long interrupts;
899    unsigned long cpus;
900
901#if !defined(CONFIG_SMP)
902    /*
903     * In a non-smp configuration, we can not use the cpu(s) arrays because
904     * there is not a 1-1 correspondence between cpus(s) and our threads.
905     * Thus we must get a local idea of the mainline threads and use the
906     * one and only 1 set as the victim. We do this once before the ldsr
907     * loop.
908     *
909     * In the SMP case, we will use the cpu(s) map to determine which cpu(s)
910     * are valid to send interrupts to.
911     */
912    int victim = 0;
913    unsigned int mainline = thread_get_mainline();
914    if (mainline == 0) {
915        panic("no mainline Linux threads to interrupt");
916        return;
917    }
918    victim = ffz(~mainline);
919    cpus = (1 << victim);
920#endif
921
922    while (1) {
923        /*
924         * If one changes this code not to reload the INT_MASK(s), you
925         * need to know that code in the lock waiting above does not
926         * reset the MASK registers back; so that code will need to be
927         * changed.
928         */
929        ldsr_lock_acquire();
930        asm volatile (
931        " move.4 INT_MASK0, %0 \n\t"
932        " move.4 INT_MASK1, %1 \n\t"
933            :
934            : "U4" (ldsr_interrupt.mask0), "U4" (ldsr_interrupt.mask1)
935        );
936        ldsr_lock_release();
937        thread_suspend();
938
939        /*
940         * Read the interrupt status registers
941         */
942        asm volatile (
943            "move.4 %0, INT_STAT0 \n\t"
944            "move.4 %1, INT_STAT1 \n\t"
945            : "=r" (stat0), "=r" (stat1)
946            :
947        );
948
949        /*
950         * We only care about interrupts that we have been told to care
951         * about. The interrupt must be enabled, unmasked, and have
952         * occurred in the hardware.
953         */
954        ldsr_lock_acquire();
955        interrupt0 = ldsr_interrupt.enabled0 &
956            ldsr_interrupt.mask0 & stat0;
957        interrupt1 = ldsr_interrupt.enabled1 &
958            ldsr_interrupt.mask1 & stat1;
959        ldsr_lock_release();
960
961        /*
962         * For each interrupt in the "snapshot" we will mask the
963         * interrupt handle the interrupt (typically calling do_IRQ()).
964         *
965         * The interrupt is unmasked by desc->chip->end() function in
966         * the per chip generic interrupt handling code
967         * (arch/ubicom32/kernel/irq.c).8
968         */
969        interrupts = ((unsigned long long)interrupt1 << 32) |
970            interrupt0;
971        while (interrupts) {
972            int all = 0;
973            int vector = ldsr_rotate_interrupts(interrupts);
974            interrupts &= ~((unsigned long long)1 << vector);
975
976            /*
977             * Now mask off this vector so that the LDSR ignores
978             * it until it is acknowledged.
979             */
980            ldsr_mask_vector(vector);
981#if !defined(CONFIG_SMP)
982            ldsr_deliver_interrupt(vector, cpus, all);
983#else
984            cpus = smp_get_affinity(vector, &all);
985            if (!cpus) {
986                /*
987                 * No CPU to deliver to so just leave
988                 * the interrupt unmasked and increase
989                 * the backout count. We will eventually
990                 * return and deliver it again.
991                 */
992                ldsr_unmask_vector(vector);
993                ldsr_interrupt.backout++;
994                continue;
995            }
996            ldsr_deliver_interrupt(vector, cpus, all);
997#endif
998        }
999    }
1000
1001    /* NOTREACHED */
1002}
1003
1004/*
1005 * ldsr_mask_vector()
1006 * Temporarily mask the interrupt vector, turn off the bit in the mask
1007 * register.
1008 */
1009void ldsr_mask_vector(unsigned int vector)
1010{
1011    unsigned int mask;
1012    if (vector < 32) {
1013        mask = ~(1 << vector);
1014        ldsr_lock_acquire();
1015        ldsr_interrupt.mask0 &= mask;
1016        ldsr_lock_release();
1017        thread_resume(ldsr_tid);
1018        return;
1019    }
1020
1021    mask = ~(1 << (vector - 32));
1022    ldsr_lock_acquire();
1023    ldsr_interrupt.mask1 &= mask;
1024    ldsr_lock_release();
1025    thread_resume(ldsr_tid);
1026}
1027
1028/*
1029 * ldsr_unmask_vector()
1030 * Unmask the interrupt vector so that it can be used, turn on the bit in
1031 * the mask register.
1032 *
1033 * Because it is legal for the interrupt path to disable an interrupt,
1034 * the unmasking code must ensure that disabled interrupts are not
1035 * unmasked.
1036 */
1037void ldsr_unmask_vector(unsigned int vector)
1038{
1039    unsigned int mask;
1040    if (vector < 32) {
1041        mask = (1 << vector);
1042        ldsr_lock_acquire();
1043        ldsr_interrupt.mask0 |= (mask & ldsr_interrupt.enabled0);
1044        ldsr_lock_release();
1045        thread_resume(ldsr_tid);
1046        return;
1047    }
1048
1049    mask = (1 << (vector - 32));
1050    ldsr_lock_acquire();
1051    ldsr_interrupt.mask1 |= (mask & ldsr_interrupt.enabled1);
1052    ldsr_lock_release();
1053    thread_resume(ldsr_tid);
1054}
1055
1056/*
1057 * ldsr_enable_vector()
1058 * The LDSR implements an interrupt controller and has a local (to the
1059 * LDSR) copy of its interrupt mask.
1060 */
1061void ldsr_enable_vector(unsigned int vector)
1062{
1063    unsigned int mask;
1064    if (vector < 32) {
1065        mask = (1 << vector);
1066        ldsr_lock_acquire();
1067        ldsr_interrupt.enabled0 |= mask;
1068        ldsr_interrupt.mask0 |= mask;
1069        ldsr_lock_release();
1070        thread_resume(ldsr_tid);
1071        return;
1072    }
1073
1074    mask = (1 << (vector - 32));
1075    ldsr_lock_acquire();
1076    ldsr_interrupt.enabled1 |= mask;
1077    ldsr_interrupt.mask1 |= mask;
1078    ldsr_lock_release();
1079    thread_resume(ldsr_tid);
1080}
1081
1082/*
1083 * ldsr_disable_vector()
1084 * The LDSR implements an interrupt controller and has a local (to the
1085 * LDSR) copy of its interrupt mask.
1086 */
1087void ldsr_disable_vector(unsigned int vector)
1088{
1089    unsigned int mask;
1090
1091    if (vector < 32) {
1092        mask = ~(1 << vector);
1093        ldsr_lock_acquire();
1094        ldsr_interrupt.enabled0 &= mask;
1095        ldsr_interrupt.mask0 &= mask;
1096        ldsr_lock_release();
1097        thread_resume(ldsr_tid);
1098        return;
1099    }
1100
1101    mask = ~(1 << (vector - 32));
1102    ldsr_lock_acquire();
1103    ldsr_interrupt.enabled1 &= mask;
1104    ldsr_interrupt.mask1 &= mask;
1105    ldsr_lock_release();
1106    thread_resume(ldsr_tid);
1107}
1108
1109/*
1110 * ldsr_get_threadid()
1111 * Return the threadid of the LDSR thread.
1112 */
1113thread_t ldsr_get_threadid(void)
1114{
1115    return ldsr_tid;
1116}
1117
1118/*
1119 * ldsr_set_trap_irq()
1120 * Save away the trap Soft IRQ
1121 *
1122 * See the per thread lock suspend code above for an explination.
1123 */
1124void ldsr_set_trap_irq(unsigned int irq)
1125{
1126    ldsr_trap_irq = irq;
1127    ldsr_trap_irq_mask = (1 << irq);
1128    ldsr_suspend_mask |= ldsr_trap_irq_mask;
1129}
1130
1131/*
1132 * ldsr_init()
1133 * Initialize the LDSR (Interrupt Controller)
1134 */
1135void ldsr_init(void)
1136{
1137#if defined(CONFIG_IRQSTACKS)
1138    int i;
1139    union irq_ctx *icp;
1140#endif
1141
1142    void *stack_high = (void *)ldsr_stack_space;
1143    stack_high += sizeof(ldsr_stack_space);
1144    stack_high -= 8;
1145
1146
1147    /*
1148     * Obtain a soft IRQ to use
1149     */
1150    if (irq_soft_alloc(&ldsr_soft_irq) < 0) {
1151        panic("no software IRQ is available\n");
1152        return;
1153    }
1154    ldsr_soft_irq_mask |= (1 << ldsr_soft_irq);
1155    ldsr_suspend_mask |= ldsr_soft_irq_mask;
1156
1157    /*
1158     * Now allocate and start the LDSR thread.
1159     */
1160    ldsr_tid = thread_alloc();
1161    if (ldsr_tid < 0) {
1162        panic("no thread available to run LDSR");
1163        return;
1164    }
1165
1166#if defined(CONFIG_IRQSTACKS)
1167    /*
1168     * Initialize the per-cpu irq thread_info structure that
1169     * is at the top of each per-cpu irq stack.
1170     */
1171    icp = (union irq_ctx *)
1172        (((unsigned long)percpu_irq_stacks + (THREAD_SIZE - 1)) & ~(THREAD_SIZE - 1));
1173    for (i = 0; i < NR_CPUS; i++) {
1174        struct thread_info *ti = &(icp->tinfo);
1175        ti->task = NULL;
1176        ti->exec_domain = NULL;
1177        ti->cpu = i;
1178        ti->preempt_count = 0;
1179        ti->interrupt_nesting = 0;
1180        percpu_irq_ctxs[i] = icp++;
1181    }
1182#endif
1183    thread_start(ldsr_tid, ldsr_thread, NULL,
1184             stack_high, THREAD_TYPE_NORMAL);
1185}
1186

Archive Download this file

Branches:
backfire
history
master
release_2010-11-17
release_2010-12-14
release_2011-02-23
release_2011-05-28
release_2011-08-27
release_2011-11-13
release_2012-03-18
release_2012-04-09
release_2012-10-11
uclibc_nptl



interactive