Root/
1 | /* |
2 | * ultra.S: Don't expand these all over the place... |
3 | * |
4 | * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) |
5 | */ |
6 | |
7 | #include <asm/asi.h> |
8 | #include <asm/pgtable.h> |
9 | #include <asm/page.h> |
10 | #include <asm/spitfire.h> |
11 | #include <asm/mmu_context.h> |
12 | #include <asm/mmu.h> |
13 | #include <asm/pil.h> |
14 | #include <asm/head.h> |
15 | #include <asm/thread_info.h> |
16 | #include <asm/cacheflush.h> |
17 | #include <asm/hypervisor.h> |
18 | #include <asm/cpudata.h> |
19 | |
20 | /* Basically, most of the Spitfire vs. Cheetah madness |
21 | * has to do with the fact that Cheetah does not support |
22 | * IMMU flushes out of the secondary context. Someone needs |
23 | * to throw a south lake birthday party for the folks |
24 | * in Microelectronics who refused to fix this shit. |
25 | */ |
26 | |
27 | /* This file is meant to be read efficiently by the CPU, not humans. |
28 | * Staraj sie tego nikomu nie pierdolnac... |
29 | */ |
30 | .text |
31 | .align 32 |
32 | .globl __flush_tlb_mm |
33 | __flush_tlb_mm: /* 18 insns */ |
34 | /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ |
35 | ldxa [%o1] ASI_DMMU, %g2 |
36 | cmp %g2, %o0 |
37 | bne,pn %icc, __spitfire_flush_tlb_mm_slow |
38 | mov 0x50, %g3 |
39 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
40 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
41 | sethi %hi(KERNBASE), %g3 |
42 | flush %g3 |
43 | retl |
44 | nop |
45 | nop |
46 | nop |
47 | nop |
48 | nop |
49 | nop |
50 | nop |
51 | nop |
52 | nop |
53 | nop |
54 | |
55 | .align 32 |
56 | .globl __flush_tlb_pending |
57 | __flush_tlb_pending: /* 26 insns */ |
58 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
59 | rdpr %pstate, %g7 |
60 | sllx %o1, 3, %o1 |
61 | andn %g7, PSTATE_IE, %g2 |
62 | wrpr %g2, %pstate |
63 | mov SECONDARY_CONTEXT, %o4 |
64 | ldxa [%o4] ASI_DMMU, %g2 |
65 | stxa %o0, [%o4] ASI_DMMU |
66 | 1: sub %o1, (1 << 3), %o1 |
67 | ldx [%o2 + %o1], %o3 |
68 | andcc %o3, 1, %g0 |
69 | andn %o3, 1, %o3 |
70 | be,pn %icc, 2f |
71 | or %o3, 0x10, %o3 |
72 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
73 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP |
74 | membar #Sync |
75 | brnz,pt %o1, 1b |
76 | nop |
77 | stxa %g2, [%o4] ASI_DMMU |
78 | sethi %hi(KERNBASE), %o4 |
79 | flush %o4 |
80 | retl |
81 | wrpr %g7, 0x0, %pstate |
82 | nop |
83 | nop |
84 | nop |
85 | nop |
86 | |
87 | .align 32 |
88 | .globl __flush_tlb_kernel_range |
89 | __flush_tlb_kernel_range: /* 16 insns */ |
90 | /* %o0=start, %o1=end */ |
91 | cmp %o0, %o1 |
92 | be,pn %xcc, 2f |
93 | sethi %hi(PAGE_SIZE), %o4 |
94 | sub %o1, %o0, %o3 |
95 | sub %o3, %o4, %o3 |
96 | or %o0, 0x20, %o0 ! Nucleus |
97 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP |
98 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP |
99 | membar #Sync |
100 | brnz,pt %o3, 1b |
101 | sub %o3, %o4, %o3 |
102 | 2: sethi %hi(KERNBASE), %o3 |
103 | flush %o3 |
104 | retl |
105 | nop |
106 | nop |
107 | |
108 | __spitfire_flush_tlb_mm_slow: |
109 | rdpr %pstate, %g1 |
110 | wrpr %g1, PSTATE_IE, %pstate |
111 | stxa %o0, [%o1] ASI_DMMU |
112 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
113 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
114 | flush %g6 |
115 | stxa %g2, [%o1] ASI_DMMU |
116 | sethi %hi(KERNBASE), %o1 |
117 | flush %o1 |
118 | retl |
119 | wrpr %g1, 0, %pstate |
120 | |
121 | /* |
122 | * The following code flushes one page_size worth. |
123 | */ |
124 | .section .kprobes.text, "ax" |
125 | .align 32 |
126 | .globl __flush_icache_page |
127 | __flush_icache_page: /* %o0 = phys_page */ |
128 | srlx %o0, PAGE_SHIFT, %o0 |
129 | sethi %uhi(PAGE_OFFSET), %g1 |
130 | sllx %o0, PAGE_SHIFT, %o0 |
131 | sethi %hi(PAGE_SIZE), %g2 |
132 | sllx %g1, 32, %g1 |
133 | add %o0, %g1, %o0 |
134 | 1: subcc %g2, 32, %g2 |
135 | bne,pt %icc, 1b |
136 | flush %o0 + %g2 |
137 | retl |
138 | nop |
139 | |
140 | #ifdef DCACHE_ALIASING_POSSIBLE |
141 | |
142 | #if (PAGE_SHIFT != 13) |
143 | #error only page shift of 13 is supported by dcache flush |
144 | #endif |
145 | |
146 | #define DTAG_MASK 0x3 |
147 | |
148 | /* This routine is Spitfire specific so the hardcoded |
149 | * D-cache size and line-size are OK. |
150 | */ |
151 | .align 64 |
152 | .globl __flush_dcache_page |
153 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ |
154 | sethi %uhi(PAGE_OFFSET), %g1 |
155 | sllx %g1, 32, %g1 |
156 | sub %o0, %g1, %o0 ! physical address |
157 | srlx %o0, 11, %o0 ! make D-cache TAG |
158 | sethi %hi(1 << 14), %o2 ! D-cache size |
159 | sub %o2, (1 << 5), %o2 ! D-cache line size |
160 | 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG |
161 | andcc %o3, DTAG_MASK, %g0 ! Valid? |
162 | be,pn %xcc, 2f ! Nope, branch |
163 | andn %o3, DTAG_MASK, %o3 ! Clear valid bits |
164 | cmp %o3, %o0 ! TAG match? |
165 | bne,pt %xcc, 2f ! Nope, branch |
166 | nop |
167 | stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG |
168 | membar #Sync |
169 | 2: brnz,pt %o2, 1b |
170 | sub %o2, (1 << 5), %o2 ! D-cache line size |
171 | |
172 | /* The I-cache does not snoop local stores so we |
173 | * better flush that too when necessary. |
174 | */ |
175 | brnz,pt %o1, __flush_icache_page |
176 | sllx %o0, 11, %o0 |
177 | retl |
178 | nop |
179 | |
180 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
181 | |
182 | .previous |
183 | |
184 | /* Cheetah specific versions, patched at boot time. */ |
185 | __cheetah_flush_tlb_mm: /* 19 insns */ |
186 | rdpr %pstate, %g7 |
187 | andn %g7, PSTATE_IE, %g2 |
188 | wrpr %g2, 0x0, %pstate |
189 | wrpr %g0, 1, %tl |
190 | mov PRIMARY_CONTEXT, %o2 |
191 | mov 0x40, %g3 |
192 | ldxa [%o2] ASI_DMMU, %g2 |
193 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 |
194 | sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 |
195 | or %o0, %o1, %o0 /* Preserve nucleus page size fields */ |
196 | stxa %o0, [%o2] ASI_DMMU |
197 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
198 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
199 | stxa %g2, [%o2] ASI_DMMU |
200 | sethi %hi(KERNBASE), %o2 |
201 | flush %o2 |
202 | wrpr %g0, 0, %tl |
203 | retl |
204 | wrpr %g7, 0x0, %pstate |
205 | |
206 | __cheetah_flush_tlb_pending: /* 27 insns */ |
207 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
208 | rdpr %pstate, %g7 |
209 | sllx %o1, 3, %o1 |
210 | andn %g7, PSTATE_IE, %g2 |
211 | wrpr %g2, 0x0, %pstate |
212 | wrpr %g0, 1, %tl |
213 | mov PRIMARY_CONTEXT, %o4 |
214 | ldxa [%o4] ASI_DMMU, %g2 |
215 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
216 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 |
217 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ |
218 | stxa %o0, [%o4] ASI_DMMU |
219 | 1: sub %o1, (1 << 3), %o1 |
220 | ldx [%o2 + %o1], %o3 |
221 | andcc %o3, 1, %g0 |
222 | be,pn %icc, 2f |
223 | andn %o3, 1, %o3 |
224 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
225 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP |
226 | membar #Sync |
227 | brnz,pt %o1, 1b |
228 | nop |
229 | stxa %g2, [%o4] ASI_DMMU |
230 | sethi %hi(KERNBASE), %o4 |
231 | flush %o4 |
232 | wrpr %g0, 0, %tl |
233 | retl |
234 | wrpr %g7, 0x0, %pstate |
235 | |
236 | #ifdef DCACHE_ALIASING_POSSIBLE |
237 | __cheetah_flush_dcache_page: /* 11 insns */ |
238 | sethi %uhi(PAGE_OFFSET), %g1 |
239 | sllx %g1, 32, %g1 |
240 | sub %o0, %g1, %o0 |
241 | sethi %hi(PAGE_SIZE), %o4 |
242 | 1: subcc %o4, (1 << 5), %o4 |
243 | stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE |
244 | membar #Sync |
245 | bne,pt %icc, 1b |
246 | nop |
247 | retl /* I-cache flush never needed on Cheetah, see callers. */ |
248 | nop |
249 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
250 | |
251 | /* Hypervisor specific versions, patched at boot time. */ |
252 | __hypervisor_tlb_tl0_error: |
253 | save %sp, -192, %sp |
254 | mov %i0, %o0 |
255 | call hypervisor_tlbop_error |
256 | mov %i1, %o1 |
257 | ret |
258 | restore |
259 | |
260 | __hypervisor_flush_tlb_mm: /* 10 insns */ |
261 | mov %o0, %o2 /* ARG2: mmu context */ |
262 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
263 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
264 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
265 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
266 | ta HV_FAST_TRAP |
267 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
268 | mov HV_FAST_MMU_DEMAP_CTX, %o1 |
269 | retl |
270 | nop |
271 | |
272 | __hypervisor_flush_tlb_pending: /* 16 insns */ |
273 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
274 | sllx %o1, 3, %g1 |
275 | mov %o2, %g2 |
276 | mov %o0, %g3 |
277 | 1: sub %g1, (1 << 3), %g1 |
278 | ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ |
279 | mov %g3, %o1 /* ARG1: mmu context */ |
280 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
281 | srlx %o0, PAGE_SHIFT, %o0 |
282 | sllx %o0, PAGE_SHIFT, %o0 |
283 | ta HV_MMU_UNMAP_ADDR_TRAP |
284 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
285 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
286 | brnz,pt %g1, 1b |
287 | nop |
288 | retl |
289 | nop |
290 | |
291 | __hypervisor_flush_tlb_kernel_range: /* 16 insns */ |
292 | /* %o0=start, %o1=end */ |
293 | cmp %o0, %o1 |
294 | be,pn %xcc, 2f |
295 | sethi %hi(PAGE_SIZE), %g3 |
296 | mov %o0, %g1 |
297 | sub %o1, %g1, %g2 |
298 | sub %g2, %g3, %g2 |
299 | 1: add %g1, %g2, %o0 /* ARG0: virtual address */ |
300 | mov 0, %o1 /* ARG1: mmu context */ |
301 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
302 | ta HV_MMU_UNMAP_ADDR_TRAP |
303 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
304 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
305 | brnz,pt %g2, 1b |
306 | sub %g2, %g3, %g2 |
307 | 2: retl |
308 | nop |
309 | |
310 | #ifdef DCACHE_ALIASING_POSSIBLE |
311 | /* XXX Niagara and friends have an 8K cache, so no aliasing is |
312 | * XXX possible, but nothing explicit in the Hypervisor API |
313 | * XXX guarantees this. |
314 | */ |
315 | __hypervisor_flush_dcache_page: /* 2 insns */ |
316 | retl |
317 | nop |
318 | #endif |
319 | |
320 | tlb_patch_one: |
321 | 1: lduw [%o1], %g1 |
322 | stw %g1, [%o0] |
323 | flush %o0 |
324 | subcc %o2, 1, %o2 |
325 | add %o1, 4, %o1 |
326 | bne,pt %icc, 1b |
327 | add %o0, 4, %o0 |
328 | retl |
329 | nop |
330 | |
331 | .globl cheetah_patch_cachetlbops |
332 | cheetah_patch_cachetlbops: |
333 | save %sp, -128, %sp |
334 | |
335 | sethi %hi(__flush_tlb_mm), %o0 |
336 | or %o0, %lo(__flush_tlb_mm), %o0 |
337 | sethi %hi(__cheetah_flush_tlb_mm), %o1 |
338 | or %o1, %lo(__cheetah_flush_tlb_mm), %o1 |
339 | call tlb_patch_one |
340 | mov 19, %o2 |
341 | |
342 | sethi %hi(__flush_tlb_pending), %o0 |
343 | or %o0, %lo(__flush_tlb_pending), %o0 |
344 | sethi %hi(__cheetah_flush_tlb_pending), %o1 |
345 | or %o1, %lo(__cheetah_flush_tlb_pending), %o1 |
346 | call tlb_patch_one |
347 | mov 27, %o2 |
348 | |
349 | #ifdef DCACHE_ALIASING_POSSIBLE |
350 | sethi %hi(__flush_dcache_page), %o0 |
351 | or %o0, %lo(__flush_dcache_page), %o0 |
352 | sethi %hi(__cheetah_flush_dcache_page), %o1 |
353 | or %o1, %lo(__cheetah_flush_dcache_page), %o1 |
354 | call tlb_patch_one |
355 | mov 11, %o2 |
356 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
357 | |
358 | ret |
359 | restore |
360 | |
361 | #ifdef CONFIG_SMP |
362 | /* These are all called by the slaves of a cross call, at |
363 | * trap level 1, with interrupts fully disabled. |
364 | * |
365 | * Register usage: |
366 | * %g5 mm->context (all tlb flushes) |
367 | * %g1 address arg 1 (tlb page and range flushes) |
368 | * %g7 address arg 2 (tlb range flush only) |
369 | * |
370 | * %g6 scratch 1 |
371 | * %g2 scratch 2 |
372 | * %g3 scratch 3 |
373 | * %g4 scratch 4 |
374 | */ |
375 | .align 32 |
376 | .globl xcall_flush_tlb_mm |
377 | xcall_flush_tlb_mm: /* 21 insns */ |
378 | mov PRIMARY_CONTEXT, %g2 |
379 | ldxa [%g2] ASI_DMMU, %g3 |
380 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
381 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 |
382 | or %g5, %g4, %g5 /* Preserve nucleus page size fields */ |
383 | stxa %g5, [%g2] ASI_DMMU |
384 | mov 0x40, %g4 |
385 | stxa %g0, [%g4] ASI_DMMU_DEMAP |
386 | stxa %g0, [%g4] ASI_IMMU_DEMAP |
387 | stxa %g3, [%g2] ASI_DMMU |
388 | retry |
389 | nop |
390 | nop |
391 | nop |
392 | nop |
393 | nop |
394 | nop |
395 | nop |
396 | nop |
397 | nop |
398 | nop |
399 | |
400 | .globl xcall_flush_tlb_pending |
401 | xcall_flush_tlb_pending: /* 21 insns */ |
402 | /* %g5=context, %g1=nr, %g7=vaddrs[] */ |
403 | sllx %g1, 3, %g1 |
404 | mov PRIMARY_CONTEXT, %g4 |
405 | ldxa [%g4] ASI_DMMU, %g2 |
406 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 |
407 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 |
408 | or %g5, %g4, %g5 |
409 | mov PRIMARY_CONTEXT, %g4 |
410 | stxa %g5, [%g4] ASI_DMMU |
411 | 1: sub %g1, (1 << 3), %g1 |
412 | ldx [%g7 + %g1], %g5 |
413 | andcc %g5, 0x1, %g0 |
414 | be,pn %icc, 2f |
415 | |
416 | andn %g5, 0x1, %g5 |
417 | stxa %g0, [%g5] ASI_IMMU_DEMAP |
418 | 2: stxa %g0, [%g5] ASI_DMMU_DEMAP |
419 | membar #Sync |
420 | brnz,pt %g1, 1b |
421 | nop |
422 | stxa %g2, [%g4] ASI_DMMU |
423 | retry |
424 | nop |
425 | |
426 | .globl xcall_flush_tlb_kernel_range |
427 | xcall_flush_tlb_kernel_range: /* 25 insns */ |
428 | sethi %hi(PAGE_SIZE - 1), %g2 |
429 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
430 | andn %g1, %g2, %g1 |
431 | andn %g7, %g2, %g7 |
432 | sub %g7, %g1, %g3 |
433 | add %g2, 1, %g2 |
434 | sub %g3, %g2, %g3 |
435 | or %g1, 0x20, %g1 ! Nucleus |
436 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP |
437 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP |
438 | membar #Sync |
439 | brnz,pt %g3, 1b |
440 | sub %g3, %g2, %g3 |
441 | retry |
442 | nop |
443 | nop |
444 | nop |
445 | nop |
446 | nop |
447 | nop |
448 | nop |
449 | nop |
450 | nop |
451 | nop |
452 | nop |
453 | |
454 | /* This runs in a very controlled environment, so we do |
455 | * not need to worry about BH races etc. |
456 | */ |
457 | .globl xcall_sync_tick |
458 | xcall_sync_tick: |
459 | |
460 | 661: rdpr %pstate, %g2 |
461 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate |
462 | .section .sun4v_2insn_patch, "ax" |
463 | .word 661b |
464 | nop |
465 | nop |
466 | .previous |
467 | |
468 | rdpr %pil, %g2 |
469 | wrpr %g0, PIL_NORMAL_MAX, %pil |
470 | sethi %hi(109f), %g7 |
471 | b,pt %xcc, etrap_irq |
472 | 109: or %g7, %lo(109b), %g7 |
473 | #ifdef CONFIG_TRACE_IRQFLAGS |
474 | call trace_hardirqs_off |
475 | nop |
476 | #endif |
477 | call smp_synchronize_tick_client |
478 | nop |
479 | b rtrap_xcall |
480 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 |
481 | |
482 | .globl xcall_fetch_glob_regs |
483 | xcall_fetch_glob_regs: |
484 | sethi %hi(global_reg_snapshot), %g1 |
485 | or %g1, %lo(global_reg_snapshot), %g1 |
486 | __GET_CPUID(%g2) |
487 | sllx %g2, 6, %g3 |
488 | add %g1, %g3, %g1 |
489 | rdpr %tstate, %g7 |
490 | stx %g7, [%g1 + GR_SNAP_TSTATE] |
491 | rdpr %tpc, %g7 |
492 | stx %g7, [%g1 + GR_SNAP_TPC] |
493 | rdpr %tnpc, %g7 |
494 | stx %g7, [%g1 + GR_SNAP_TNPC] |
495 | stx %o7, [%g1 + GR_SNAP_O7] |
496 | stx %i7, [%g1 + GR_SNAP_I7] |
497 | /* Don't try this at home kids... */ |
498 | rdpr %cwp, %g2 |
499 | sub %g2, 1, %g7 |
500 | wrpr %g7, %cwp |
501 | mov %i7, %g7 |
502 | wrpr %g2, %cwp |
503 | stx %g7, [%g1 + GR_SNAP_RPC] |
504 | sethi %hi(trap_block), %g7 |
505 | or %g7, %lo(trap_block), %g7 |
506 | sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 |
507 | add %g7, %g2, %g7 |
508 | ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 |
509 | stx %g3, [%g1 + GR_SNAP_THREAD] |
510 | retry |
511 | |
512 | #ifdef DCACHE_ALIASING_POSSIBLE |
513 | .align 32 |
514 | .globl xcall_flush_dcache_page_cheetah |
515 | xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ |
516 | sethi %hi(PAGE_SIZE), %g3 |
517 | 1: subcc %g3, (1 << 5), %g3 |
518 | stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE |
519 | membar #Sync |
520 | bne,pt %icc, 1b |
521 | nop |
522 | retry |
523 | nop |
524 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
525 | |
526 | .globl xcall_flush_dcache_page_spitfire |
527 | xcall_flush_dcache_page_spitfire: /* %g1 == physical page address |
528 | %g7 == kernel page virtual address |
529 | %g5 == (page->mapping != NULL) */ |
530 | #ifdef DCACHE_ALIASING_POSSIBLE |
531 | srlx %g1, (13 - 2), %g1 ! Form tag comparitor |
532 | sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K |
533 | sub %g3, (1 << 5), %g3 ! D$ linesize == 32 |
534 | 1: ldxa [%g3] ASI_DCACHE_TAG, %g2 |
535 | andcc %g2, 0x3, %g0 |
536 | be,pn %xcc, 2f |
537 | andn %g2, 0x3, %g2 |
538 | cmp %g2, %g1 |
539 | |
540 | bne,pt %xcc, 2f |
541 | nop |
542 | stxa %g0, [%g3] ASI_DCACHE_TAG |
543 | membar #Sync |
544 | 2: cmp %g3, 0 |
545 | bne,pt %xcc, 1b |
546 | sub %g3, (1 << 5), %g3 |
547 | |
548 | brz,pn %g5, 2f |
549 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
550 | sethi %hi(PAGE_SIZE), %g3 |
551 | |
552 | 1: flush %g7 |
553 | subcc %g3, (1 << 5), %g3 |
554 | bne,pt %icc, 1b |
555 | add %g7, (1 << 5), %g7 |
556 | |
557 | 2: retry |
558 | nop |
559 | nop |
560 | |
561 | /* %g5: error |
562 | * %g6: tlb op |
563 | */ |
564 | __hypervisor_tlb_xcall_error: |
565 | mov %g5, %g4 |
566 | mov %g6, %g5 |
567 | ba,pt %xcc, etrap |
568 | rd %pc, %g7 |
569 | mov %l4, %o0 |
570 | call hypervisor_tlbop_error_xcall |
571 | mov %l5, %o1 |
572 | ba,a,pt %xcc, rtrap |
573 | |
574 | .globl __hypervisor_xcall_flush_tlb_mm |
575 | __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ |
576 | /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ |
577 | mov %o0, %g2 |
578 | mov %o1, %g3 |
579 | mov %o2, %g4 |
580 | mov %o3, %g1 |
581 | mov %o5, %g7 |
582 | clr %o0 /* ARG0: CPU lists unimplemented */ |
583 | clr %o1 /* ARG1: CPU lists unimplemented */ |
584 | mov %g5, %o2 /* ARG2: mmu context */ |
585 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
586 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
587 | ta HV_FAST_TRAP |
588 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
589 | brnz,pn %o0, __hypervisor_tlb_xcall_error |
590 | mov %o0, %g5 |
591 | mov %g2, %o0 |
592 | mov %g3, %o1 |
593 | mov %g4, %o2 |
594 | mov %g1, %o3 |
595 | mov %g7, %o5 |
596 | membar #Sync |
597 | retry |
598 | |
599 | .globl __hypervisor_xcall_flush_tlb_pending |
600 | __hypervisor_xcall_flush_tlb_pending: /* 21 insns */ |
601 | /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ |
602 | sllx %g1, 3, %g1 |
603 | mov %o0, %g2 |
604 | mov %o1, %g3 |
605 | mov %o2, %g4 |
606 | 1: sub %g1, (1 << 3), %g1 |
607 | ldx [%g7 + %g1], %o0 /* ARG0: virtual address */ |
608 | mov %g5, %o1 /* ARG1: mmu context */ |
609 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
610 | srlx %o0, PAGE_SHIFT, %o0 |
611 | sllx %o0, PAGE_SHIFT, %o0 |
612 | ta HV_MMU_UNMAP_ADDR_TRAP |
613 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
614 | brnz,a,pn %o0, __hypervisor_tlb_xcall_error |
615 | mov %o0, %g5 |
616 | brnz,pt %g1, 1b |
617 | nop |
618 | mov %g2, %o0 |
619 | mov %g3, %o1 |
620 | mov %g4, %o2 |
621 | membar #Sync |
622 | retry |
623 | |
624 | .globl __hypervisor_xcall_flush_tlb_kernel_range |
625 | __hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ |
626 | /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ |
627 | sethi %hi(PAGE_SIZE - 1), %g2 |
628 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
629 | andn %g1, %g2, %g1 |
630 | andn %g7, %g2, %g7 |
631 | sub %g7, %g1, %g3 |
632 | add %g2, 1, %g2 |
633 | sub %g3, %g2, %g3 |
634 | mov %o0, %g2 |
635 | mov %o1, %g4 |
636 | mov %o2, %g7 |
637 | 1: add %g1, %g3, %o0 /* ARG0: virtual address */ |
638 | mov 0, %o1 /* ARG1: mmu context */ |
639 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
640 | ta HV_MMU_UNMAP_ADDR_TRAP |
641 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
642 | brnz,pn %o0, __hypervisor_tlb_xcall_error |
643 | mov %o0, %g5 |
644 | sethi %hi(PAGE_SIZE), %o2 |
645 | brnz,pt %g3, 1b |
646 | sub %g3, %o2, %g3 |
647 | mov %g2, %o0 |
648 | mov %g4, %o1 |
649 | mov %g7, %o2 |
650 | membar #Sync |
651 | retry |
652 | |
653 | /* These just get rescheduled to PIL vectors. */ |
654 | .globl xcall_call_function |
655 | xcall_call_function: |
656 | wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint |
657 | retry |
658 | |
659 | .globl xcall_call_function_single |
660 | xcall_call_function_single: |
661 | wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint |
662 | retry |
663 | |
664 | .globl xcall_receive_signal |
665 | xcall_receive_signal: |
666 | wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint |
667 | retry |
668 | |
669 | .globl xcall_capture |
670 | xcall_capture: |
671 | wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint |
672 | retry |
673 | |
674 | .globl xcall_new_mmu_context_version |
675 | xcall_new_mmu_context_version: |
676 | wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint |
677 | retry |
678 | |
679 | #ifdef CONFIG_KGDB |
680 | .globl xcall_kgdb_capture |
681 | xcall_kgdb_capture: |
682 | wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint |
683 | retry |
684 | #endif |
685 | |
686 | #endif /* CONFIG_SMP */ |
687 | |
688 | |
689 | .globl hypervisor_patch_cachetlbops |
690 | hypervisor_patch_cachetlbops: |
691 | save %sp, -128, %sp |
692 | |
693 | sethi %hi(__flush_tlb_mm), %o0 |
694 | or %o0, %lo(__flush_tlb_mm), %o0 |
695 | sethi %hi(__hypervisor_flush_tlb_mm), %o1 |
696 | or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 |
697 | call tlb_patch_one |
698 | mov 10, %o2 |
699 | |
700 | sethi %hi(__flush_tlb_pending), %o0 |
701 | or %o0, %lo(__flush_tlb_pending), %o0 |
702 | sethi %hi(__hypervisor_flush_tlb_pending), %o1 |
703 | or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 |
704 | call tlb_patch_one |
705 | mov 16, %o2 |
706 | |
707 | sethi %hi(__flush_tlb_kernel_range), %o0 |
708 | or %o0, %lo(__flush_tlb_kernel_range), %o0 |
709 | sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 |
710 | or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 |
711 | call tlb_patch_one |
712 | mov 16, %o2 |
713 | |
714 | #ifdef DCACHE_ALIASING_POSSIBLE |
715 | sethi %hi(__flush_dcache_page), %o0 |
716 | or %o0, %lo(__flush_dcache_page), %o0 |
717 | sethi %hi(__hypervisor_flush_dcache_page), %o1 |
718 | or %o1, %lo(__hypervisor_flush_dcache_page), %o1 |
719 | call tlb_patch_one |
720 | mov 2, %o2 |
721 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
722 | |
723 | #ifdef CONFIG_SMP |
724 | sethi %hi(xcall_flush_tlb_mm), %o0 |
725 | or %o0, %lo(xcall_flush_tlb_mm), %o0 |
726 | sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 |
727 | or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 |
728 | call tlb_patch_one |
729 | mov 21, %o2 |
730 | |
731 | sethi %hi(xcall_flush_tlb_pending), %o0 |
732 | or %o0, %lo(xcall_flush_tlb_pending), %o0 |
733 | sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 |
734 | or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 |
735 | call tlb_patch_one |
736 | mov 21, %o2 |
737 | |
738 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 |
739 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 |
740 | sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
741 | or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
742 | call tlb_patch_one |
743 | mov 25, %o2 |
744 | #endif /* CONFIG_SMP */ |
745 | |
746 | ret |
747 | restore |
748 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9