Root/arch/tile/lib/memcpy_32.S

1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <arch/chip.h>
16
17
18/*
19 * This file shares the implementation of the userspace memcpy and
20 * the kernel's memcpy, copy_to_user and copy_from_user.
21 */
22
23#include <linux/linkage.h>
24
25/* On TILE64, we wrap these functions via arch/tile/lib/memcpy_tile64.c */
26#if !CHIP_HAS_COHERENT_LOCAL_CACHE()
27#define memcpy __memcpy_asm
28#define __copy_to_user_inatomic __copy_to_user_inatomic_asm
29#define __copy_from_user_inatomic __copy_from_user_inatomic_asm
30#define __copy_from_user_zeroing __copy_from_user_zeroing_asm
31#endif
32
33#define IS_MEMCPY 0
34#define IS_COPY_FROM_USER 1
35#define IS_COPY_FROM_USER_ZEROING 2
36#define IS_COPY_TO_USER -1
37
38    .section .text.memcpy_common, "ax"
39    .align 64
40
41/* Use this to preface each bundle that can cause an exception so
42 * the kernel can clean up properly. The special cleanup code should
43 * not use these, since it knows what it is doing.
44 */
45#define EX \
46    .pushsection __ex_table, "a"; \
47    .word 9f, memcpy_common_fixup; \
48    .popsection; \
49    9
50
51
52/* __copy_from_user_inatomic takes the kernel target address in r0,
53 * the user source in r1, and the bytes to copy in r2.
54 * It returns the number of uncopiable bytes (hopefully zero) in r0.
55 */
56ENTRY(__copy_from_user_inatomic)
57.type __copy_from_user_inatomic, @function
58    FEEDBACK_ENTER_EXPLICIT(__copy_from_user_inatomic, \
59      .text.memcpy_common, \
60      .Lend_memcpy_common - __copy_from_user_inatomic)
61    { movei r29, IS_COPY_FROM_USER; j memcpy_common }
62    .size __copy_from_user_inatomic, . - __copy_from_user_inatomic
63
64/* __copy_from_user_zeroing is like __copy_from_user_inatomic, but
65 * any uncopiable bytes are zeroed in the target.
66 */
67ENTRY(__copy_from_user_zeroing)
68.type __copy_from_user_zeroing, @function
69    FEEDBACK_REENTER(__copy_from_user_inatomic)
70    { movei r29, IS_COPY_FROM_USER_ZEROING; j memcpy_common }
71    .size __copy_from_user_zeroing, . - __copy_from_user_zeroing
72
73/* __copy_to_user_inatomic takes the user target address in r0,
74 * the kernel source in r1, and the bytes to copy in r2.
75 * It returns the number of uncopiable bytes (hopefully zero) in r0.
76 */
77ENTRY(__copy_to_user_inatomic)
78.type __copy_to_user_inatomic, @function
79    FEEDBACK_REENTER(__copy_from_user_inatomic)
80    { movei r29, IS_COPY_TO_USER; j memcpy_common }
81    .size __copy_to_user_inatomic, . - __copy_to_user_inatomic
82
83ENTRY(memcpy)
84.type memcpy, @function
85    FEEDBACK_REENTER(__copy_from_user_inatomic)
86    { movei r29, IS_MEMCPY }
87    .size memcpy, . - memcpy
88    /* Fall through */
89
90    .type memcpy_common, @function
91memcpy_common:
92    /* On entry, r29 holds one of the IS_* macro values from above. */
93
94
95    /* r0 is the dest, r1 is the source, r2 is the size. */
96
97    /* Save aside original dest so we can return it at the end. */
98    { sw sp, lr; move r23, r0; or r4, r0, r1 }
99
100    /* Check for an empty size. */
101    { bz r2, .Ldone; andi r4, r4, 3 }
102
103    /* Save aside original values in case of a fault. */
104    { move r24, r1; move r25, r2 }
105    move r27, lr
106
107    /* Check for an unaligned source or dest. */
108    { bnz r4, .Lcopy_unaligned_maybe_many; addli r4, r2, -256 }
109
110.Lcheck_aligned_copy_size:
111    /* If we are copying < 256 bytes, branch to simple case. */
112    { blzt r4, .Lcopy_8_check; slti_u r8, r2, 8 }
113
114    /* Copying >= 256 bytes, so jump to complex prefetching loop. */
115    { andi r6, r1, 63; j .Lcopy_many }
116
117/*
118 *
119 * Aligned 4 byte at a time copy loop
120 *
121 */
122
123.Lcopy_8_loop:
124    /* Copy two words at a time to hide load latency. */
125EX: { lw r3, r1; addi r1, r1, 4; slti_u r8, r2, 16 }
126EX: { lw r4, r1; addi r1, r1, 4 }
127EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }
128EX: { sw r0, r4; addi r0, r0, 4; addi r2, r2, -4 }
129.Lcopy_8_check:
130    { bzt r8, .Lcopy_8_loop; slti_u r4, r2, 4 }
131
132    /* Copy odd leftover word, if any. */
133    { bnzt r4, .Lcheck_odd_stragglers }
134EX: { lw r3, r1; addi r1, r1, 4 }
135EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }
136
137.Lcheck_odd_stragglers:
138    { bnz r2, .Lcopy_unaligned_few }
139
140.Ldone:
141    /* For memcpy return original dest address, else zero. */
142    { mz r0, r29, r23; jrp lr }
143
144
145/*
146 *
147 * Prefetching multiple cache line copy handler (for large transfers).
148 *
149 */
150
151    /* Copy words until r1 is cache-line-aligned. */
152.Lalign_loop:
153EX: { lw r3, r1; addi r1, r1, 4 }
154    { andi r6, r1, 63 }
155EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }
156.Lcopy_many:
157    { bnzt r6, .Lalign_loop; addi r9, r0, 63 }
158
159    { addi r3, r1, 60; andi r9, r9, -64 }
160
161#if CHIP_HAS_WH64()
162    /* No need to prefetch dst, we'll just do the wh64
163     * right before we copy a line.
164     */
165#endif
166
167EX: { lw r5, r3; addi r3, r3, 64; movei r4, 1 }
168    /* Intentionally stall for a few cycles to leave L2 cache alone. */
169    { bnzt zero, .; move r27, lr }
170EX: { lw r6, r3; addi r3, r3, 64 }
171    /* Intentionally stall for a few cycles to leave L2 cache alone. */
172    { bnzt zero, . }
173EX: { lw r7, r3; addi r3, r3, 64 }
174#if !CHIP_HAS_WH64()
175    /* Prefetch the dest */
176    /* Intentionally stall for a few cycles to leave L2 cache alone. */
177    { bnzt zero, . }
178    /* Use a real load to cause a TLB miss if necessary. We aren't using
179     * r28, so this should be fine.
180     */
181EX: { lw r28, r9; addi r9, r9, 64 }
182    /* Intentionally stall for a few cycles to leave L2 cache alone. */
183    { bnzt zero, . }
184    { prefetch r9; addi r9, r9, 64 }
185    /* Intentionally stall for a few cycles to leave L2 cache alone. */
186    { bnzt zero, . }
187    { prefetch r9; addi r9, r9, 64 }
188#endif
189    /* Intentionally stall for a few cycles to leave L2 cache alone. */
190    { bz zero, .Lbig_loop2 }
191
192    /* On entry to this loop:
193     * - r0 points to the start of dst line 0
194     * - r1 points to start of src line 0
195     * - r2 >= (256 - 60), only the first time the loop trips.
196     * - r3 contains r1 + 128 + 60 [pointer to end of source line 2]
197     * This is our prefetch address. When we get near the end
198     * rather than prefetching off the end this is changed to point
199     * to some "safe" recently loaded address.
200     * - r5 contains *(r1 + 60) [i.e. last word of source line 0]
201     * - r6 contains *(r1 + 64 + 60) [i.e. last word of source line 1]
202     * - r9 contains ((r0 + 63) & -64)
203     * [start of next dst cache line.]
204     */
205
206.Lbig_loop:
207    { jal .Lcopy_line2; add r15, r1, r2 }
208
209.Lbig_loop2:
210    /* Copy line 0, first stalling until r5 is ready. */
211EX: { move r12, r5; lw r16, r1 }
212    { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
213    /* Prefetch several lines ahead. */
214EX: { lw r5, r3; addi r3, r3, 64 }
215    { jal .Lcopy_line }
216
217    /* Copy line 1, first stalling until r6 is ready. */
218EX: { move r12, r6; lw r16, r1 }
219    { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
220    /* Prefetch several lines ahead. */
221EX: { lw r6, r3; addi r3, r3, 64 }
222    { jal .Lcopy_line }
223
224    /* Copy line 2, first stalling until r7 is ready. */
225EX: { move r12, r7; lw r16, r1 }
226    { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
227    /* Prefetch several lines ahead. */
228EX: { lw r7, r3; addi r3, r3, 64 }
229    /* Use up a caches-busy cycle by jumping back to the top of the
230     * loop. Might as well get it out of the way now.
231     */
232    { j .Lbig_loop }
233
234
235    /* On entry:
236     * - r0 points to the destination line.
237     * - r1 points to the source line.
238     * - r3 is the next prefetch address.
239     * - r9 holds the last address used for wh64.
240     * - r12 = WORD_15
241     * - r16 = WORD_0.
242     * - r17 == r1 + 16.
243     * - r27 holds saved lr to restore.
244     *
245     * On exit:
246     * - r0 is incremented by 64.
247     * - r1 is incremented by 64, unless that would point to a word
248     * beyond the end of the source array, in which case it is redirected
249     * to point to an arbitrary word already in the cache.
250     * - r2 is decremented by 64.
251     * - r3 is unchanged, unless it points to a word beyond the
252     * end of the source array, in which case it is redirected
253     * to point to an arbitrary word already in the cache.
254     * Redirecting is OK since if we are that close to the end
255     * of the array we will not come back to this subroutine
256     * and use the contents of the prefetched address.
257     * - r4 is nonzero iff r2 >= 64.
258     * - r9 is incremented by 64, unless it points beyond the
259     * end of the last full destination cache line, in which
260     * case it is redirected to a "safe address" that can be
261     * clobbered (sp - 64)
262     * - lr contains the value in r27.
263     */
264
265/* r26 unused */
266
267.Lcopy_line:
268    /* TODO: when r3 goes past the end, we would like to redirect it
269     * to prefetch the last partial cache line (if any) just once, for the
270     * benefit of the final cleanup loop. But we don't want to
271     * prefetch that line more than once, or subsequent prefetches
272     * will go into the RTF. But then .Lbig_loop should unconditionally
273     * branch to top of loop to execute final prefetch, and its
274     * nop should become a conditional branch.
275     */
276
277    /* We need two non-memory cycles here to cover the resources
278     * used by the loads initiated by the caller.
279     */
280    { add r15, r1, r2 }
281.Lcopy_line2:
282    { slt_u r13, r3, r15; addi r17, r1, 16 }
283
284    /* NOTE: this will stall for one cycle as L1 is busy. */
285
286    /* Fill second L1D line. */
287EX: { lw r17, r17; addi r1, r1, 48; mvz r3, r13, r1 } /* r17 = WORD_4 */
288
289#if CHIP_HAS_WH64()
290    /* Prepare destination line for writing. */
291EX: { wh64 r9; addi r9, r9, 64 }
292#else
293    /* Prefetch dest line */
294    { prefetch r9; addi r9, r9, 64 }
295#endif
296    /* Load seven words that are L1D hits to cover wh64 L2 usage. */
297
298    /* Load the three remaining words from the last L1D line, which
299     * we know has already filled the L1D.
300     */
301EX: { lw r4, r1; addi r1, r1, 4; addi r20, r1, 16 } /* r4 = WORD_12 */
302EX: { lw r8, r1; addi r1, r1, 4; slt_u r13, r20, r15 }/* r8 = WORD_13 */
303EX: { lw r11, r1; addi r1, r1, -52; mvz r20, r13, r1 } /* r11 = WORD_14 */
304
305    /* Load the three remaining words from the first L1D line, first
306     * stalling until it has filled by "looking at" r16.
307     */
308EX: { lw r13, r1; addi r1, r1, 4; move zero, r16 } /* r13 = WORD_1 */
309EX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_2 */
310EX: { lw r15, r1; addi r1, r1, 8; addi r10, r0, 60 } /* r15 = WORD_3 */
311
312    /* Load second word from the second L1D line, first
313     * stalling until it has filled by "looking at" r17.
314     */
315EX: { lw r19, r1; addi r1, r1, 4; move zero, r17 } /* r19 = WORD_5 */
316
317    /* Store last word to the destination line, potentially dirtying it
318     * for the first time, which keeps the L2 busy for two cycles.
319     */
320EX: { sw r10, r12 } /* store(WORD_15) */
321
322    /* Use two L1D hits to cover the sw L2 access above. */
323EX: { lw r10, r1; addi r1, r1, 4 } /* r10 = WORD_6 */
324EX: { lw r12, r1; addi r1, r1, 4 } /* r12 = WORD_7 */
325
326    /* Fill third L1D line. */
327EX: { lw r18, r1; addi r1, r1, 4 } /* r18 = WORD_8 */
328
329    /* Store first L1D line. */
330EX: { sw r0, r16; addi r0, r0, 4; add r16, r0, r2 } /* store(WORD_0) */
331EX: { sw r0, r13; addi r0, r0, 4; andi r16, r16, -64 } /* store(WORD_1) */
332EX: { sw r0, r14; addi r0, r0, 4; slt_u r16, r9, r16 } /* store(WORD_2) */
333#if CHIP_HAS_WH64()
334EX: { sw r0, r15; addi r0, r0, 4; addi r13, sp, -64 } /* store(WORD_3) */
335#else
336    /* Back up the r9 to a cache line we are already storing to
337     * if it gets past the end of the dest vector. Strictly speaking,
338     * we don't need to back up to the start of a cache line, but it's free
339     * and tidy, so why not?
340     */
341EX: { sw r0, r15; addi r0, r0, 4; andi r13, r0, -64 } /* store(WORD_3) */
342#endif
343    /* Store second L1D line. */
344EX: { sw r0, r17; addi r0, r0, 4; mvz r9, r16, r13 }/* store(WORD_4) */
345EX: { sw r0, r19; addi r0, r0, 4 } /* store(WORD_5) */
346EX: { sw r0, r10; addi r0, r0, 4 } /* store(WORD_6) */
347EX: { sw r0, r12; addi r0, r0, 4 } /* store(WORD_7) */
348
349EX: { lw r13, r1; addi r1, r1, 4; move zero, r18 } /* r13 = WORD_9 */
350EX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_10 */
351EX: { lw r15, r1; move r1, r20 } /* r15 = WORD_11 */
352
353    /* Store third L1D line. */
354EX: { sw r0, r18; addi r0, r0, 4 } /* store(WORD_8) */
355EX: { sw r0, r13; addi r0, r0, 4 } /* store(WORD_9) */
356EX: { sw r0, r14; addi r0, r0, 4 } /* store(WORD_10) */
357EX: { sw r0, r15; addi r0, r0, 4 } /* store(WORD_11) */
358
359    /* Store rest of fourth L1D line. */
360EX: { sw r0, r4; addi r0, r0, 4 } /* store(WORD_12) */
361    {
362EX: sw r0, r8 /* store(WORD_13) */
363    addi r0, r0, 4
364    /* Will r2 be > 64 after we subtract 64 below? */
365    shri r4, r2, 7
366    }
367    {
368EX: sw r0, r11 /* store(WORD_14) */
369    addi r0, r0, 8
370    /* Record 64 bytes successfully copied. */
371    addi r2, r2, -64
372    }
373
374    { jrp lr; move lr, r27 }
375
376    /* Convey to the backtrace library that the stack frame is size
377     * zero, and the real return address is on the stack rather than
378     * in 'lr'.
379     */
380    { info 8 }
381
382    .align 64
383.Lcopy_unaligned_maybe_many:
384    /* Skip the setup overhead if we aren't copying many bytes. */
385    { slti_u r8, r2, 20; sub r4, zero, r0 }
386    { bnzt r8, .Lcopy_unaligned_few; andi r4, r4, 3 }
387    { bz r4, .Ldest_is_word_aligned; add r18, r1, r2 }
388
389/*
390 *
391 * unaligned 4 byte at a time copy handler.
392 *
393 */
394
395    /* Copy single bytes until r0 == 0 mod 4, so we can store words. */
396.Lalign_dest_loop:
397EX: { lb_u r3, r1; addi r1, r1, 1; addi r4, r4, -1 }
398EX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 }
399    { bnzt r4, .Lalign_dest_loop; andi r3, r1, 3 }
400
401    /* If source and dest are now *both* aligned, do an aligned copy. */
402    { bz r3, .Lcheck_aligned_copy_size; addli r4, r2, -256 }
403
404.Ldest_is_word_aligned:
405
406#if CHIP_HAS_DWORD_ALIGN()
407EX: { andi r8, r0, 63; lwadd_na r6, r1, 4}
408    { slti_u r9, r2, 64; bz r8, .Ldest_is_L2_line_aligned }
409
410    /* This copies unaligned words until either there are fewer
411     * than 4 bytes left to copy, or until the destination pointer
412     * is cache-aligned, whichever comes first.
413     *
414     * On entry:
415     * - r0 is the next store address.
416     * - r1 points 4 bytes past the load address corresponding to r0.
417     * - r2 >= 4
418     * - r6 is the next aligned word loaded.
419     */
420.Lcopy_unaligned_src_words:
421EX: { lwadd_na r7, r1, 4; slti_u r8, r2, 4 + 4 }
422    /* stall */
423    { dword_align r6, r7, r1; slti_u r9, r2, 64 + 4 }
424EX: { swadd r0, r6, 4; addi r2, r2, -4 }
425    { bnz r8, .Lcleanup_unaligned_words; andi r8, r0, 63 }
426    { bnzt r8, .Lcopy_unaligned_src_words; move r6, r7 }
427
428    /* On entry:
429     * - r0 is the next store address.
430     * - r1 points 4 bytes past the load address corresponding to r0.
431     * - r2 >= 4 (# of bytes left to store).
432     * - r6 is the next aligned src word value.
433     * - r9 = (r2 < 64U).
434     * - r18 points one byte past the end of source memory.
435     */
436.Ldest_is_L2_line_aligned:
437
438    {
439    /* Not a full cache line remains. */
440    bnz r9, .Lcleanup_unaligned_words
441    move r7, r6
442    }
443
444    /* r2 >= 64 */
445
446    /* Kick off two prefetches, but don't go past the end. */
447    { addi r3, r1, 63 - 4; addi r8, r1, 64 + 63 - 4 }
448    { prefetch r3; move r3, r8; slt_u r8, r8, r18 }
449    { mvz r3, r8, r1; addi r8, r3, 64 }
450    { prefetch r3; move r3, r8; slt_u r8, r8, r18 }
451    { mvz r3, r8, r1; movei r17, 0 }
452
453.Lcopy_unaligned_line:
454    /* Prefetch another line. */
455    { prefetch r3; addi r15, r1, 60; addi r3, r3, 64 }
456    /* Fire off a load of the last word we are about to copy. */
457EX: { lw_na r15, r15; slt_u r8, r3, r18 }
458
459EX: { mvz r3, r8, r1; wh64 r0 }
460
461    /* This loop runs twice.
462     *
463     * On entry:
464     * - r17 is even before the first iteration, and odd before
465     * the second. It is incremented inside the loop. Encountering
466     * an even value at the end of the loop makes it stop.
467     */
468.Lcopy_half_an_unaligned_line:
469EX: {
470    /* Stall until the last byte is ready. In the steady state this
471     * guarantees all words to load below will be in the L2 cache, which
472     * avoids shunting the loads to the RTF.
473     */
474    move zero, r15
475    lwadd_na r7, r1, 16
476    }
477EX: { lwadd_na r11, r1, 12 }
478EX: { lwadd_na r14, r1, -24 }
479EX: { lwadd_na r8, r1, 4 }
480EX: { lwadd_na r9, r1, 4 }
481EX: {
482    lwadd_na r10, r1, 8
483    /* r16 = (r2 < 64), after we subtract 32 from r2 below. */
484    slti_u r16, r2, 64 + 32
485    }
486EX: { lwadd_na r12, r1, 4; addi r17, r17, 1 }
487EX: { lwadd_na r13, r1, 8; dword_align r6, r7, r1 }
488EX: { swadd r0, r6, 4; dword_align r7, r8, r1 }
489EX: { swadd r0, r7, 4; dword_align r8, r9, r1 }
490EX: { swadd r0, r8, 4; dword_align r9, r10, r1 }
491EX: { swadd r0, r9, 4; dword_align r10, r11, r1 }
492EX: { swadd r0, r10, 4; dword_align r11, r12, r1 }
493EX: { swadd r0, r11, 4; dword_align r12, r13, r1 }
494EX: { swadd r0, r12, 4; dword_align r13, r14, r1 }
495EX: { swadd r0, r13, 4; addi r2, r2, -32 }
496    { move r6, r14; bbst r17, .Lcopy_half_an_unaligned_line }
497
498    { bzt r16, .Lcopy_unaligned_line; move r7, r6 }
499
500    /* On entry:
501     * - r0 is the next store address.
502     * - r1 points 4 bytes past the load address corresponding to r0.
503     * - r2 >= 0 (# of bytes left to store).
504     * - r7 is the next aligned src word value.
505     */
506.Lcleanup_unaligned_words:
507    /* Handle any trailing bytes. */
508    { bz r2, .Lcopy_unaligned_done; slti_u r8, r2, 4 }
509    { bzt r8, .Lcopy_unaligned_src_words; move r6, r7 }
510
511    /* Move r1 back to the point where it corresponds to r0. */
512    { addi r1, r1, -4 }
513
514#else /* !CHIP_HAS_DWORD_ALIGN() */
515
516    /* Compute right/left shift counts and load initial source words. */
517    { andi r5, r1, -4; andi r3, r1, 3 }
518EX: { lw r6, r5; addi r5, r5, 4; shli r3, r3, 3 }
519EX: { lw r7, r5; addi r5, r5, 4; sub r4, zero, r3 }
520
521    /* Load and store one word at a time, using shifts and ORs
522     * to correct for the misaligned src.
523     */
524.Lcopy_unaligned_src_loop:
525    { shr r6, r6, r3; shl r8, r7, r4 }
526EX: { lw r7, r5; or r8, r8, r6; move r6, r7 }
527EX: { sw r0, r8; addi r0, r0, 4; addi r2, r2, -4 }
528    { addi r5, r5, 4; slti_u r8, r2, 8 }
529    { bzt r8, .Lcopy_unaligned_src_loop; addi r1, r1, 4 }
530
531    { bz r2, .Lcopy_unaligned_done }
532#endif /* !CHIP_HAS_DWORD_ALIGN() */
533
534    /* Fall through */
535
536/*
537 *
538 * 1 byte at a time copy handler.
539 *
540 */
541
542.Lcopy_unaligned_few:
543EX: { lb_u r3, r1; addi r1, r1, 1 }
544EX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 }
545    { bnzt r2, .Lcopy_unaligned_few }
546
547.Lcopy_unaligned_done:
548
549    /* For memcpy return original dest address, else zero. */
550    { mz r0, r29, r23; jrp lr }
551
552.Lend_memcpy_common:
553    .size memcpy_common, .Lend_memcpy_common - memcpy_common
554
555    .section .fixup,"ax"
556memcpy_common_fixup:
557    .type memcpy_common_fixup, @function
558
559    /* Skip any bytes we already successfully copied.
560     * r2 (num remaining) is correct, but r0 (dst) and r1 (src)
561     * may not be quite right because of unrolling and prefetching.
562     * So we need to recompute their values as the address just
563     * after the last byte we are sure was successfully loaded and
564     * then stored.
565     */
566
567    /* Determine how many bytes we successfully copied. */
568    { sub r3, r25, r2 }
569
570    /* Add this to the original r0 and r1 to get their new values. */
571    { add r0, r23, r3; add r1, r24, r3 }
572
573    { bzt r29, memcpy_fixup_loop }
574    { blzt r29, copy_to_user_fixup_loop }
575
576copy_from_user_fixup_loop:
577    /* Try copying the rest one byte at a time, expecting a load fault. */
578.Lcfu: { lb_u r3, r1; addi r1, r1, 1 }
579    { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 }
580    { bnzt r2, copy_from_user_fixup_loop }
581
582.Lcopy_from_user_fixup_zero_remainder:
583    { bbs r29, 2f } /* low bit set means IS_COPY_FROM_USER */
584    /* byte-at-a-time loop faulted, so zero the rest. */
585    { move r3, r2; bz r2, 2f /* should be impossible, but handle it. */ }
5861: { sb r0, zero; addi r0, r0, 1; addi r3, r3, -1 }
587    { bnzt r3, 1b }
5882: move lr, r27
589    { move r0, r2; jrp lr }
590
591copy_to_user_fixup_loop:
592    /* Try copying the rest one byte at a time, expecting a store fault. */
593    { lb_u r3, r1; addi r1, r1, 1 }
594.Lctu: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 }
595    { bnzt r2, copy_to_user_fixup_loop }
596.Lcopy_to_user_fixup_done:
597    move lr, r27
598    { move r0, r2; jrp lr }
599
600memcpy_fixup_loop:
601    /* Try copying the rest one byte at a time. We expect a disastrous
602     * fault to happen since we are in fixup code, but let it happen.
603     */
604    { lb_u r3, r1; addi r1, r1, 1 }
605    { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 }
606    { bnzt r2, memcpy_fixup_loop }
607    /* This should be unreachable, we should have faulted again.
608     * But be paranoid and handle it in case some interrupt changed
609     * the TLB or something.
610     */
611    move lr, r27
612    { move r0, r23; jrp lr }
613
614    .size memcpy_common_fixup, . - memcpy_common_fixup
615
616    .section __ex_table,"a"
617    .word .Lcfu, .Lcopy_from_user_fixup_zero_remainder
618    .word .Lctu, .Lcopy_to_user_fixup_done
619

Archive Download this file



interactive