/*
* Copyright (C) 2002 Paul Mackerras, IBM Corp.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/export.h>
#ifdef __BIG_ENDIAN__
#define sLd sld /* Shift towards low-numbered address. */
#define sHd srd /* Shift towards high-numbered address. */
#else
#define sLd srd /* Shift towards low-numbered address. */
#define sHd sld /* Shift towards high-numbered address. */
#endif
.align 7
_GLOBAL_TOC(__copy_tofrom_user)
BEGIN_FTR_SECTION
nop
FTR_SECTION_ELSE
b __copy_tofrom_user_power7
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
_GLOBAL(__copy_tofrom_user_base)
/* first check for a whole page copy on a page boundary */
cmpldi cr1,r5,16
cmpdi cr6,r5,4096
or r0,r3,r4
neg r6,r3 /* LS 3 bits = # bytes to 8-byte dest bdry */
andi. r0,r0,4095
std r3,-24(r1)
crand cr0*4+2,cr0*4+2,cr6*4+2
std r4,-16(r1)
std r5,-8(r1)
dcbt 0,r4
beq .Lcopy_page_4K
andi. r6,r6,7
PPC_MTOCRF(0x01,r5)
blt cr1,.Lshort_copy
/* Below we want to nop out the bne if we're on a CPU that has the
* CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
* cleared.
* At the time of writing the only CPU that has this combination of bits
* set is Power6.
*/
BEGIN_FTR_SECTION
nop
FTR_SECTION_ELSE
bne .Ldst_unaligned
ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
CPU_FTR_UNALIGNED_LD_STD)
.Ldst_aligned:
addi r3,r3,-16
BEGIN_FTR_SECTION
andi. r0,r4,7
bne .Lsrc_unaligned
END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
blt cr1,.Ldo_tail /* if < 16 bytes to copy */
srdi r0,r5,5
cmpdi cr1,r0,0
20: ld r7,0(r4)
220: ld r6,8(r4)
addi r4,r4,16
mtctr r0
andi. r0,r5,0x10
beq 22f
addi r3,r3,16
addi r4,r4,-16
mr r9,r7
mr r8,r6
beq cr1,72f
21: ld r7,16(r4)
221: ld r6,24(r4)
addi r4,r4,32
70: std r9,0(r3)
270: std r8,8(r3)
22: ld r9,0(r4)
222: ld r8,8(r4)
71: std r7,16(r3)
271: std r6,24(r3)
addi r3,r3,32
bdnz 21b
72: std r9,0(r3)
272: std r8,8(r3)
andi. r5,r5,0xf
beq+ 3f
addi r4,r4,16
.Ldo_tail:
addi r3,r3,16
bf cr7*4+0,246f
244: ld r9,0(r4)
addi r4,r4,8
245: std r9,0(r3)
addi r3,r3,8
246: bf cr7*4+1,1f
23: lwz r9,0(r4)
addi r4,r4,4
73: stw r9,0(r3)
addi r3,r3,4
1: bf cr7*4+2,2f
44: lhz r9,0(r4)
addi r4,r4,2
74: sth r9,0(r3)
addi r3,r3,2
2: bf cr7*4+3,3f
45: lbz r9,0(r4)
75: stb r9,0(r3)
3: li r3,0
blr
.Lsrc_unaligned:
srdi r6,r5,3
addi r5,r5,-16
subf r4,r0,r4
srdi r7,r5,4
sldi r10,r0,3
cmpldi cr6,r6,3
andi. r5,r5,7
mtctr r7
subfic r11,r10,64
add r5,r5,r0
bt cr7*4+0,28f
24: ld r9,0(r4) /* 3+2n loads, 2+2n stores */
25: ld r0,8(r4)
sLd r6,r9,r10
26: ldu r9,16(r4)
sHd r7,r0,r11
sLd r8,r0,r10
or r7,r7,r6
blt cr6,79f
27: ld r0,8(r4)
b 2f
28: ld r0,0(r4) /* 4+2n loads, 3+2n stores */
29: ldu r9,8(r4)
sLd r8,r0,r10
addi r3,r3,-8
blt cr6,5f
30: ld r0,8(r4)
sHd r12,r9,r11
sLd r6,r9,r10
31: ldu r9,16(r4)
or r12,r8,r12
sHd r7,r0,r11
sLd r8,r0,r10
addi r3,r3,16
beq cr6,78f
1: or r7,r7,r6
32: ld r0,8(r4)
76: std r12,8(r3)
2: sHd r12,r9,r11
sLd r6,r9,r10
33: ldu r9,16(r4)
or r12,r8,r12
77: stdu r7,16(r3)
sHd r7,r0,r11
sLd r8,r0,r10
bdnz 1b
78: std r12,8(r3)
or r7,r7,r6
79: std r7,16(r3)
5: sHd r12,r9,r11
or r12,r8,r12
80: std r12,24(r3)
bne 6f
li r3,0
blr
6: cmpwi cr1,r5,8
addi r3,r3,32
sLd r9,r9,r10
ble cr1,7f
34: ld r0,8(r4)
sHd r7,r0,r11
or r9,r7,r9
7:
bf cr7*4+1,1f
#ifdef __BIG_ENDIAN__
rotldi r9,r9,32
#endif
94: stw r9,0(r3)
#ifdef __LITTLE_ENDIAN__
rotrdi r9,r9,32
#endif
addi r3,r3,4
1: bf cr7*4+2,2f
#ifdef __BIG_ENDIAN__
rotldi r9,r9,16
#endif
95: sth r9,0(r3)
#ifdef __LITTLE_ENDIAN__
rotrdi r9,r9,16
#endif
addi r3,r3,2
2: bf cr7*4+3,3f
#ifdef __BIG_ENDIAN__
rotldi r9,r9,8
#endif
96: stb r9,0(r3)
#ifdef __LITTLE_ENDIAN__
rotrdi r9,r9,8
#endif
3: li r3,0
blr
.Ldst_unaligned:
PPC_MTOCRF(0x01,r6) /* put #bytes to 8B bdry into cr7 */
subf r5,r6,r5
li r7,0
cmpldi cr1,r5,16
bf cr7*4+3,1f
35: lbz r0,0(r4)
81: stb r0,0(r3)
addi r7,r7,1
1: bf cr7*4+2,2f
36: lhzx r0,r7,r4
82: sthx r0,r7,r3
addi r7,r7,2
2: bf cr7*4+1,3f
37: lwzx r0,r7,r4
83: stwx r0,r7,r3
3: PPC_MTOCRF(0x01,r5)
add r4,r6,r4
add r3,r6,r3
b .Ldst_aligned
.Lshort_copy:
bf cr7*4+0,1f
38: lwz r0,0(r4)
39: lwz r9,4(r4)
addi r4,r4,8
84: stw r0,0(r3)
85: stw r9,4(r3)
addi r3,r3,8
1: bf cr7*4+1,2f
40: lwz r0,0(r4)
addi r4,r4,4
86: stw r0,0(r3)
addi r3,r3,4
2: bf cr7*4+2,3f
41: lhz r0,0(r4)
addi r4,r4,2
87: sth r0,0(r3)
addi r3,r3,2
3: bf cr7*4+3,4f
42: lbz r0,0(r4)
88: stb r0,0(r3)
4: li r3,0
blr
/*
* exception handlers follow
* we have to return the number of bytes not copied
* for an exception on a load, we set the rest of the destination to 0
*/
136:
137:
add r3,r3,r7
b 1f
130:
131:
addi r3,r3,8
120:
320:
122:
322:
124:
125:
126:
127:
128:
129:
133:
addi r3,r3,8
132:
addi r3,r3,8
121:
321:
344:
134:
135:
138:
139:
140:
141:
142:
123:
144:
145:
/*
* here we have had a fault on a load and r3 points to the first
* unmodified byte of the destination
*/
1: ld r6,-24(r1)
ld r4,-16(r1)
ld r5,-8(r1)
subf r6,r6,r3
add r4,r4,r6
subf r5,r6,r5 /* #bytes left to go */
/*
* first see if we can copy any more bytes before hitting another exception
*/
mtctr r5
43: lbz r0,0(r4)
addi r4,r4,1
89: stb r0,0(r3)
addi r3,r3,1
bdnz 43b
li r3,0 /* huh? all copied successfully this time? */
blr
/*
* here we have trapped again, need to clear ctr bytes starting at r3
*/
143: mfctr r5
li r0,0
mr r4,r3
mr r3,r5 /* return the number of bytes not copied */
1: andi. r9,r4,7
beq 3f
90: stb r0,0(r4)
addic. r5,r5,-1
addi r4,r4,1
bne 1b
blr
3: cmpldi cr1,r5,8
srdi r9,r5,3
andi. r5,r5,7
blt cr1,93f
mtctr r9
91: std r0,0(r4)
addi r4,r4,8
bdnz 91b
93: beqlr
mtctr r5
92: stb r0,0(r4)
addi r4,r4,1
bdnz 92b
blr
/*
* exception handlers for stores: we just need to work
* out how many bytes weren't copied
*/
182:
183:
add r3,r3,r7
b 1f
371:
180:
addi r3,r3,8
171:
177:
179:
addi r3,r3,8
370:
372:
176:
178:
addi r3,r3,4
185:
addi r3,r3,4
170:
172:
345:
173:
174:
175:
181:
184:
186:
187:
188:
189:
194:
195:
196:
1:
ld r6,-24(r1)
ld r5,-8(r1)
add r6,r6,r5
subf r3,r3,r6 /* #bytes not copied */
190:
191:
192:
blr /* #bytes not copied in r3 */
EX_TABLE(20b,120b)
EX_TABLE(220b,320b)
EX_TABLE(21b,121b)
EX_TABLE(221b,321b)
EX_TABLE(70b,170b)
EX_TABLE(270b,370b)
EX_TABLE(22b,122b)
EX_TABLE(222b,322b)
EX_TABLE(71b,171b)
EX_TABLE(271b,371b)
EX_TABLE(72b,172b)
EX_TABLE(272b,372b)
EX_TABLE(244b,344b)
EX_TABLE(245b,345b)
EX_TABLE(23b,123b)
EX_TABLE(73b,173b)
EX_TABLE(44b,144b)
EX_TABLE(74b,174b)
EX_TABLE(45b,145b)
EX_TABLE(75b,175b)
EX_TABLE(24b,124b)
EX_TABLE(25b,125b)
EX_TABLE(26b,126b)
EX_TABLE(27b,127b)
EX_TABLE(28b,128b)
EX_TABLE(29b,129b)
EX_TABLE(30b,130b)
EX_TABLE(31b,131b)
EX_TABLE(32b,132b)
EX_TABLE(76b,176b)
EX_TABLE(33b,133b)
EX_TABLE(77b,177b)
EX_TABLE(78b,178b)
EX_TABLE(79b,179b)
EX_TABLE(80b,180b)
EX_TABLE(34b,134b)
EX_TABLE(94b,194b)
EX_TABLE(95b,195b)
EX_TABLE(96b,196b)
EX_TABLE(35b,135b)
EX_TABLE(81b,181b)
EX_TABLE(36b,136b)
EX_TABLE(82b,182b)
EX_TABLE(37b,137b)
EX_TABLE(83b,183b)
EX_TABLE(38b,138b)
EX_TABLE(39b,139b)
EX_TABLE(84b,184b)
EX_TABLE(85b,185b)
EX_TABLE(40b,140b)
EX_TABLE(86b,186b)
EX_TABLE(41b,141b)
EX_TABLE(87b,187b)
EX_TABLE(42b,142b)
EX_TABLE(88b,188b)
EX_TABLE(43b,143b)
EX_TABLE(89b,189b)
EX_TABLE(90b,190b)
EX_TABLE(91b,191b)
EX_TABLE(92b,192b)
/*
* Routine to copy a whole page of data, optimized for POWER4.
* On POWER4 it is more than 50% faster than the simple loop
* above (following the .Ldst_aligned label).
*/
.Lcopy_page_4K:
std r31,-32(1)
std r30,-40(1)
std r29,-48(1)
std r28,-56(1)
std r27,-64(1)
std r26,-72(1)
std r25,-80(1)
std r24,-88(1)
std r23,-96(1)
std r22,-104(1)
std r21,-112(1)
std r20,-120(1)
li r5,4096/32 - 1
addi r3,r3,-8
li r0,5
0: addi r5,r5,-24
mtctr r0
20: ld r22,640(4)
21: ld r21,512(4)
22: ld r20,384(4)
23: ld r11,256(4)
24: ld r9,128(4)
25: ld r7,0(4)
26: ld r25,648(4)
27: ld r24,520(4)
28: ld r23,392(4)
29: ld r10,264(4)
30: ld r8,136(4)
31: ldu r6,8(4)
cmpwi r5,24
1:
32: std r22,648(3)
33: std r21,520(3)
34: std r20,392(3)
35: std r11,264(3)
36: std r9,136(3)
37: std r7,8(3)
38: ld r28,648(4)
39: ld r27,520(4)
40: ld r26,392(4)
41: ld r31,264(4)
42: ld r30,136(4)
43: ld r29,8(4)
44: std r25,656(3)
45: std r24,528(3)
46: std r23,400(3)
47: std r10,272(3)
48: std r8,144(3)
49: std r6,16(3)
50: ld r22,656(4)
51: ld r21,528(4)
52: ld r20,400(4)
53: ld r11,272(4)
54: ld r9,144(4)
55: ld r7,16(4)
56: std r28,664(3)
57: std r27,536(3)
58: std r26,408(3)
59: std r31,280(3)
60: std r30,152(3)
61: stdu r29,24(3)
62: ld r25,664(4)
63: ld r24,536(4)
64: ld r23,408(4)
65: ld r10,280(4)
66: ld r8,152(4)
67: ldu r6,24(4)
bdnz 1b
68: std r22,648(3)
69: std r21,520(3)
70: std r20,392(3)
71: std r11,264(3)
72: std r9,136(3)
73: std r7,8(3)
74: addi r4,r4,640
75: addi r3,r3,648
bge 0b
mtctr r5
76: ld r7,0(4)
77: ld r8,8(4)
78: ldu r9,16(4)
3:
79: ld r10,8(4)
80: std r7,8(3)
81: ld r7,16(4)
82: std r8,16(3)
83: ld r8,24(4)
84: std r9,24(3)
85: ldu r9,32(4)
86: stdu r10,32(3)
bdnz 3b
4:
87: ld r10,8(4)
88: std r7,8(3)
89: std r8,16(3)
90: std r9,24(3)
91: std r10,32(3)
9: ld r20,-120(1)
ld r21,-112(1)
ld r22,-104(1)
ld r23,-96(1)
ld r24,-88(1)
ld r25,-80(1)
ld r26,-72(1)
ld r27,-64(1)
ld r28,-56(1)
ld r29,-48(1)
ld r30,-40(1)
ld r31,-32(1)
li r3,0
blr
/*
* on an exception, reset to the beginning and jump back into the
* standard __copy_tofrom_user
*/
100: ld r20,-120(1)
ld r21,-112(1)
ld r22,-104(1)
ld r23,-96(1)
ld r24,-88(1)
ld r25,-80(1)
ld r26,-72(1)
ld r27,-64(1)
ld r28,-56(1)
ld r29,-48(1)
ld r30,-40(1)
ld r31,-32(1)
ld r3,-24(r1)
ld r4,-16(r1)
li r5,4096
b .Ldst_aligned
EX_TABLE(20b,100b)
EX_TABLE(21b,100b)
EX_TABLE(22b,100b)
EX_TABLE(23b,100b)
EX_TABLE(24b,100b)
EX_TABLE(25b,100b)
EX_TABLE(26b,100b)
EX_TABLE(27b,100b)
EX_TABLE(28b,100b)
EX_TABLE(29b,100b)
EX_TABLE(30b,100b)
EX_TABLE(31b,100b)
EX_TABLE(32b,100b)
EX_TABLE(33b,100b)
EX_TABLE(34b,100b)
EX_TABLE(35b,100b)
EX_TABLE(36b,100b)
EX_TABLE(37b,100b)
EX_TABLE(38b,100b)
EX_TABLE(39b,100b)
EX_TABLE(40b,100b)
EX_TABLE(41b,100b)
EX_TABLE(42b,100b)
EX_TABLE(43b,100b)
EX_TABLE(44b,100b)
EX_TABLE(45b,100b)
EX_TABLE(46b,100b)
EX_TABLE(47b,100b)
EX_TABLE(48b,100b)
EX_TABLE(49b,100b)
EX_TABLE(50b,100b)
EX_TABLE(51b,100b)
EX_TABLE(52b,100b)
EX_TABLE(53b,100b)
EX_TABLE(54b,100b)
EX_TABLE(55b,100b)
EX_TABLE(56b,100b)
EX_TABLE(57b,100b)
EX_TABLE(58b,100b)
EX_TABLE(59b,100b)
EX_TABLE(60b,100b)
EX_TABLE(61b,100b)
EX_TABLE(62b,100b)
EX_TABLE(63b,100b)
EX_TABLE(64b,100b)
EX_TABLE(65b,100b)
EX_TABLE(66b,100b)
EX_TABLE(67b,100b)
EX_TABLE(68b,100b)
EX_TABLE(69b,100b)
EX_TABLE(70b,100b)
EX_TABLE(71b,100b)
EX_TABLE(72b,100b)
EX_TABLE(73b,100b)
EX_TABLE(74b,100b)
EX_TABLE(75b,100b)
EX_TABLE(76b,100b)
EX_TABLE(77b,100b)
EX_TABLE(78b,100b)
EX_TABLE(79b,100b)
EX_TABLE(80b,100b)
EX_TABLE(81b,100b)
EX_TABLE(82b,100b)
EX_TABLE(83b,100b)
EX_TABLE(84b,100b)
EX_TABLE(85b,100b)
EX_TABLE(86b,100b)
EX_TABLE(87b,100b)
EX_TABLE(88b,100b)
EX_TABLE(89b,100b)
EX_TABLE(90b,100b)
EX_TABLE(91b,100b)
EXPORT_SYMBOL(__copy_tofrom_user)
