1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * linux/arch/arm/lib/memcpy.S 4 * 5 * Author: Nicolas Pitre 6 * Created: Sep 28, 2005 7 * Copyright: MontaVista Software, Inc. 8 */ 9 10 #include <linux/linkage.h> 11 #include <asm/assembler.h> 12 13 #define LDR1W_SHIFT 0 14 #define STR1W_SHIFT 0 15 16 .macro ldr1w ptr reg abort 17 W(ldr) \reg, [\ptr], #4 18 .endm 19 20 .macro ldr4w ptr reg1 reg2 reg3 reg4 abort 21 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4} 22 .endm 23 24 .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 25 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 26 .endm 27 28 .macro ldr1b ptr reg cond=al abort 29 ldrb\cond\() \reg, [\ptr], #1 30 .endm 31 32 .macro str1w ptr reg abort 33 W(str) \reg, [\ptr], #4 34 .endm 35 36 .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 37 stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 38 .endm 39 40 .macro str1b ptr reg cond=al abort 41 strb\cond\() \reg, [\ptr], #1 42 .endm 43 44 .macro enter reg1 reg2 45 stmdb sp!, {r0, \reg1, \reg2} 46 .endm 47 48 .macro exit reg1 reg2 49 ldmfd sp!, {r0, \reg1, \reg2} 50 .endm 51 52 .text 53 54 /* Prototype: void *memcpy(void *dest, const void *src, size_t n); */ 55 .syntax unified 56 #if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD) 57 .thumb 58 .thumb_func 59 #endif 60 ENTRY(memcpy) 61 cmp r0, r1 62 moveq pc, lr 63 64 enter r4, lr 65 66 subs r2, r2, #4 67 blt 8f 68 ands ip, r0, #3 69 PLD( pld [r1, #0] ) 70 bne 9f 71 ands ip, r1, #3 72 bne 10f 73 74 1: subs r2, r2, #(28) 75 stmfd sp!, {r5 - r8} 76 blt 5f 77 78 CALGN( ands ip, r0, #31 ) 79 CALGN( rsb r3, ip, #32 ) 80 CALGN( sbcsne r4, r3, r2 ) @ C is always set here 81 CALGN( bcs 2f ) 82 CALGN( adr r4, 6f ) 83 CALGN( subs r2, r2, r3 ) @ C gets set 84 CALGN( add pc, r4, ip ) 85 86 PLD( pld [r1, #0] ) 87 2: PLD( subs r2, r2, #96 ) 88 PLD( pld [r1, #28] ) 89 PLD( blt 4f ) 90 PLD( pld [r1, #60] ) 91 PLD( pld [r1, #92] ) 92 93 3: PLD( pld [r1, #124] ) 94 4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 95 subs r2, r2, #32 96 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 97 bge 3b 98 PLD( cmn r2, #96 ) 99 PLD( bge 4b ) 100 101 5: ands ip, r2, #28 102 rsb ip, ip, #32 103 #if LDR1W_SHIFT > 0 104 lsl ip, ip, #LDR1W_SHIFT 105 #endif 106 addne pc, pc, ip @ C is always clear here 107 b 7f 108 6: 109 .rept (1 << LDR1W_SHIFT) 110 W(nop) 111 .endr 112 ldr1w r1, r3, abort=20f 113 ldr1w r1, r4, abort=20f 114 ldr1w r1, r5, abort=20f 115 ldr1w r1, r6, abort=20f 116 ldr1w r1, r7, abort=20f 117 ldr1w r1, r8, abort=20f 118 ldr1w r1, lr, abort=20f 119 120 #if LDR1W_SHIFT < STR1W_SHIFT 121 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT 122 #elif LDR1W_SHIFT > STR1W_SHIFT 123 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT 124 #endif 125 add pc, pc, ip 126 nop 127 .rept (1 << STR1W_SHIFT) 128 W(nop) 129 .endr 130 str1w r0, r3, abort=20f 131 str1w r0, r4, abort=20f 132 str1w r0, r5, abort=20f 133 str1w r0, r6, abort=20f 134 str1w r0, r7, abort=20f 135 str1w r0, r8, abort=20f 136 str1w r0, lr, abort=20f 137 138 CALGN( bcs 2b ) 139 140 7: ldmfd sp!, {r5 - r8} 141 142 8: movs r2, r2, lsl #31 143 ldr1b r1, r3, ne, abort=21f 144 ldr1b r1, r4, cs, abort=21f 145 ldr1b r1, ip, cs, abort=21f 146 str1b r0, r3, ne, abort=21f 147 str1b r0, r4, cs, abort=21f 148 str1b r0, ip, cs, abort=21f 149 150 exit r4, pc 151 152 9: rsb ip, ip, #4 153 cmp ip, #2 154 ldr1b r1, r3, gt, abort=21f 155 ldr1b r1, r4, ge, abort=21f 156 ldr1b r1, lr, abort=21f 157 str1b r0, r3, gt, abort=21f 158 str1b r0, r4, ge, abort=21f 159 subs r2, r2, ip 160 str1b r0, lr, abort=21f 161 blt 8b 162 ands ip, r1, #3 163 beq 1b 164 165 10: bic r1, r1, #3 166 cmp ip, #2 167 ldr1w r1, lr, abort=21f 168 beq 17f 169 bgt 18f 170 171 172 .macro forward_copy_shift pull push 173 174 subs r2, r2, #28 175 blt 14f 176 177 CALGN( ands ip, r0, #31 ) 178 CALGN( rsb ip, ip, #32 ) 179 CALGN( sbcsne r4, ip, r2 ) @ C is always set here 180 CALGN( subcc r2, r2, ip ) 181 CALGN( bcc 15f ) 182 183 11: stmfd sp!, {r5 - r9} 184 185 PLD( pld [r1, #0] ) 186 PLD( subs r2, r2, #96 ) 187 PLD( pld [r1, #28] ) 188 PLD( blt 13f ) 189 PLD( pld [r1, #60] ) 190 PLD( pld [r1, #92] ) 191 192 12: PLD( pld [r1, #124] ) 193 13: ldr4w r1, r4, r5, r6, r7, abort=19f 194 mov r3, lr, lspull #\pull 195 subs r2, r2, #32 196 ldr4w r1, r8, r9, ip, lr, abort=19f 197 orr r3, r3, r4, lspush #\push 198 mov r4, r4, lspull #\pull 199 orr r4, r4, r5, lspush #\push 200 mov r5, r5, lspull #\pull 201 orr r5, r5, r6, lspush #\push 202 mov r6, r6, lspull #\pull 203 orr r6, r6, r7, lspush #\push 204 mov r7, r7, lspull #\pull 205 orr r7, r7, r8, lspush #\push 206 mov r8, r8, lspull #\pull 207 orr r8, r8, r9, lspush #\push 208 mov r9, r9, lspull #\pull 209 orr r9, r9, ip, lspush #\push 210 mov ip, ip, lspull #\pull 211 orr ip, ip, lr, lspush #\push 212 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f 213 bge 12b 214 PLD( cmn r2, #96 ) 215 PLD( bge 13b ) 216 217 ldmfd sp!, {r5 - r9} 218 219 14: ands ip, r2, #28 220 beq 16f 221 222 15: mov r3, lr, lspull #\pull 223 ldr1w r1, lr, abort=21f 224 subs ip, ip, #4 225 orr r3, r3, lr, lspush #\push 226 str1w r0, r3, abort=21f 227 bgt 15b 228 CALGN( cmp r2, #0 ) 229 CALGN( bge 11b ) 230 231 16: sub r1, r1, #(\push / 8) 232 b 8b 233 234 .endm 235 236 237 forward_copy_shift pull=8 push=24 238 239 17: forward_copy_shift pull=16 push=16 240 241 18: forward_copy_shift pull=24 push=8 242 243 244 /* 245 * Abort preamble and completion macros. 246 * If a fixup handler is required then those macros must surround it. 247 * It is assumed that the fixup code will handle the private part of 248 * the exit macro. 249 */ 250 251 .macro copy_abort_preamble 252 19: ldmfd sp!, {r5 - r9} 253 b 21f 254 20: ldmfd sp!, {r5 - r8} 255 21: 256 .endm 257 258 .macro copy_abort_end 259 ldmfd sp!, {r4, pc} 260 .endm 261 262 ENDPROC(memcpy) 263
