1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * (C) Copyright 2013 4 * David Feng <fenghua (at) phytium.com.cn> 5 * 6 * This file is based on sample code from ARMv8 ARM. 7 */ 8 9 #include <asm-offsets.h> 10 #include <config.h> 11 #include <asm/macro.h> 12 #include <asm/system.h> 13 #include <linux/linkage.h> 14 15 /* 16 * void __asm_dcache_level(level) 17 * 18 * flush or invalidate one level cache. 19 * 20 * x0: cache level 21 * x1: 0 clean & invalidate, 1 invalidate only 22 * x2~x9: clobbered 23 */ 24 .pushsection .text.__asm_dcache_level, "ax" 25 ENTRY(__asm_dcache_level) 26 lsl x12, x0, #1 27 msr csselr_el1, x12 /* select cache level */ 28 isb /* sync change of cssidr_el1 */ 29 mrs x6, ccsidr_el1 /* read the new cssidr_el1 */ 30 and x2, x6, #7 /* x2 <- log2(cache line size)-4 */ 31 add x2, x2, #4 /* x2 <- log2(cache line size) */ 32 mov x3, #0x3ff 33 and x3, x3, x6, lsr #3 /* x3 <- max number of #ways */ 34 clz w5, w3 /* bit position of #ways */ 35 mov x4, #0x7fff 36 and x4, x4, x6, lsr #13 /* x4 <- max number of #sets */ 37 /* x12 <- cache level << 1 */ 38 /* x2 <- line length offset */ 39 /* x3 <- number of cache ways - 1 */ 40 /* x4 <- number of cache sets - 1 */ 41 /* x5 <- bit position of #ways */ 42 43 loop_set: 44 mov x6, x3 /* x6 <- working copy of #ways */ 45 loop_way: 46 lsl x7, x6, x5 47 orr x9, x12, x7 /* map way and level to cisw value */ 48 lsl x7, x4, x2 49 orr x9, x9, x7 /* map set number to cisw value */ 50 tbz w1, #0, 1f 51 dc isw, x9 52 b 2f 53 1: dc cisw, x9 /* clean & invalidate by set/way */ 54 2: subs x6, x6, #1 /* decrement the way */ 55 b.ge loop_way 56 subs x4, x4, #1 /* decrement the set */ 57 b.ge loop_set 58 59 ret 60 ENDPROC(__asm_dcache_level) 61 .popsection 62 63 /* 64 * void __asm_flush_dcache_all(int invalidate_only) 65 * 66 * x0: 0 clean & invalidate, 1 invalidate only 67 * 68 * flush or invalidate all data cache by SET/WAY. 69 */ 70 .pushsection .text.__asm_dcache_all, "ax" 71 ENTRY(__asm_dcache_all) 72 mov x1, x0 73 dsb sy 74 mrs x10, clidr_el1 /* read clidr_el1 */ 75 lsr x11, x10, #24 76 and x11, x11, #0x7 /* x11 <- loc */ 77 cbz x11, finished /* if loc is 0, exit */ 78 mov x15, lr 79 mov x0, #0 /* start flush at cache level 0 */ 80 /* x0 <- cache level */ 81 /* x10 <- clidr_el1 */ 82 /* x11 <- loc */ 83 /* x15 <- return address */ 84 85 loop_level: 86 lsl x12, x0, #1 87 add x12, x12, x0 /* x0 <- tripled cache level */ 88 lsr x12, x10, x12 89 and x12, x12, #7 /* x12 <- cache type */ 90 cmp x12, #2 91 b.lt skip /* skip if no cache or icache */ 92 bl __asm_dcache_level /* x1 = 0 flush, 1 invalidate */ 93 skip: 94 add x0, x0, #1 /* increment cache level */ 95 cmp x11, x0 96 b.gt loop_level 97 98 mov x0, #0 99 msr csselr_el1, x0 /* restore csselr_el1 */ 100 dsb sy 101 isb 102 mov lr, x15 103 104 finished: 105 ret 106 ENDPROC(__asm_dcache_all) 107 .popsection 108 109 .pushsection .text.__asm_flush_dcache_all, "ax" 110 ENTRY(__asm_flush_dcache_all) 111 mov x0, #0 112 b __asm_dcache_all 113 ENDPROC(__asm_flush_dcache_all) 114 .popsection 115 116 .pushsection .text.__asm_invalidate_dcache_all, "ax" 117 ENTRY(__asm_invalidate_dcache_all) 118 mov x0, #0x1 119 b __asm_dcache_all 120 ENDPROC(__asm_invalidate_dcache_all) 121 .popsection 122 123 /* 124 * void __asm_flush_dcache_range(start, end) 125 * 126 * clean & invalidate data cache in the range 127 * 128 * x0: start address 129 * x1: end address 130 */ 131 .pushsection .text.__asm_flush_dcache_range, "ax" 132 ENTRY(__asm_flush_dcache_range) 133 mrs x3, ctr_el0 134 lsr x3, x3, #16 135 and x3, x3, #0xf 136 mov x2, #4 137 lsl x2, x2, x3 /* cache line size */ 138 139 /* x2 <- minimal cache line size in cache system */ 140 sub x3, x2, #1 141 bic x0, x0, x3 142 1: dc civac, x0 /* clean & invalidate data or unified cache */ 143 add x0, x0, x2 144 cmp x0, x1 145 b.lo 1b 146 dsb sy 147 ret 148 ENDPROC(__asm_flush_dcache_range) 149 .popsection 150 /* 151 * void __asm_invalidate_dcache_range(start, end) 152 * 153 * invalidate data cache in the range 154 * 155 * x0: start address 156 * x1: end address 157 */ 158 .pushsection .text.__asm_invalidate_dcache_range, "ax" 159 ENTRY(__asm_invalidate_dcache_range) 160 mrs x3, ctr_el0 161 ubfm x3, x3, #16, #19 162 mov x2, #4 163 lsl x2, x2, x3 /* cache line size */ 164 165 /* x2 <- minimal cache line size in cache system */ 166 sub x3, x2, #1 167 bic x0, x0, x3 168 1: dc ivac, x0 /* invalidate data or unified cache */ 169 add x0, x0, x2 170 cmp x0, x1 171 b.lo 1b 172 dsb sy 173 ret 174 ENDPROC(__asm_invalidate_dcache_range) 175 .popsection 176 177 /* 178 * void __asm_invalidate_icache_all(void) 179 * 180 * invalidate all tlb entries. 181 */ 182 .pushsection .text.__asm_invalidate_icache_all, "ax" 183 ENTRY(__asm_invalidate_icache_all) 184 ic ialluis 185 isb sy 186 ret 187 ENDPROC(__asm_invalidate_icache_all) 188 .popsection 189 190 .pushsection .text.__asm_invalidate_l3_dcache, "ax" 191 ENTRY(__asm_invalidate_l3_dcache) 192 mov x0, #0 /* return status as success */ 193 ret 194 ENDPROC(__asm_invalidate_l3_dcache) 195 .weak __asm_invalidate_l3_dcache 196 .popsection 197 198 .pushsection .text.__asm_flush_l3_dcache, "ax" 199 ENTRY(__asm_flush_l3_dcache) 200 mov x0, #0 /* return status as success */ 201 ret 202 ENDPROC(__asm_flush_l3_dcache) 203 .weak __asm_flush_l3_dcache 204 .popsection 205 206 .pushsection .text.__asm_invalidate_l3_icache, "ax" 207 ENTRY(__asm_invalidate_l3_icache) 208 mov x0, #0 /* return status as success */ 209 ret 210 ENDPROC(__asm_invalidate_l3_icache) 211 .weak __asm_invalidate_l3_icache 212 .popsection 213 214 /* 215 * void __asm_switch_ttbr(ulong new_ttbr) 216 * 217 * Safely switches to a new page table. 218 */ 219 .pushsection .text.__asm_switch_ttbr, "ax" 220 ENTRY(__asm_switch_ttbr) 221 /* x2 = SCTLR (alive throghout the function) */ 222 switch_el x4, 3f, 2f, 1f 223 3: mrs x2, sctlr_el3 224 b 0f 225 2: mrs x2, sctlr_el2 226 b 0f 227 1: mrs x2, sctlr_el1 228 0: 229 230 /* Unset CR_M | CR_C | CR_I from SCTLR to disable all caches */ 231 movn x1, #(CR_M | CR_C | CR_I) 232 and x1, x2, x1 233 switch_el x4, 3f, 2f, 1f 234 3: msr sctlr_el3, x1 235 b 0f 236 2: msr sctlr_el2, x1 237 b 0f 238 1: msr sctlr_el1, x1 239 0: isb 240 241 /* This call only clobbers x30 (lr) and x9 (unused) */ 242 mov x3, x30 243 bl __asm_invalidate_tlb_all 244 245 /* From here on we're running safely with caches disabled */ 246 247 /* Set TTBR to our first argument */ 248 switch_el x4, 3f, 2f, 1f 249 3: msr ttbr0_el3, x0 250 b 0f 251 2: msr ttbr0_el2, x0 252 b 0f 253 1: msr ttbr0_el1, x0 254 0: isb 255 256 /* Restore original SCTLR and thus enable caches again */ 257 switch_el x4, 3f, 2f, 1f 258 3: msr sctlr_el3, x2 259 b 0f 260 2: msr sctlr_el2, x2 261 b 0f 262 1: msr sctlr_el1, x2 263 0: isb 264 265 ret x3 266 ENDPROC(__asm_switch_ttbr) 267 .popsection 268
