1 /* 2 * Copyright (c) 2016-2017, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <arch.h> 8 #include <arch_helpers.h> 9 #include <assert.h> 10 #include <cassert.h> 11 #include <common_def.h> 12 #include <debug.h> 13 #include <platform_def.h> 14 #include <string.h> 15 #include <types.h> 16 #include <utils.h> 17 #include <xlat_tables.h> 18 #include "xlat_tables_private.h" 19 20 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 21 #define LVL0_SPACER "" 22 #define LVL1_SPACER " " 23 #define LVL2_SPACER " " 24 #define LVL3_SPACER " " 25 #define get_level_spacer(level) \ 26 (((level) == U(0)) ? LVL0_SPACER : \ 27 (((level) == U(1)) ? LVL1_SPACER : \ 28 (((level) == U(2)) ? LVL2_SPACER : LVL3_SPACER))) 29 #define debug_print(...) tf_printf(__VA_ARGS__) 30 #else 31 #define debug_print(...) ((void)0) 32 #endif 33 34 #define UNSET_DESC ~0ull 35 36 static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES] 37 __aligned(XLAT_TABLE_SIZE) __section("xlat_table"); 38 39 static unsigned int next_xlat; 40 static unsigned long long xlat_max_pa; 41 static uintptr_t xlat_max_va; 42 43 static uint64_t execute_never_mask; 44 45 /* 46 * Array of all memory regions stored in order of ascending base address. 47 * The list is terminated by the first entry with size == 0. 48 */ 49 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; 50 51 52 void print_mmap(void) 53 { 54 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 55 debug_print("mmap:\n"); 56 mmap_region_t *mm = mmap; 57 while (mm->size) { 58 debug_print(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", 59 (void *)mm->base_va, mm->base_pa, 60 mm->size, mm->attr); 61 ++mm; 62 }; 63 debug_print("\n"); 64 #endif 65 } 66 67 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, 68 size_t size, mmap_attr_t attr) 69 { 70 mmap_region_t *mm = mmap; 71 mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1; 72 unsigned long long end_pa = base_pa + size - 1; 73 uintptr_t end_va = base_va + size - 1; 74 75 assert(IS_PAGE_ALIGNED(base_pa)); 76 assert(IS_PAGE_ALIGNED(base_va)); 77 assert(IS_PAGE_ALIGNED(size)); 78 79 if (!size) 80 return; 81 82 assert(base_pa < end_pa); /* Check for overflows */ 83 assert(base_va < end_va); 84 85 assert((base_va + (uintptr_t)size - (uintptr_t)1) <= 86 (PLAT_VIRT_ADDR_SPACE_SIZE - 1)); 87 assert((base_pa + (unsigned long long)size - 1ULL) <= 88 (PLAT_PHY_ADDR_SPACE_SIZE - 1)); 89 90 #if ENABLE_ASSERTIONS 91 92 /* Check for PAs and VAs overlaps with all other regions */ 93 for (mm = mmap; mm->size; ++mm) { 94 95 uintptr_t mm_end_va = mm->base_va + mm->size - 1; 96 97 /* 98 * Check if one of the regions is completely inside the other 99 * one. 100 */ 101 int fully_overlapped_va = 102 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) || 103 ((mm->base_va >= base_va) && (mm_end_va <= end_va)); 104 105 /* 106 * Full VA overlaps are only allowed if both regions are 107 * identity mapped (zero offset) or have the same VA to PA 108 * offset. Also, make sure that it's not the exact same area. 109 */ 110 if (fully_overlapped_va) { 111 assert((mm->base_va - mm->base_pa) == 112 (base_va - base_pa)); 113 assert((base_va != mm->base_va) || (size != mm->size)); 114 } else { 115 /* 116 * If the regions do not have fully overlapping VAs, 117 * then they must have fully separated VAs and PAs. 118 * Partial overlaps are not allowed 119 */ 120 121 unsigned long long mm_end_pa = 122 mm->base_pa + mm->size - 1; 123 124 int separated_pa = 125 (end_pa < mm->base_pa) || (base_pa > mm_end_pa); 126 int separated_va = 127 (end_va < mm->base_va) || (base_va > mm_end_va); 128 129 assert(separated_va && separated_pa); 130 } 131 } 132 133 mm = mmap; /* Restore pointer to the start of the array */ 134 135 #endif /* ENABLE_ASSERTIONS */ 136 137 /* Find correct place in mmap to insert new region */ 138 while (mm->base_va < base_va && mm->size) 139 ++mm; 140 141 /* 142 * If a section is contained inside another one with the same base 143 * address, it must be placed after the one it is contained in: 144 * 145 * 1st |-----------------------| 146 * 2nd |------------| 147 * 3rd |------| 148 * 149 * This is required for mmap_region_attr() to get the attributes of the 150 * small region correctly. 151 */ 152 while ((mm->base_va == base_va) && (mm->size > size)) 153 ++mm; 154 155 /* Make room for new region by moving other regions up by one place */ 156 memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); 157 158 /* Check we haven't lost the empty sentinal from the end of the array */ 159 assert(mm_last->size == 0); 160 161 mm->base_pa = base_pa; 162 mm->base_va = base_va; 163 mm->size = size; 164 mm->attr = attr; 165 166 if (end_pa > xlat_max_pa) 167 xlat_max_pa = end_pa; 168 if (end_va > xlat_max_va) 169 xlat_max_va = end_va; 170 } 171 172 void mmap_add(const mmap_region_t *mm) 173 { 174 while (mm->size) { 175 mmap_add_region(mm->base_pa, mm->base_va, mm->size, mm->attr); 176 ++mm; 177 } 178 } 179 180 static uint64_t mmap_desc(mmap_attr_t attr, unsigned long long addr_pa, 181 unsigned int level) 182 { 183 uint64_t desc; 184 int mem_type; 185 186 /* Make sure that the granularity is fine enough to map this address. */ 187 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0); 188 189 desc = addr_pa; 190 /* 191 * There are different translation table descriptors for level 3 and the 192 * rest. 193 */ 194 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; 195 desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; 196 desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); 197 desc |= LOWER_ATTRS(ACCESS_FLAG); 198 199 /* 200 * Deduce shareability domain and executability of the memory region 201 * from the memory type. 202 * 203 * Data accesses to device memory and non-cacheable normal memory are 204 * coherent for all observers in the system, and correspondingly are 205 * always treated as being Outer Shareable. Therefore, for these 2 types 206 * of memory, it is not strictly needed to set the shareability field 207 * in the translation tables. 208 */ 209 mem_type = MT_TYPE(attr); 210 if (mem_type == MT_DEVICE) { 211 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); 212 /* 213 * Always map device memory as execute-never. 214 * This is to avoid the possibility of a speculative instruction 215 * fetch, which could be an issue if this memory region 216 * corresponds to a read-sensitive peripheral. 217 */ 218 desc |= execute_never_mask; 219 220 } else { /* Normal memory */ 221 /* 222 * Always map read-write normal memory as execute-never. 223 * (Trusted Firmware doesn't self-modify its code, therefore 224 * R/W memory is reserved for data storage, which must not be 225 * executable.) 226 * Note that setting the XN bit here is for consistency only. 227 * The function that enables the MMU sets the SCTLR_ELx.WXN bit, 228 * which makes any writable memory region to be treated as 229 * execute-never, regardless of the value of the XN bit in the 230 * translation table. 231 * 232 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER 233 * attribute to figure out the value of the XN bit. 234 */ 235 if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) { 236 desc |= execute_never_mask; 237 } 238 239 if (mem_type == MT_MEMORY) { 240 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); 241 } else { 242 assert(mem_type == MT_NON_CACHEABLE); 243 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); 244 } 245 } 246 247 debug_print((mem_type == MT_MEMORY) ? "MEM" : 248 ((mem_type == MT_NON_CACHEABLE) ? "NC" : "DEV")); 249 debug_print(attr & MT_RW ? "-RW" : "-RO"); 250 debug_print(attr & MT_NS ? "-NS" : "-S"); 251 debug_print(attr & MT_EXECUTE_NEVER ? "-XN" : "-EXEC"); 252 return desc; 253 } 254 255 /* 256 * Look for the innermost region that contains the area at `base_va` with size 257 * `size`. Populate *attr with the attributes of this region. 258 * 259 * On success, this function returns 0. 260 * If there are partial overlaps (meaning that a smaller size is needed) or if 261 * the region can't be found in the given area, it returns -1. In this case the 262 * value pointed by attr should be ignored by the caller. 263 */ 264 static int mmap_region_attr(mmap_region_t *mm, uintptr_t base_va, 265 size_t size, mmap_attr_t *attr) 266 { 267 /* Don't assume that the area is contained in the first region */ 268 int ret = -1; 269 270 /* 271 * Get attributes from last (innermost) region that contains the 272 * requested area. Don't stop as soon as one region doesn't contain it 273 * because there may be other internal regions that contain this area: 274 * 275 * |-----------------------------1-----------------------------| 276 * |----2----| |-------3-------| |----5----| 277 * |--4--| 278 * 279 * |---| <- Area we want the attributes of. 280 * 281 * In this example, the area is contained in regions 1, 3 and 4 but not 282 * in region 2. The loop shouldn't stop at region 2 as inner regions 283 * have priority over outer regions, it should stop at region 5. 284 */ 285 for (;; ++mm) { 286 287 if (!mm->size) 288 return ret; /* Reached end of list */ 289 290 if (mm->base_va > base_va + size - 1) 291 return ret; /* Next region is after area so end */ 292 293 if (mm->base_va + mm->size - 1 < base_va) 294 continue; /* Next region has already been overtaken */ 295 296 if (!ret && mm->attr == *attr) 297 continue; /* Region doesn't override attribs so skip */ 298 299 if (mm->base_va > base_va || 300 mm->base_va + mm->size - 1 < base_va + size - 1) 301 return -1; /* Region doesn't fully cover our area */ 302 303 *attr = mm->attr; 304 ret = 0; 305 } 306 return ret; 307 } 308 309 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, 310 uintptr_t base_va, 311 uint64_t *table, 312 unsigned int level) 313 { 314 assert(level >= XLAT_TABLE_LEVEL_MIN && level <= XLAT_TABLE_LEVEL_MAX); 315 316 unsigned int level_size_shift = 317 L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT; 318 u_register_t level_size = (u_register_t)1 << level_size_shift; 319 u_register_t level_index_mask = 320 ((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift; 321 322 debug_print("New xlat table:\n"); 323 324 do { 325 uint64_t desc = UNSET_DESC; 326 327 if (!mm->size) { 328 /* Done mapping regions; finish zeroing the table */ 329 desc = INVALID_DESC; 330 } else if (mm->base_va + mm->size - 1 < base_va) { 331 /* This area is after the region so get next region */ 332 ++mm; 333 continue; 334 } 335 336 debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level), 337 (void *)base_va, (unsigned long long)level_size); 338 339 if (mm->base_va > base_va + level_size - 1) { 340 /* Next region is after this area. Nothing to map yet */ 341 desc = INVALID_DESC; 342 /* Make sure that the current level allows block descriptors */ 343 } else if (level >= XLAT_BLOCK_LEVEL_MIN) { 344 /* 345 * Try to get attributes of this area. It will fail if 346 * there are partially overlapping regions. On success, 347 * it will return the innermost region's attributes. 348 */ 349 mmap_attr_t attr; 350 int r = mmap_region_attr(mm, base_va, level_size, &attr); 351 352 if (!r) { 353 desc = mmap_desc(attr, 354 base_va - mm->base_va + mm->base_pa, 355 level); 356 } 357 } 358 359 if (desc == UNSET_DESC) { 360 /* Area not covered by a region so need finer table */ 361 uint64_t *new_table = xlat_tables[next_xlat++]; 362 assert(next_xlat <= MAX_XLAT_TABLES); 363 desc = TABLE_DESC | (uintptr_t)new_table; 364 365 /* Recurse to fill in new table */ 366 mm = init_xlation_table_inner(mm, base_va, 367 new_table, level+1); 368 } 369 370 debug_print("\n"); 371 372 *table++ = desc; 373 base_va += level_size; 374 } while ((base_va & level_index_mask) && 375 (base_va - 1 < PLAT_VIRT_ADDR_SPACE_SIZE - 1)); 376 377 return mm; 378 } 379 380 void init_xlation_table(uintptr_t base_va, uint64_t *table, 381 unsigned int level, uintptr_t *max_va, 382 unsigned long long *max_pa) 383 { 384 execute_never_mask = xlat_arch_get_xn_desc(xlat_arch_current_el()); 385 init_xlation_table_inner(mmap, base_va, table, level); 386 *max_va = xlat_max_va; 387 *max_pa = xlat_max_pa; 388 } 389
