1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */ 2 3 /* 4 * Copyright (C) 2014 Rob Clark <robclark (at) freedesktop.org> 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Rob Clark <robclark (at) freedesktop.org> 27 */ 28 29 #include "util/u_math.h" 30 31 #include "freedreno_util.h" 32 33 #include "ir3.h" 34 35 /* 36 * Legalize: 37 * 38 * We currently require that scheduling ensures that we have enough nop's 39 * in all the right places. The legalize step mostly handles fixing up 40 * instruction flags ((ss)/(sy)/(ei)), and collapses sequences of nop's 41 * into fewer nop's w/ rpt flag. 42 */ 43 44 struct ir3_legalize_ctx { 45 bool has_samp; 46 bool has_ssbo; 47 int max_bary; 48 }; 49 50 /* We want to evaluate each block from the position of any other 51 * predecessor block, in order that the flags set are the union 52 * of all possible program paths. For stopping condition, we 53 * want to stop when the pair of <pred-block, current-block> has 54 * been visited already. 55 * 56 * XXX is that completely true? We could have different needs_xyz 57 * flags set depending on path leading to pred-block.. we could 58 * do *most* of this based on chasing src instructions ptrs (and 59 * following all phi srcs).. except the write-after-read hazzard. 60 * 61 * For now we just set ss/sy flag on first instruction on block, 62 * and handle everything within the block as before. 63 */ 64 65 static void 66 legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) 67 { 68 struct ir3_instruction *last_input = NULL; 69 struct ir3_instruction *last_rel = NULL; 70 struct ir3_instruction *last_n = NULL; 71 struct list_head instr_list; 72 regmask_t needs_ss_war; /* write after read */ 73 regmask_t needs_ss; 74 regmask_t needs_sy; 75 76 regmask_init(&needs_ss_war); 77 regmask_init(&needs_ss); 78 regmask_init(&needs_sy); 79 80 /* remove all the instructions from the list, we'll be adding 81 * them back in as we go 82 */ 83 list_replace(&block->instr_list, &instr_list); 84 list_inithead(&block->instr_list); 85 86 list_for_each_entry_safe (struct ir3_instruction, n, &instr_list, node) { 87 struct ir3_register *reg; 88 unsigned i; 89 90 if (is_meta(n)) 91 continue; 92 93 if (is_input(n)) { 94 struct ir3_register *inloc = n->regs[1]; 95 assert(inloc->flags & IR3_REG_IMMED); 96 ctx->max_bary = MAX2(ctx->max_bary, inloc->iim_val); 97 } 98 99 if (last_n && is_barrier(last_n)) 100 n->flags |= IR3_INSTR_SS | IR3_INSTR_SY; 101 102 /* NOTE: consider dst register too.. it could happen that 103 * texture sample instruction (for example) writes some 104 * components which are unused. A subsequent instruction 105 * that writes the same register can race w/ the sam instr 106 * resulting in undefined results: 107 */ 108 for (i = 0; i < n->regs_count; i++) { 109 reg = n->regs[i]; 110 111 if (reg_gpr(reg)) { 112 113 /* TODO: we probably only need (ss) for alu 114 * instr consuming sfu result.. need to make 115 * some tests for both this and (sy).. 116 */ 117 if (regmask_get(&needs_ss, reg)) { 118 n->flags |= IR3_INSTR_SS; 119 regmask_init(&needs_ss); 120 } 121 122 if (regmask_get(&needs_sy, reg)) { 123 n->flags |= IR3_INSTR_SY; 124 regmask_init(&needs_sy); 125 } 126 } 127 128 /* TODO: is it valid to have address reg loaded from a 129 * relative src (ie. mova a0, c<a0.x+4>)? If so, the 130 * last_rel check below should be moved ahead of this: 131 */ 132 if (reg->flags & IR3_REG_RELATIV) 133 last_rel = n; 134 } 135 136 if (n->regs_count > 0) { 137 reg = n->regs[0]; 138 if (regmask_get(&needs_ss_war, reg)) { 139 n->flags |= IR3_INSTR_SS; 140 regmask_init(&needs_ss_war); // ??? I assume? 141 } 142 143 if (last_rel && (reg->num == regid(REG_A0, 0))) { 144 last_rel->flags |= IR3_INSTR_UL; 145 last_rel = NULL; 146 } 147 } 148 149 /* cat5+ does not have an (ss) bit, if needed we need to 150 * insert a nop to carry the sync flag. Would be kinda 151 * clever if we were aware of this during scheduling, but 152 * this should be a pretty rare case: 153 */ 154 if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) { 155 struct ir3_instruction *nop; 156 nop = ir3_NOP(block); 157 nop->flags |= IR3_INSTR_SS; 158 n->flags &= ~IR3_INSTR_SS; 159 } 160 161 /* need to be able to set (ss) on first instruction: */ 162 if (list_empty(&block->instr_list) && (opc_cat(n->opc) >= 5)) 163 ir3_NOP(block); 164 165 if (is_nop(n) && !list_empty(&block->instr_list)) { 166 struct ir3_instruction *last = list_last_entry(&block->instr_list, 167 struct ir3_instruction, node); 168 if (is_nop(last) && (last->repeat < 5)) { 169 last->repeat++; 170 last->flags |= n->flags; 171 continue; 172 } 173 } 174 175 list_addtail(&n->node, &block->instr_list); 176 177 if (is_sfu(n)) 178 regmask_set(&needs_ss, n->regs[0]); 179 180 if (is_tex(n)) { 181 /* this ends up being the # of samp instructions.. but that 182 * is ok, everything else only cares whether it is zero or 183 * not. We do this here, rather than when we encounter a 184 * SAMP decl, because (especially in binning pass shader) 185 * the samp instruction(s) could get eliminated if the 186 * result is not used. 187 */ 188 ctx->has_samp = true; 189 regmask_set(&needs_sy, n->regs[0]); 190 } else if (n->opc == OPC_RESINFO) { 191 regmask_set(&needs_ss, n->regs[0]); 192 ir3_NOP(block)->flags |= IR3_INSTR_SS; 193 } else if (is_load(n)) { 194 /* seems like ldlv needs (ss) bit instead?? which is odd but 195 * makes a bunch of flat-varying tests start working on a4xx. 196 */ 197 if ((n->opc == OPC_LDLV) || (n->opc == OPC_LDL)) 198 regmask_set(&needs_ss, n->regs[0]); 199 else 200 regmask_set(&needs_sy, n->regs[0]); 201 } else if (is_atomic(n->opc)) { 202 if (n->flags & IR3_INSTR_G) 203 regmask_set(&needs_sy, n->regs[0]); 204 else 205 regmask_set(&needs_ss, n->regs[0]); 206 } 207 208 if (is_ssbo(n->opc) || (is_atomic(n->opc) && (n->flags & IR3_INSTR_G))) 209 ctx->has_ssbo = true; 210 211 /* both tex/sfu appear to not always immediately consume 212 * their src register(s): 213 */ 214 if (is_tex(n) || is_sfu(n) || is_mem(n)) { 215 foreach_src(reg, n) { 216 if (reg_gpr(reg)) 217 regmask_set(&needs_ss_war, reg); 218 } 219 } 220 221 if (is_input(n)) 222 last_input = n; 223 224 last_n = n; 225 } 226 227 if (last_input) { 228 /* special hack.. if using ldlv to bypass interpolation, 229 * we need to insert a dummy bary.f on which we can set 230 * the (ei) flag: 231 */ 232 if (is_mem(last_input) && (last_input->opc == OPC_LDLV)) { 233 struct ir3_instruction *baryf; 234 235 /* (ss)bary.f (ei)r63.x, 0, r0.x */ 236 baryf = ir3_instr_create(block, OPC_BARY_F); 237 baryf->flags |= IR3_INSTR_SS; 238 ir3_reg_create(baryf, regid(63, 0), 0); 239 ir3_reg_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0; 240 ir3_reg_create(baryf, regid(0, 0), 0); 241 242 /* insert the dummy bary.f after last_input: */ 243 list_delinit(&baryf->node); 244 list_add(&baryf->node, &last_input->node); 245 246 last_input = baryf; 247 } 248 last_input->regs[0]->flags |= IR3_REG_EI; 249 } 250 251 if (last_rel) 252 last_rel->flags |= IR3_INSTR_UL; 253 254 list_first_entry(&block->instr_list, struct ir3_instruction, node) 255 ->flags |= IR3_INSTR_SS | IR3_INSTR_SY; 256 } 257 258 /* NOTE: branch instructions are always the last instruction(s) 259 * in the block. We take advantage of this as we resolve the 260 * branches, since "if (foo) break;" constructs turn into 261 * something like: 262 * 263 * block3 { 264 * ... 265 * 0029:021: mov.s32s32 r62.x, r1.y 266 * 0082:022: br !p0.x, target=block5 267 * 0083:023: br p0.x, target=block4 268 * // succs: if _[0029:021: mov.s32s32] block4; else block5; 269 * } 270 * block4 { 271 * 0084:024: jump, target=block6 272 * // succs: block6; 273 * } 274 * block5 { 275 * 0085:025: jump, target=block7 276 * // succs: block7; 277 * } 278 * 279 * ie. only instruction in block4/block5 is a jump, so when 280 * resolving branches we can easily detect this by checking 281 * that the first instruction in the target block is itself 282 * a jump, and setup the br directly to the jump's target 283 * (and strip back out the now unreached jump) 284 * 285 * TODO sometimes we end up with things like: 286 * 287 * br !p0.x, #2 288 * br p0.x, #12 289 * add.u r0.y, r0.y, 1 290 * 291 * If we swapped the order of the branches, we could drop one. 292 */ 293 static struct ir3_block * 294 resolve_dest_block(struct ir3_block *block) 295 { 296 /* special case for last block: */ 297 if (!block->successors[0]) 298 return block; 299 300 /* NOTE that we may or may not have inserted the jump 301 * in the target block yet, so conditions to resolve 302 * the dest to the dest block's successor are: 303 * 304 * (1) successor[1] == NULL && 305 * (2) (block-is-empty || only-instr-is-jump) 306 */ 307 if (block->successors[1] == NULL) { 308 if (list_empty(&block->instr_list)) { 309 return block->successors[0]; 310 } else if (list_length(&block->instr_list) == 1) { 311 struct ir3_instruction *instr = list_first_entry( 312 &block->instr_list, struct ir3_instruction, node); 313 if (instr->opc == OPC_JUMP) 314 return block->successors[0]; 315 } 316 } 317 return block; 318 } 319 320 static bool 321 resolve_jump(struct ir3_instruction *instr) 322 { 323 struct ir3_block *tblock = 324 resolve_dest_block(instr->cat0.target); 325 struct ir3_instruction *target; 326 327 if (tblock != instr->cat0.target) { 328 list_delinit(&instr->cat0.target->node); 329 instr->cat0.target = tblock; 330 return true; 331 } 332 333 target = list_first_entry(&tblock->instr_list, 334 struct ir3_instruction, node); 335 336 if ((!target) || (target->ip == (instr->ip + 1))) { 337 list_delinit(&instr->node); 338 return true; 339 } else { 340 instr->cat0.immed = 341 (int)target->ip - (int)instr->ip; 342 } 343 return false; 344 } 345 346 /* resolve jumps, removing jumps/branches to immediately following 347 * instruction which we end up with from earlier stages. Since 348 * removing an instruction can invalidate earlier instruction's 349 * branch offsets, we need to do this iteratively until no more 350 * branches are removed. 351 */ 352 static bool 353 resolve_jumps(struct ir3 *ir) 354 { 355 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) 356 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) 357 if (is_flow(instr) && instr->cat0.target) 358 if (resolve_jump(instr)) 359 return true; 360 361 return false; 362 } 363 364 /* we want to mark points where divergent flow control re-converges 365 * with (jp) flags. For now, since we don't do any optimization for 366 * things that start out as a 'do {} while()', re-convergence points 367 * will always be a branch or jump target. Note that this is overly 368 * conservative, since unconditional jump targets are not convergence 369 * points, we are just assuming that the other path to reach the jump 370 * target was divergent. If we were clever enough to optimize the 371 * jump at end of a loop back to a conditional branch into a single 372 * conditional branch, ie. like: 373 * 374 * add.f r1.w, r0.x, (neg)(r)c2.x <= loop start 375 * mul.f r1.z, r1.z, r0.x 376 * mul.f r1.y, r1.y, r0.x 377 * mul.f r0.z, r1.x, r0.x 378 * mul.f r0.w, r0.y, r0.x 379 * cmps.f.ge r0.x, (r)c2.y, (r)r1.w 380 * add.s r0.x, (r)r0.x, (r)-1 381 * sel.f32 r0.x, (r)c3.y, (r)r0.x, c3.x 382 * cmps.f.eq p0.x, r0.x, c3.y 383 * mov.f32f32 r0.x, r1.w 384 * mov.f32f32 r0.y, r0.w 385 * mov.f32f32 r1.x, r0.z 386 * (rpt2)nop 387 * br !p0.x, #-13 388 * (jp)mul.f r0.x, c263.y, r1.y 389 * 390 * Then we'd have to be more clever, as the convergence point is no 391 * longer a branch or jump target. 392 */ 393 static void 394 mark_convergence_points(struct ir3 *ir) 395 { 396 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) { 397 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) { 398 if (is_flow(instr) && instr->cat0.target) { 399 struct ir3_instruction *target = 400 list_first_entry(&instr->cat0.target->instr_list, 401 struct ir3_instruction, node); 402 target->flags |= IR3_INSTR_JP; 403 } 404 } 405 } 406 } 407 408 void 409 ir3_legalize(struct ir3 *ir, bool *has_samp, bool *has_ssbo, int *max_bary) 410 { 411 struct ir3_legalize_ctx ctx = { 412 .max_bary = -1, 413 }; 414 415 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) { 416 legalize_block(&ctx, block); 417 } 418 419 *has_samp = ctx.has_samp; 420 *has_ssbo = ctx.has_ssbo; 421 *max_bary = ctx.max_bary; 422 423 do { 424 ir3_count_instructions(ir); 425 } while(resolve_jumps(ir)); 426 427 mark_convergence_points(ir); 428 } 429
