1 /* 2 * Copyright 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 /** @file brw_fs_combine_constants.cpp 25 * 26 * This file contains the opt_combine_constants() pass that runs after the 27 * regular optimization loop. It passes over the instruction list and 28 * selectively promotes immediate values to registers by emitting a mov(1) 29 * instruction. 30 * 31 * This is useful on Gen 7 particularly, because a few instructions can be 32 * coissued (i.e., issued in the same cycle as another thread on the same EU 33 * issues an instruction) under some circumstances, one of which is that they 34 * cannot use immediate values. 35 */ 36 37 #include "brw_fs.h" 38 #include "brw_cfg.h" 39 40 using namespace brw; 41 42 static const bool debug = false; 43 44 /* Returns whether an instruction could co-issue if its immediate source were 45 * replaced with a GRF source. 46 */ 47 static bool 48 could_coissue(const struct gen_device_info *devinfo, const fs_inst *inst) 49 { 50 if (devinfo->gen != 7) 51 return false; 52 53 switch (inst->opcode) { 54 case BRW_OPCODE_MOV: 55 case BRW_OPCODE_CMP: 56 case BRW_OPCODE_ADD: 57 case BRW_OPCODE_MUL: 58 return true; 59 default: 60 return false; 61 } 62 } 63 64 /** 65 * Returns true for instructions that don't support immediate sources. 66 */ 67 static bool 68 must_promote_imm(const struct gen_device_info *devinfo, const fs_inst *inst) 69 { 70 switch (inst->opcode) { 71 case SHADER_OPCODE_POW: 72 return devinfo->gen < 8; 73 case BRW_OPCODE_MAD: 74 case BRW_OPCODE_LRP: 75 return true; 76 default: 77 return false; 78 } 79 } 80 81 /** A box for putting fs_regs in a linked list. */ 82 struct reg_link { 83 DECLARE_RALLOC_CXX_OPERATORS(reg_link) 84 85 reg_link(fs_reg *reg) : reg(reg) {} 86 87 struct exec_node link; 88 fs_reg *reg; 89 }; 90 91 static struct exec_node * 92 link(void *mem_ctx, fs_reg *reg) 93 { 94 reg_link *l = new(mem_ctx) reg_link(reg); 95 return &l->link; 96 } 97 98 /** 99 * Information about an immediate value. 100 */ 101 struct imm { 102 /** The common ancestor of all blocks using this immediate value. */ 103 bblock_t *block; 104 105 /** 106 * The instruction generating the immediate value, if all uses are contained 107 * within a single basic block. Otherwise, NULL. 108 */ 109 fs_inst *inst; 110 111 /** 112 * A list of fs_regs that refer to this immediate. If we promote it, we'll 113 * have to patch these up to refer to the new GRF. 114 */ 115 exec_list *uses; 116 117 /** The immediate value. We currently only handle floats. */ 118 float val; 119 120 /** 121 * The GRF register and subregister number where we've decided to store the 122 * constant value. 123 */ 124 uint8_t subreg_offset; 125 uint16_t nr; 126 127 /** The number of coissuable instructions using this immediate. */ 128 uint16_t uses_by_coissue; 129 130 /** 131 * Whether this constant is used by an instruction that can't handle an 132 * immediate source (and already has to be promoted to a GRF). 133 */ 134 bool must_promote; 135 136 uint16_t first_use_ip; 137 uint16_t last_use_ip; 138 }; 139 140 /** The working set of information about immediates. */ 141 struct table { 142 struct imm *imm; 143 int size; 144 int len; 145 }; 146 147 static struct imm * 148 find_imm(struct table *table, float val) 149 { 150 for (int i = 0; i < table->len; i++) { 151 if (table->imm[i].val == val) { 152 return &table->imm[i]; 153 } 154 } 155 return NULL; 156 } 157 158 static struct imm * 159 new_imm(struct table *table, void *mem_ctx) 160 { 161 if (table->len == table->size) { 162 table->size *= 2; 163 table->imm = reralloc(mem_ctx, table->imm, struct imm, table->size); 164 } 165 return &table->imm[table->len++]; 166 } 167 168 /** 169 * Comparator used for sorting an array of imm structures. 170 * 171 * We sort by basic block number, then last use IP, then first use IP (least 172 * to greatest). This sorting causes immediates live in the same area to be 173 * allocated to the same register in the hopes that all values will be dead 174 * about the same time and the register can be reused. 175 */ 176 static int 177 compare(const void *_a, const void *_b) 178 { 179 const struct imm *a = (const struct imm *)_a, 180 *b = (const struct imm *)_b; 181 182 int block_diff = a->block->num - b->block->num; 183 if (block_diff) 184 return block_diff; 185 186 int end_diff = a->last_use_ip - b->last_use_ip; 187 if (end_diff) 188 return end_diff; 189 190 return a->first_use_ip - b->first_use_ip; 191 } 192 193 bool 194 fs_visitor::opt_combine_constants() 195 { 196 void *const_ctx = ralloc_context(NULL); 197 198 struct table table; 199 table.size = 8; 200 table.len = 0; 201 table.imm = ralloc_array(const_ctx, struct imm, table.size); 202 203 cfg->calculate_idom(); 204 unsigned ip = -1; 205 206 /* Make a pass through all instructions and count the number of times each 207 * constant is used by coissueable instructions or instructions that cannot 208 * take immediate arguments. 209 */ 210 foreach_block_and_inst(block, fs_inst, inst, cfg) { 211 ip++; 212 213 if (!could_coissue(devinfo, inst) && !must_promote_imm(devinfo, inst)) 214 continue; 215 216 for (int i = 0; i < inst->sources; i++) { 217 if (inst->src[i].file != IMM || 218 inst->src[i].type != BRW_REGISTER_TYPE_F) 219 continue; 220 221 float val = !inst->can_do_source_mods(devinfo) ? inst->src[i].f : 222 fabs(inst->src[i].f); 223 struct imm *imm = find_imm(&table, val); 224 225 if (imm) { 226 bblock_t *intersection = cfg_t::intersect(block, imm->block); 227 if (intersection != imm->block) 228 imm->inst = NULL; 229 imm->block = intersection; 230 imm->uses->push_tail(link(const_ctx, &inst->src[i])); 231 imm->uses_by_coissue += could_coissue(devinfo, inst); 232 imm->must_promote = imm->must_promote || must_promote_imm(devinfo, inst); 233 imm->last_use_ip = ip; 234 } else { 235 imm = new_imm(&table, const_ctx); 236 imm->block = block; 237 imm->inst = inst; 238 imm->uses = new(const_ctx) exec_list(); 239 imm->uses->push_tail(link(const_ctx, &inst->src[i])); 240 imm->val = val; 241 imm->uses_by_coissue = could_coissue(devinfo, inst); 242 imm->must_promote = must_promote_imm(devinfo, inst); 243 imm->first_use_ip = ip; 244 imm->last_use_ip = ip; 245 } 246 } 247 } 248 249 /* Remove constants from the table that don't have enough uses to make them 250 * profitable to store in a register. 251 */ 252 for (int i = 0; i < table.len;) { 253 struct imm *imm = &table.imm[i]; 254 255 if (!imm->must_promote && imm->uses_by_coissue < 4) { 256 table.imm[i] = table.imm[table.len - 1]; 257 table.len--; 258 continue; 259 } 260 i++; 261 } 262 if (table.len == 0) { 263 ralloc_free(const_ctx); 264 return false; 265 } 266 if (cfg->num_blocks != 1) 267 qsort(table.imm, table.len, sizeof(struct imm), compare); 268 269 /* Insert MOVs to load the constant values into GRFs. */ 270 fs_reg reg(VGRF, alloc.allocate(1)); 271 reg.stride = 0; 272 for (int i = 0; i < table.len; i++) { 273 struct imm *imm = &table.imm[i]; 274 /* Insert it either before the instruction that generated the immediate 275 * or after the last non-control flow instruction of the common ancestor. 276 */ 277 exec_node *n = (imm->inst ? imm->inst : 278 imm->block->last_non_control_flow_inst()->next); 279 const fs_builder ibld = bld.at(imm->block, n).exec_all().group(1, 0); 280 281 ibld.MOV(reg, brw_imm_f(imm->val)); 282 imm->nr = reg.nr; 283 imm->subreg_offset = reg.offset; 284 285 reg.offset += sizeof(float); 286 if (reg.offset == 8 * sizeof(float)) { 287 reg.nr = alloc.allocate(1); 288 reg.offset = 0; 289 } 290 } 291 promoted_constants = table.len; 292 293 /* Rewrite the immediate sources to refer to the new GRFs. */ 294 for (int i = 0; i < table.len; i++) { 295 foreach_list_typed(reg_link, link, link, table.imm[i].uses) { 296 fs_reg *reg = link->reg; 297 reg->file = VGRF; 298 reg->nr = table.imm[i].nr; 299 reg->offset = table.imm[i].subreg_offset; 300 reg->stride = 0; 301 reg->negate = signbit(reg->f) != signbit(table.imm[i].val); 302 assert((isnan(reg->f) && isnan(table.imm[i].val)) || 303 fabsf(reg->f) == fabs(table.imm[i].val)); 304 } 305 } 306 307 if (debug) { 308 for (int i = 0; i < table.len; i++) { 309 struct imm *imm = &table.imm[i]; 310 311 printf("%.3fF - block %3d, reg %3d sub %2d, Uses: (%2d, %2d), " 312 "IP: %4d to %4d, length %4d\n", 313 imm->val, 314 imm->block->num, 315 imm->nr, 316 imm->subreg_offset, 317 imm->must_promote, 318 imm->uses_by_coissue, 319 imm->first_use_ip, 320 imm->last_use_ip, 321 imm->last_use_ip - imm->first_use_ip); 322 } 323 } 324 325 ralloc_free(const_ctx); 326 invalidate_live_intervals(); 327 328 return true; 329 } 330