1 ; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VMEM -check-prefix=GCN %s 2 ; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=1 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VGPR -check-prefix=GCN %s 3 4 ; Verify registers used for tracking exec mask changes when all 5 ; registers are spilled at the end of the block. The SGPR spill 6 ; placement relative to the exec modifications are important. 7 8 ; FIXME: This checks with SGPR to VGPR spilling disabled, but this may 9 ; not work correctly in cases where no workitems take a branch. 10 11 12 ; GCN-LABEL: {{^}}divergent_if_endif: 13 ; VGPR: workitem_private_segment_byte_size = 12{{$}} 14 15 16 ; GCN: {{^}}; %bb.0: 17 ; GCN: s_mov_b32 m0, -1 18 ; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 19 20 ; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 21 ; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 22 ; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, [[CMP0]] 23 24 ; Spill saved exec 25 ; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 26 ; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 27 28 29 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] 30 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 offset:4 ; 4-byte Folded Spill 31 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] 32 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:8 ; 4-byte Folded Spill 33 34 ; Spill load 35 ; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill 36 37 ; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} 38 39 ; GCN: mask branch [[ENDIF:BB[0-9]+_[0-9]+]] 40 41 ; GCN: {{^}}BB{{[0-9]+}}_1: ; %if 42 ; GCN: s_mov_b32 m0, -1 43 ; GCN: ds_read_b32 [[LOAD1:v[0-9]+]] 44 ; GCN: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], s7 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload 45 ; GCN: s_waitcnt vmcnt(0) lgkmcnt(0) 46 47 48 ; Spill val register 49 ; GCN: v_add_i32_e32 [[VAL:v[0-9]+]], vcc, [[LOAD1]], [[RELOAD_LOAD0]] 50 ; GCN: buffer_store_dword [[VAL]], off, s[0:3], s7 offset:[[VAL_OFFSET:[0-9]+]] ; 4-byte Folded Spill 51 52 ; VMEM: [[ENDIF]]: 53 54 ; Reload and restore exec mask 55 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 56 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 57 58 59 60 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload 61 ; VMEM: s_waitcnt vmcnt(0) 62 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] 63 64 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:8 ; 4-byte Folded Reload 65 ; VMEM: s_waitcnt vmcnt(0) 66 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] 67 68 ; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 69 70 ; Restore val 71 ; GCN: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], s7 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload 72 73 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RELOAD_VAL]] 74 define amdgpu_kernel void @divergent_if_endif(i32 addrspace(1)* %out) #0 { 75 entry: 76 %tid = call i32 @llvm.amdgcn.workitem.id.x() 77 %load0 = load volatile i32, i32 addrspace(3)* undef 78 %cmp0 = icmp eq i32 %tid, 0 79 br i1 %cmp0, label %if, label %endif 80 81 if: 82 %load1 = load volatile i32, i32 addrspace(3)* undef 83 %val = add i32 %load0, %load1 84 br label %endif 85 86 endif: 87 %tmp4 = phi i32 [ %val, %if ], [ 0, %entry ] 88 store i32 %tmp4, i32 addrspace(1)* %out 89 ret void 90 } 91 92 ; GCN-LABEL: {{^}}divergent_loop: 93 ; VGPR: workitem_private_segment_byte_size = 16{{$}} 94 95 ; GCN: {{^}}; %bb.0: 96 97 ; GCN: s_mov_b32 m0, -1 98 ; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 99 100 ; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 101 102 ; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 103 ; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] 104 105 ; Spill load 106 ; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 offset:4 ; 4-byte Folded Spill 107 108 ; Spill saved exec 109 ; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 110 ; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 111 112 113 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] 114 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 offset:20 ; 4-byte Folded Spill 115 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] 116 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:24 ; 4-byte Folded Spill 117 118 ; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} 119 120 ; GCN-NEXT: ; mask branch [[END:BB[0-9]+_[0-9]+]] 121 ; GCN-NEXT: s_cbranch_execz [[END]] 122 123 124 ; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: 125 ; GCN: buffer_load_dword v[[VAL_LOOP_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload 126 ; GCN: v_subrev_i32_e32 [[VAL_LOOP:v[0-9]+]], vcc, v{{[0-9]+}}, v[[VAL_LOOP_RELOAD]] 127 ; GCN: v_cmp_ne_u32_e32 vcc, 128 ; GCN: s_and_b64 vcc, exec, vcc 129 ; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill 130 ; GCN-NEXT: s_cbranch_vccnz [[LOOP]] 131 132 133 ; GCN: [[END]]: 134 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 135 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 136 137 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:20 ; 4-byte Folded Reload 138 ; VMEM: s_waitcnt vmcnt(0) 139 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] 140 141 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:24 ; 4-byte Folded Reload 142 ; VMEM: s_waitcnt vmcnt(0) 143 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] 144 145 ; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 146 ; GCN: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload 147 148 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[VAL_END]] 149 define amdgpu_kernel void @divergent_loop(i32 addrspace(1)* %out) #0 { 150 entry: 151 %tid = call i32 @llvm.amdgcn.workitem.id.x() 152 %load0 = load volatile i32, i32 addrspace(3)* undef 153 %cmp0 = icmp eq i32 %tid, 0 154 br i1 %cmp0, label %loop, label %end 155 156 loop: 157 %i = phi i32 [ %i.inc, %loop ], [ 0, %entry ] 158 %val = phi i32 [ %val.sub, %loop ], [ %load0, %entry ] 159 %load1 = load volatile i32, i32 addrspace(3)* undef 160 %i.inc = add i32 %i, 1 161 %val.sub = sub i32 %val, %load1 162 %cmp1 = icmp ne i32 %i, 256 163 br i1 %cmp1, label %loop, label %end 164 165 end: 166 %tmp4 = phi i32 [ %val.sub, %loop ], [ 0, %entry ] 167 store i32 %tmp4, i32 addrspace(1)* %out 168 ret void 169 } 170 171 ; GCN-LABEL: {{^}}divergent_if_else_endif: 172 ; GCN: {{^}}; %bb.0: 173 174 ; GCN: s_mov_b32 m0, -1 175 ; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 176 177 ; GCN: v_cmp_ne_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], v0, 178 179 ; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 180 ; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] 181 ; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}} 182 183 ; Spill load 184 ; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 offset:4 ; 4-byte Folded Spill 185 186 ; Spill saved exec 187 ; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 188 ; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 189 190 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] 191 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 offset:[[SAVEEXEC_LO_OFFSET:[0-9]+]] ; 4-byte Folded Spill 192 ; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] 193 ; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:[[SAVEEXEC_HI_OFFSET:[0-9]+]] ; 4-byte Folded Spill 194 195 ; GCN: s_mov_b64 exec, [[CMP0]] 196 197 ; FIXME: It makes no sense to put this skip here 198 ; GCN-NEXT: ; mask branch [[FLOW:BB[0-9]+_[0-9]+]] 199 ; GCN: s_cbranch_execz [[FLOW]] 200 ; GCN-NEXT: s_branch [[ELSE:BB[0-9]+_[0-9]+]] 201 202 ; GCN: [[FLOW]]: ; %Flow 203 ; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 204 ; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 205 206 207 ; VMEM: buffer_load_dword v[[FLOW_V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:[[SAVEEXEC_LO_OFFSET]] 208 ; VMEM: s_waitcnt vmcnt(0) 209 ; VMEM: v_readfirstlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC_LO]] 210 211 ; VMEM: buffer_load_dword v[[FLOW_V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:[[SAVEEXEC_HI_OFFSET]] ; 4-byte Folded Reload 212 ; VMEM: s_waitcnt vmcnt(0) 213 ; VMEM: v_readfirstlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC_HI]] 214 215 ; GCN: s_or_saveexec_b64 s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}}, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} 216 217 ; Regular spill value restored after exec modification 218 ; GCN: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], s7 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload 219 220 221 ; Spill saved exec 222 ; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]], [[FLOW_SAVEEXEC_LO_LANE:[0-9]+]] 223 ; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]], [[FLOW_SAVEEXEC_HI_LANE:[0-9]+]] 224 225 226 ; VMEM: v_mov_b32_e32 v[[FLOW_V_SAVEEXEC_LO:[0-9]+]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]] 227 ; VMEM: buffer_store_dword v[[FLOW_V_SAVEEXEC_LO]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_LO_OFFSET:[0-9]+]] ; 4-byte Folded Spill 228 ; VMEM: v_mov_b32_e32 v[[FLOW_V_SAVEEXEC_HI:[0-9]+]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]] 229 ; VMEM: buffer_store_dword v[[FLOW_V_SAVEEXEC_HI]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_HI_OFFSET:[0-9]+]] ; 4-byte Folded Spill 230 231 ; GCN: buffer_store_dword [[FLOW_VAL]], off, s[0:3], s7 offset:[[RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill 232 ; GCN: s_xor_b64 exec, exec, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} 233 ; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9]+_[0-9]+]] 234 ; GCN-NEXT: s_cbranch_execz [[ENDIF]] 235 236 237 ; GCN: BB{{[0-9]+}}_2: ; %if 238 ; GCN: ds_read_b32 239 ; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload 240 ; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] 241 ; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[RESULT_OFFSET]] ; 4-byte Folded Spill 242 ; GCN-NEXT: s_branch [[ENDIF:BB[0-9]+_[0-9]+]] 243 244 ; GCN: [[ELSE]]: ; %else 245 ; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload 246 ; GCN: v_subrev_i32_e32 [[SUB:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] 247 ; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[FLOW_RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill 248 ; GCN-NEXT: s_branch [[FLOW]] 249 250 ; GCN: [[ENDIF]]: 251 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_LO_LANE]] 252 ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_HI_LANE]] 253 254 255 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_LO_OFFSET]] ; 4-byte Folded Reload 256 ; VMEM: s_waitcnt vmcnt(0) 257 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] 258 259 ; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_HI_OFFSET]] ; 4-byte Folded Reload 260 ; VMEM: s_waitcnt vmcnt(0) 261 ; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] 262 263 ; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 264 265 ; GCN: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], s7 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload 266 ; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RESULT]] 267 define amdgpu_kernel void @divergent_if_else_endif(i32 addrspace(1)* %out) #0 { 268 entry: 269 %tid = call i32 @llvm.amdgcn.workitem.id.x() 270 %load0 = load volatile i32, i32 addrspace(3)* undef 271 %cmp0 = icmp eq i32 %tid, 0 272 br i1 %cmp0, label %if, label %else 273 274 if: 275 %load1 = load volatile i32, i32 addrspace(3)* undef 276 %val0 = add i32 %load0, %load1 277 br label %endif 278 279 else: 280 %load2 = load volatile i32, i32 addrspace(3)* undef 281 %val1 = sub i32 %load0, %load2 282 br label %endif 283 284 endif: 285 %result = phi i32 [ %val0, %if ], [ %val1, %else ] 286 store i32 %result, i32 addrspace(1)* %out 287 ret void 288 } 289 290 declare i32 @llvm.amdgcn.workitem.id.x() #1 291 292 attributes #0 = { nounwind } 293 attributes #1 = { nounwind readnone } 294
