1 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s 2 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s 3 4 declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) 5 declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) 6 declare void @llvm.amdgcn.s.barrier() #1 7 declare i32 @llvm.amdgcn.workitem.id.x() #2 8 9 10 @stored_lds_ptr = addrspace(3) global i32 addrspace(3)* undef, align 4 11 @stored_constant_ptr = addrspace(3) global i32 addrspace(4)* undef, align 8 12 @stored_global_ptr = addrspace(3) global i32 addrspace(1)* undef, align 8 13 14 ; GCN-LABEL: {{^}}reorder_local_load_global_store_local_load: 15 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3 16 ; CI: buffer_store_dword 17 18 ; GFX9: global_store_dword 19 ; GFX9: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3 20 ; GFX9: global_store_dword 21 define amdgpu_kernel void @reorder_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 { 22 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4 23 24 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1 25 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3 26 27 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4 28 store i32 99, i32 addrspace(1)* %gptr, align 4 29 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4 30 31 %add = add nsw i32 %tmp1, %tmp2 32 33 store i32 %add, i32 addrspace(1)* %out, align 4 34 ret void 35 } 36 37 ; GCN-LABEL: {{^}}no_reorder_local_load_volatile_global_store_local_load: 38 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4 39 ; CI: buffer_store_dword 40 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12 41 42 ; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4 43 ; GFX9: global_store_dword 44 ; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12 45 define amdgpu_kernel void @no_reorder_local_load_volatile_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 { 46 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4 47 48 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1 49 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3 50 51 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4 52 store volatile i32 99, i32 addrspace(1)* %gptr, align 4 53 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4 54 55 %add = add nsw i32 %tmp1, %tmp2 56 57 store i32 %add, i32 addrspace(1)* %out, align 4 58 ret void 59 } 60 61 ; GCN-LABEL: {{^}}no_reorder_barrier_local_load_global_store_local_load: 62 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4 63 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12 64 ; CI: buffer_store_dword 65 66 ; GFX9-DAG: global_store_dword 67 ; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4 68 ; GFX9: s_barrier 69 ; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12 70 ; GFX9-DAG: global_store_dword 71 define amdgpu_kernel void @no_reorder_barrier_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 { 72 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4 73 74 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1 75 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3 76 77 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4 78 store i32 99, i32 addrspace(1)* %gptr, align 4 79 call void @llvm.amdgcn.s.barrier() #1 80 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4 81 82 %add = add nsw i32 %tmp1, %tmp2 83 84 store i32 %add, i32 addrspace(1)* %out, align 4 85 ret void 86 } 87 88 ; GCN-LABEL: {{^}}reorder_constant_load_global_store_constant_load: 89 ; GCN-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}} 90 ; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}} 91 ; CI: buffer_store_dword 92 93 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1 94 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3 95 96 ; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4 97 ; GFX9: global_store_dword 98 ; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc 99 100 ; CI: buffer_store_dword 101 ; GFX9: global_store_dword 102 define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 { 103 %ptr0 = load i32 addrspace(4)*, i32 addrspace(4)* addrspace(3)* @stored_constant_ptr, align 8 104 105 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1 106 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 3 107 108 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4 109 store i32 99, i32 addrspace(1)* %gptr, align 4 110 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4 111 112 %add = add nsw i32 %tmp1, %tmp2 113 114 store i32 %add, i32 addrspace(1)* %out, align 4 115 ret void 116 } 117 118 ; GCN-LABEL: {{^}}reorder_constant_load_local_store_constant_load: 119 ; GCN: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}} 120 ; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}} 121 122 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1 123 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3 124 125 ; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4 126 ; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc 127 128 ; GCN: ds_write_b32 129 ; CI: buffer_store_dword 130 ; GFX9: global_store_dword 131 define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr) #0 { 132 %ptr0 = load i32 addrspace(4)*, i32 addrspace(4)* addrspace(3)* @stored_constant_ptr, align 8 133 134 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1 135 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 3 136 137 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4 138 store i32 99, i32 addrspace(3)* %lptr, align 4 139 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4 140 141 %add = add nsw i32 %tmp1, %tmp2 142 143 store i32 %add, i32 addrspace(1)* %out, align 4 144 ret void 145 } 146 147 ; GCN-LABEL: {{^}}reorder_smrd_load_local_store_smrd_load: 148 ; GCN: s_load_dword 149 ; GCN: s_load_dword 150 ; GCN: s_load_dword 151 ; GCN: ds_write_b32 152 ; CI: buffer_store_dword 153 ; GFX9: global_store_dword 154 define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(i32 addrspace(1)* %out, i32 addrspace(3)* noalias %lptr, i32 addrspace(4)* %ptr0) #0 { 155 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1 156 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 2 157 158 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4 159 store i32 99, i32 addrspace(3)* %lptr, align 4 160 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4 161 162 %add = add nsw i32 %tmp1, %tmp2 163 164 store i32 %add, i32 addrspace(1)* %out, align 4 165 ret void 166 } 167 168 ; GCN-LABEL: {{^}}reorder_global_load_local_store_global_load: 169 ; CI: ds_write_b32 170 ; CI: buffer_load_dword 171 ; CI: buffer_load_dword 172 ; CI: buffer_store_dword 173 174 ; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4 175 ; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:12 176 ; GFX9: ds_write_b32 177 define amdgpu_kernel void @reorder_global_load_local_store_global_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr, i32 addrspace(1)* %ptr0) #0 { 178 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 1 179 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 3 180 181 %tmp1 = load i32, i32 addrspace(1)* %ptr1, align 4 182 store i32 99, i32 addrspace(3)* %lptr, align 4 183 %tmp2 = load i32, i32 addrspace(1)* %ptr2, align 4 184 185 %add = add nsw i32 %tmp1, %tmp2 186 187 store i32 %add, i32 addrspace(1)* %out, align 4 188 ret void 189 } 190 191 ; GCN-LABEL: {{^}}reorder_local_offsets: 192 ; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:100 offset1:102 193 ; GCN-DAG: ds_write2_b32 {{v[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:3 offset1:100 194 ; GCN-DAG: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:408 195 ; CI: buffer_store_dword 196 ; GFX9: global_store_dword 197 ; GCN: s_endpgm 198 define amdgpu_kernel void @reorder_local_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(3)* noalias nocapture %ptr0) #0 { 199 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3 200 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 100 201 %ptr3 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 102 202 203 store i32 123, i32 addrspace(3)* %ptr1, align 4 204 %tmp1 = load i32, i32 addrspace(3)* %ptr2, align 4 205 %tmp2 = load i32, i32 addrspace(3)* %ptr3, align 4 206 store i32 123, i32 addrspace(3)* %ptr2, align 4 207 %tmp3 = load i32, i32 addrspace(3)* %ptr1, align 4 208 store i32 789, i32 addrspace(3)* %ptr3, align 4 209 210 %add.0 = add nsw i32 %tmp2, %tmp1 211 %add.1 = add nsw i32 %add.0, %tmp3 212 store i32 %add.1, i32 addrspace(1)* %out, align 4 213 ret void 214 } 215 216 ; GCN-LABEL: {{^}}reorder_global_offsets: 217 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400 218 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408 219 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12 220 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400 221 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408 222 ; CI: buffer_store_dword 223 ; CI: s_endpgm 224 225 ; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:400 226 ; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:408 227 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:12 228 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:400 229 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:408 230 ; GFX9: global_store_dword 231 ; GFX9: s_endpgm 232 define amdgpu_kernel void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(1)* noalias nocapture %ptr0) #0 { 233 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3 234 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 100 235 %ptr3 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 102 236 237 store i32 123, i32 addrspace(1)* %ptr1, align 4 238 %tmp1 = load i32, i32 addrspace(1)* %ptr2, align 4 239 %tmp2 = load i32, i32 addrspace(1)* %ptr3, align 4 240 store i32 123, i32 addrspace(1)* %ptr2, align 4 241 %tmp3 = load i32, i32 addrspace(1)* %ptr1, align 4 242 store i32 789, i32 addrspace(1)* %ptr3, align 4 243 244 %add.0 = add nsw i32 %tmp2, %tmp1 245 %add.1 = add nsw i32 %add.0, %tmp3 246 store i32 %add.1, i32 addrspace(1)* %out, align 4 247 ret void 248 } 249 250 ; GCN-LABEL: {{^}}reorder_global_offsets_addr64_soffset0: 251 ; CI: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}} 252 ; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:28{{$}} 253 ; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:44{{$}} 254 255 ; CI: v_mov_b32 256 ; CI: v_mov_b32 257 258 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} 259 260 ; CI: v_add_i32 261 ; CI: v_add_i32 262 263 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20{{$}} 264 265 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:36{{$}} 266 ; CI-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:52{{$}} 267 268 269 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:12 270 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:28 271 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:44 272 273 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off{{$}} 274 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:20 275 276 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:36 277 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:52 278 define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(i32 addrspace(1)* noalias nocapture %ptr.base) #0 { 279 %id = call i32 @llvm.amdgcn.workitem.id.x() 280 %id.ext = sext i32 %id to i64 281 282 %ptr0 = getelementptr inbounds i32, i32 addrspace(1)* %ptr.base, i64 %id.ext 283 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3 284 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 5 285 %ptr3 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 7 286 %ptr4 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 9 287 %ptr5 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 11 288 %ptr6 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 13 289 290 store i32 789, i32 addrspace(1)* %ptr0, align 4 291 %tmp1 = load i32, i32 addrspace(1)* %ptr1, align 4 292 store i32 123, i32 addrspace(1)* %ptr2, align 4 293 %tmp2 = load i32, i32 addrspace(1)* %ptr3, align 4 294 %add.0 = add nsw i32 %tmp1, %tmp2 295 store i32 %add.0, i32 addrspace(1)* %ptr4, align 4 296 %tmp3 = load i32, i32 addrspace(1)* %ptr5, align 4 297 %add.1 = add nsw i32 %add.0, %tmp3 298 store i32 %add.1, i32 addrspace(1)* %ptr6, align 4 299 ret void 300 } 301 302 ; XGCN-LABEL: {{^}}reorder_local_load_tbuffer_store_local_load: 303 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x4 304 ; XCI: TBUFFER_STORE_FORMAT 305 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x8 306 ; define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(i32 addrspace(1)* %out, i32 %a1, i32 %vaddr) #0 { 307 ; %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4 308 309 ; %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1 310 ; %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 2 311 312 ; %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4 313 314 ; %vdata = insertelement <4 x i32> undef, i32 %a1, i32 0 315 ; call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %vdata, <4 x i32> undef, 316 ; i32 %vaddr, i32 0, i32 0, i32 32, i32 14, i32 4, i1 1, i1 1) 317 318 ; %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4 319 320 ; %add = add nsw i32 %tmp1, %tmp2 321 322 ; store i32 %add, i32 addrspace(1)* %out, align 4 323 ; ret void 324 ; } 325 326 attributes #0 = { nounwind } 327 attributes #1 = { nounwind convergent } 328 attributes #2 = { nounwind readnone } 329
