1 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s 2 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9 %s 3 4 @lds = addrspace(3) global [512 x float] undef, align 4 5 @lds.f64 = addrspace(3) global [512 x double] undef, align 8 6 7 ; GCN-LABEL: {{^}}simple_write2_one_val_f32: 8 ; CI-DAG: s_mov_b32 m0 9 ; GFX9-NOT: m0 10 11 ; GCN-DAG: {{buffer|flat|global}}_load_dword [[VAL:v[0-9]+]] 12 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 13 ; GCN: ds_write2_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:8 14 ; GCN: s_endpgm 15 define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { 16 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 17 %in.gep = getelementptr float, float addrspace(1)* %in, i32 %x.i 18 %val = load float, float addrspace(1)* %in.gep, align 4 19 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 20 store float %val, float addrspace(3)* %arrayidx0, align 4 21 %add.x = add nsw i32 %x.i, 8 22 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 23 store float %val, float addrspace(3)* %arrayidx1, align 4 24 ret void 25 } 26 27 ; GCN-LABEL: {{^}}simple_write2_two_val_f32: 28 ; CI-DAG: s_mov_b32 m0 29 ; GFX9-NOT: m0 30 31 ; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} 32 ; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 33 34 ; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}} 35 ; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4 36 37 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 38 ; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8 39 ; GCN: s_endpgm 40 define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { 41 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 42 %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i 43 %in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1 44 %val0 = load volatile float, float addrspace(1)* %in.gep.0, align 4 45 %val1 = load volatile float, float addrspace(1)* %in.gep.1, align 4 46 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 47 store float %val0, float addrspace(3)* %arrayidx0, align 4 48 %add.x = add nsw i32 %x.i, 8 49 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 50 store float %val1, float addrspace(3)* %arrayidx1, align 4 51 ret void 52 } 53 54 ; GCN-LABEL: @simple_write2_two_val_f32_volatile_0 55 ; CI-DAG: s_mov_b32 m0 56 ; GFX9-NOT: m0 57 58 ; GCN-NOT: ds_write2_b32 59 ; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} 60 ; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32 61 ; GCN: s_endpgm 62 define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { 63 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 64 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i 65 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i 66 %val0 = load volatile float, float addrspace(1)* %in0.gep, align 4 67 %val1 = load volatile float, float addrspace(1)* %in1.gep, align 4 68 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 69 store volatile float %val0, float addrspace(3)* %arrayidx0, align 4 70 %add.x = add nsw i32 %x.i, 8 71 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 72 store float %val1, float addrspace(3)* %arrayidx1, align 4 73 ret void 74 } 75 76 ; GCN-LABEL: @simple_write2_two_val_f32_volatile_1 77 ; CI-DAG: s_mov_b32 m0 78 ; GFX9-NOT: m0 79 80 ; GCN-NOT: ds_write2_b32 81 ; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} 82 ; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32 83 ; GCN: s_endpgm 84 define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { 85 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 86 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i 87 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i 88 %val0 = load volatile float, float addrspace(1)* %in0.gep, align 4 89 %val1 = load volatile float, float addrspace(1)* %in1.gep, align 4 90 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 91 store float %val0, float addrspace(3)* %arrayidx0, align 4 92 %add.x = add nsw i32 %x.i, 8 93 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 94 store volatile float %val1, float addrspace(3)* %arrayidx1, align 4 95 ret void 96 } 97 98 ; 2 data subregisters from different super registers. 99 ; GCN-LABEL: {{^}}simple_write2_two_val_subreg2_mixed_f32: 100 ; GFX9-NOT: m0 101 102 ; CI: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}} 103 ; CI: buffer_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}} 104 ; CI-DAG: s_mov_b32 m0 105 106 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 107 108 ; GFX9: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}} 109 ; GFX9: global_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}} 110 111 ; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 112 ; GCN: s_endpgm 113 define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 { 114 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 115 %in.gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i 116 %in.gep.1 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in.gep.0, i32 1 117 %val0 = load volatile <2 x float>, <2 x float> addrspace(1)* %in.gep.0, align 8 118 %val1 = load volatile <2 x float>, <2 x float> addrspace(1)* %in.gep.1, align 8 119 %val0.0 = extractelement <2 x float> %val0, i32 0 120 %val1.1 = extractelement <2 x float> %val1, i32 1 121 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 122 store float %val0.0, float addrspace(3)* %arrayidx0, align 4 123 %add.x = add nsw i32 %x.i, 8 124 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 125 store float %val1.1, float addrspace(3)* %arrayidx1, align 4 126 ret void 127 } 128 129 ; GCN-LABEL: @simple_write2_two_val_subreg2_f32 130 ; CI-DAG: s_mov_b32 m0 131 ; GFX9-NOT: m0 132 133 ; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} 134 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 135 ; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 136 ; GCN: s_endpgm 137 define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 { 138 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 139 %in.gep = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i 140 %val = load <2 x float>, <2 x float> addrspace(1)* %in.gep, align 8 141 %val0 = extractelement <2 x float> %val, i32 0 142 %val1 = extractelement <2 x float> %val, i32 1 143 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 144 store float %val0, float addrspace(3)* %arrayidx0, align 4 145 %add.x = add nsw i32 %x.i, 8 146 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 147 store float %val1, float addrspace(3)* %arrayidx1, align 4 148 ret void 149 } 150 151 ; GCN-LABEL: @simple_write2_two_val_subreg4_f32 152 ; CI-DAG: s_mov_b32 m0 153 ; GFX9-NOT: m0 154 155 ; GCN-DAG: {{buffer|global}}_load_dwordx4 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} 156 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 157 ; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8 158 ; GCN: s_endpgm 159 define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)* %C, <4 x float> addrspace(1)* %in) #0 { 160 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 161 %in.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 %x.i 162 %val = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 16 163 %val0 = extractelement <4 x float> %val, i32 0 164 %val1 = extractelement <4 x float> %val, i32 3 165 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 166 store float %val0, float addrspace(3)* %arrayidx0, align 4 167 %add.x = add nsw i32 %x.i, 8 168 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 169 store float %val1, float addrspace(3)* %arrayidx1, align 4 170 ret void 171 } 172 173 ; GCN-LABEL: @simple_write2_two_val_max_offset_f32 174 ; CI-DAG: s_mov_b32 m0 175 ; GFX9-NOT: m0 176 177 ; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} 178 ; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 179 180 ; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}} 181 ; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4 182 183 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}} 184 ; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255 185 ; GCN: s_endpgm 186 define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 { 187 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 188 %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i 189 %in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1 190 %val0 = load volatile float, float addrspace(1)* %in.gep.0, align 4 191 %val1 = load volatile float, float addrspace(1)* %in.gep.1, align 4 192 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 193 store float %val0, float addrspace(3)* %arrayidx0, align 4 194 %add.x = add nsw i32 %x.i, 255 195 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 196 store float %val1, float addrspace(3)* %arrayidx1, align 4 197 ret void 198 } 199 200 ; GCN-LABEL: @simple_write2_two_val_too_far_f32 201 ; CI-DAG: s_mov_b32 m0 202 ; GFX9-NOT: m0 203 204 ; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} 205 ; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 206 ; GCN: s_endpgm 207 define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { 208 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 209 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i 210 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i 211 %val0 = load float, float addrspace(1)* %in0.gep, align 4 212 %val1 = load float, float addrspace(1)* %in1.gep, align 4 213 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 214 store float %val0, float addrspace(3)* %arrayidx0, align 4 215 %add.x = add nsw i32 %x.i, 257 216 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 217 store float %val1, float addrspace(3)* %arrayidx1, align 4 218 ret void 219 } 220 221 ; GCN-LABEL: @simple_write2_two_val_f32_x2 222 ; CI-DAG: s_mov_b32 m0 223 ; GFX9-NOT: m0 224 225 ; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8 226 ; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 227 ; GCN: s_endpgm 228 define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { 229 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 230 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x 231 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x 232 %val0 = load float, float addrspace(1)* %in0.gep, align 4 233 %val1 = load float, float addrspace(1)* %in1.gep, align 4 234 235 %idx.0 = add nsw i32 %tid.x, 0 236 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 237 store float %val0, float addrspace(3)* %arrayidx0, align 4 238 239 %idx.1 = add nsw i32 %tid.x, 8 240 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 241 store float %val1, float addrspace(3)* %arrayidx1, align 4 242 243 %idx.2 = add nsw i32 %tid.x, 11 244 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 245 store float %val0, float addrspace(3)* %arrayidx2, align 4 246 247 %idx.3 = add nsw i32 %tid.x, 27 248 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 249 store float %val1, float addrspace(3)* %arrayidx3, align 4 250 251 ret void 252 } 253 254 ; GCN-LABEL: @simple_write2_two_val_f32_x2_nonzero_base 255 ; CI-DAG: s_mov_b32 m0 256 ; GFX9-NOT: m0 257 258 ; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8 259 ; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 260 ; GCN: s_endpgm 261 define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { 262 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 263 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x 264 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x 265 %val0 = load float, float addrspace(1)* %in0.gep, align 4 266 %val1 = load float, float addrspace(1)* %in1.gep, align 4 267 268 %idx.0 = add nsw i32 %tid.x, 3 269 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 270 store float %val0, float addrspace(3)* %arrayidx0, align 4 271 272 %idx.1 = add nsw i32 %tid.x, 8 273 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 274 store float %val1, float addrspace(3)* %arrayidx1, align 4 275 276 %idx.2 = add nsw i32 %tid.x, 11 277 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 278 store float %val0, float addrspace(3)* %arrayidx2, align 4 279 280 %idx.3 = add nsw i32 %tid.x, 27 281 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 282 store float %val1, float addrspace(3)* %arrayidx3, align 4 283 284 ret void 285 } 286 287 ; GCN-LABEL: @write2_ptr_subreg_arg_two_val_f32 288 ; CI-DAG: s_mov_b32 m0 289 ; GFX9-NOT: m0 290 291 ; GCN-NOT: ds_write2_b32 292 ; GCN: ds_write_b32 293 ; GCN: ds_write_b32 294 ; GCN: s_endpgm 295 define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1, <2 x float addrspace(3)*> %lds.ptr) #0 { 296 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 297 %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i 298 %in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i 299 %val0 = load float, float addrspace(1)* %in0.gep, align 4 300 %val1 = load float, float addrspace(1)* %in1.gep, align 4 301 302 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 303 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 304 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 305 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 306 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 307 308 ; Apply an additional offset after the vector that will be more obviously folded. 309 %gep.1.offset = getelementptr float, float addrspace(3)* %gep.1, i32 8 310 store float %val0, float addrspace(3)* %gep.0, align 4 311 312 %add.x = add nsw i32 %x.i, 8 313 store float %val1, float addrspace(3)* %gep.1.offset, align 4 314 ret void 315 } 316 317 ; GCN-LABEL: @simple_write2_one_val_f64 318 ; CI-DAG: s_mov_b32 m0 319 ; GFX9-NOT: m0 320 321 ; GCN-DAG: {{buffer|global}}_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], 322 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}} 323 ; GCN: ds_write2_b64 [[VPTR]], [[VAL]], [[VAL]] offset1:8 324 ; GCN: s_endpgm 325 define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 { 326 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 327 %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i 328 %val = load double, double addrspace(1)* %in.gep, align 8 329 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 330 store double %val, double addrspace(3)* %arrayidx0, align 8 331 %add.x = add nsw i32 %x.i, 8 332 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 333 store double %val, double addrspace(3)* %arrayidx1, align 8 334 ret void 335 } 336 337 ; GCN-LABEL: @misaligned_simple_write2_one_val_f64 338 ; CI-DAG: s_mov_b32 m0 339 ; GFX9-NOT: m0 340 341 ; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}} 342 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}} 343 ; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:1 344 ; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset0:14 offset1:15 345 ; GCN: s_endpgm 346 define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 { 347 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 348 %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i 349 %val = load double, double addrspace(1)* %in.gep, align 8 350 %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i 351 store double %val, double addrspace(3)* %arrayidx0, align 4 352 %add.x = add nsw i32 %x.i, 7 353 %arrayidx1 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %add.x 354 store double %val, double addrspace(3)* %arrayidx1, align 4 355 ret void 356 } 357 358 ; GCN-LABEL: @simple_write2_two_val_f64 359 ; CI-DAG: s_mov_b32 m0 360 ; GFX9-NOT: m0 361 362 ; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} 363 ; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 364 365 ; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}} 366 ; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off offset:8 367 368 369 ; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}} 370 ; GCN: ds_write2_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8 371 ; GCN: s_endpgm 372 define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 { 373 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 374 %in.gep.0 = getelementptr double, double addrspace(1)* %in, i32 %x.i 375 %in.gep.1 = getelementptr double, double addrspace(1)* %in.gep.0, i32 1 376 %val0 = load volatile double, double addrspace(1)* %in.gep.0, align 8 377 %val1 = load volatile double, double addrspace(1)* %in.gep.1, align 8 378 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 379 store double %val0, double addrspace(3)* %arrayidx0, align 8 380 %add.x = add nsw i32 %x.i, 8 381 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 382 store double %val1, double addrspace(3)* %arrayidx1, align 8 383 ret void 384 } 385 386 @foo = addrspace(3) global [4 x i32] undef, align 4 387 388 ; GCN-LABEL: @store_constant_adjacent_offsets 389 ; CI-DAG: s_mov_b32 m0 390 ; GFX9-NOT: m0 391 392 ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 393 ; GCN: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 394 define amdgpu_kernel void @store_constant_adjacent_offsets() { 395 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 396 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 397 ret void 398 } 399 400 ; GCN-LABEL: @store_constant_disjoint_offsets 401 ; CI-DAG: s_mov_b32 m0 402 ; GFX9-NOT: m0 403 404 ; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b{{$}} 405 ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 406 ; GCN: ds_write2_b32 [[ZERO]], [[VAL]], [[VAL]] offset1:2 407 define amdgpu_kernel void @store_constant_disjoint_offsets() { 408 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 409 store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 410 ret void 411 } 412 413 @bar = addrspace(3) global [4 x i64] undef, align 4 414 415 ; GCN-LABEL: @store_misaligned64_constant_offsets 416 ; CI-DAG: s_mov_b32 m0 417 ; GFX9-NOT: m0 418 419 ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 420 ; GCN-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 421 ; GCN-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3 422 ; GCN: s_endpgm 423 define amdgpu_kernel void @store_misaligned64_constant_offsets() { 424 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 425 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 426 ret void 427 } 428 429 @bar.large = addrspace(3) global [4096 x i64] undef, align 4 430 431 ; GCN-LABEL: @store_misaligned64_constant_large_offsets 432 ; CI-DAG: s_mov_b32 m0 433 ; GFX9-NOT: m0 434 435 ; GCN-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}} 436 ; GCN-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000{{$}} 437 ; GCN-DAG: ds_write2_b32 [[BASE0]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 438 ; GCN-DAG: ds_write2_b32 [[BASE1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1 439 ; GCN: s_endpgm 440 define amdgpu_kernel void @store_misaligned64_constant_large_offsets() { 441 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 442 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 443 ret void 444 } 445 446 @sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4 447 @sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 448 449 define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb, float addrspace(1)* %in) #0 { 450 %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 451 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 452 %val = load float, float addrspace(1)* %in 453 %arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i 454 store float %val, float addrspace(3)* %arrayidx44, align 4 455 %add47 = add nsw i32 %x.i, 1 456 %arrayidx48 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add47 457 store float %val, float addrspace(3)* %arrayidx48, align 4 458 %add51 = add nsw i32 %x.i, 16 459 %arrayidx52 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add51 460 store float %val, float addrspace(3)* %arrayidx52, align 4 461 %add55 = add nsw i32 %x.i, 17 462 %arrayidx56 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add55 463 store float %val, float addrspace(3)* %arrayidx56, align 4 464 %arrayidx60 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %y.i 465 store float %val, float addrspace(3)* %arrayidx60, align 4 466 %add63 = add nsw i32 %y.i, 1 467 %arrayidx64 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add63 468 store float %val, float addrspace(3)* %arrayidx64, align 4 469 %add67 = add nsw i32 %y.i, 32 470 %arrayidx68 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add67 471 store float %val, float addrspace(3)* %arrayidx68, align 4 472 %add71 = add nsw i32 %y.i, 33 473 %arrayidx72 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add71 474 store float %val, float addrspace(3)* %arrayidx72, align 4 475 %add75 = add nsw i32 %y.i, 64 476 %arrayidx76 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add75 477 store float %val, float addrspace(3)* %arrayidx76, align 4 478 %add79 = add nsw i32 %y.i, 65 479 %arrayidx80 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add79 480 store float %val, float addrspace(3)* %arrayidx80, align 4 481 ret void 482 } 483 484 ; GCN-LABEL: {{^}}simple_write2_v4f32_superreg_align4: 485 ; CI: s_mov_b32 m0 486 ; GFX9-NOT: m0 487 488 ; GCN: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}} 489 ; GCN: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}} 490 define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrspace(3)* %out, <4 x float> addrspace(1)* %in) #0 { 491 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 492 %in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in 493 %val0 = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 4 494 %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(3)* %out, i32 %x.i 495 store <4 x float> %val0, <4 x float> addrspace(3)* %out.gep, align 4 496 ret void 497 } 498 499 declare i32 @llvm.amdgcn.workgroup.id.x() #1 500 declare i32 @llvm.amdgcn.workgroup.id.y() #1 501 declare i32 @llvm.amdgcn.workitem.id.x() #1 502 declare i32 @llvm.amdgcn.workitem.id.y() #1 503 504 attributes #0 = { nounwind } 505 attributes #1 = { nounwind readnone speculatable } 506 attributes #2 = { convergent nounwind } 507
