1 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s 2 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s 3 4 ; Test that doing a shift of a pointer with a constant add will be 5 ; folded into the constant offset addressing mode even if the add has 6 ; multiple uses. This is relevant to accessing 2 separate, adjacent 7 ; LDS globals. 8 9 10 declare i32 @llvm.amdgcn.workitem.id.x() #1 11 12 @lds0 = addrspace(3) global [512 x float] undef, align 4 13 @lds1 = addrspace(3) global [512 x float] undef, align 4 14 15 16 ; Make sure the (add tid, 2) << 2 gets folded into the ds's offset as (tid << 2) + 8 17 18 ; SI-LABEL: {{^}}load_shl_base_lds_0: 19 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 20 ; SI: ds_read_b32 {{v[0-9]+}}, [[PTR]] offset:8 21 ; SI: s_endpgm 22 define void @load_shl_base_lds_0(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 23 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 24 %idx.0 = add nsw i32 %tid.x, 2 25 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds0, i32 0, i32 %idx.0 26 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 27 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 28 store float %val0, float addrspace(1)* %out 29 ret void 30 } 31 32 ; Make sure once the first use is folded into the addressing mode, the 33 ; remaining add use goes through the normal shl + add constant fold. 34 35 ; SI-LABEL: {{^}}load_shl_base_lds_1: 36 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 37 ; SI: ds_read_b32 [[RESULT:v[0-9]+]], [[PTR]] offset:8 38 ; SI: v_add_i32_e32 [[ADDUSE:v[0-9]+]], vcc, 8, v{{[0-9]+}} 39 ; SI-DAG: buffer_store_dword [[RESULT]] 40 ; SI-DAG: buffer_store_dword [[ADDUSE]] 41 ; SI: s_endpgm 42 define void @load_shl_base_lds_1(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 43 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 44 %idx.0 = add nsw i32 %tid.x, 2 45 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds0, i32 0, i32 %idx.0 46 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 47 %shl_add_use = shl i32 %idx.0, 2 48 store i32 %shl_add_use, i32 addrspace(1)* %add_use, align 4 49 store float %val0, float addrspace(1)* %out 50 ret void 51 } 52 53 @maxlds = addrspace(3) global [65536 x i8] undef, align 4 54 55 ; SI-LABEL: {{^}}load_shl_base_lds_max_offset 56 ; SI: ds_read_u8 v{{[0-9]+}}, v{{[0-9]+}} offset:65535 57 ; SI: s_endpgm 58 define void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)* %lds, i32 addrspace(1)* %add_use) #0 { 59 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 60 %idx.0 = add nsw i32 %tid.x, 65535 61 %arrayidx0 = getelementptr inbounds [65536 x i8], [65536 x i8] addrspace(3)* @maxlds, i32 0, i32 %idx.0 62 %val0 = load i8, i8 addrspace(3)* %arrayidx0 63 store i32 %idx.0, i32 addrspace(1)* %add_use 64 store i8 %val0, i8 addrspace(1)* %out 65 ret void 66 } 67 68 ; The two globals are placed adjacent in memory, so the same base 69 ; pointer can be used with an offset into the second one. 70 71 ; SI-LABEL: {{^}}load_shl_base_lds_2: 72 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 73 ; SI: s_mov_b32 m0, -1 74 ; SI-NEXT: ds_read2st64_b32 {{v\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:1 offset1:9 75 ; SI: s_endpgm 76 define void @load_shl_base_lds_2(float addrspace(1)* %out) #0 { 77 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 78 %idx.0 = add nsw i32 %tid.x, 64 79 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds0, i32 0, i32 %idx.0 80 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 81 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds1, i32 0, i32 %idx.0 82 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 83 %sum = fadd float %val0, %val1 84 store float %sum, float addrspace(1)* %out, align 4 85 ret void 86 } 87 88 ; SI-LABEL: {{^}}store_shl_base_lds_0: 89 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 90 ; SI: ds_write_b32 [[PTR]], {{v[0-9]+}} offset:8 91 ; SI: s_endpgm 92 define void @store_shl_base_lds_0(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 93 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 94 %idx.0 = add nsw i32 %tid.x, 2 95 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds0, i32 0, i32 %idx.0 96 store float 1.0, float addrspace(3)* %arrayidx0, align 4 97 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 98 ret void 99 } 100 101 102 ; -------------------------------------------------------------------------------- 103 ; Atomics. 104 105 @lds2 = addrspace(3) global [512 x i32] undef, align 4 106 107 ; define void @atomic_load_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 108 ; %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 109 ; %idx.0 = add nsw i32 %tid.x, 2 110 ; %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 111 ; %val = load atomic i32, i32 addrspace(3)* %arrayidx0 seq_cst, align 4 112 ; store i32 %val, i32 addrspace(1)* %out, align 4 113 ; store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 114 ; ret void 115 ; } 116 117 118 ; SI-LABEL: {{^}}atomic_cmpxchg_shl_base_lds_0: 119 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 120 ; SI: ds_cmpst_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}}, {{v[0-9]+}} offset:8 121 ; SI: s_endpgm 122 define void @atomic_cmpxchg_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use, i32 %swap) #0 { 123 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 124 %idx.0 = add nsw i32 %tid.x, 2 125 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 126 %pair = cmpxchg i32 addrspace(3)* %arrayidx0, i32 7, i32 %swap seq_cst monotonic 127 %result = extractvalue { i32, i1 } %pair, 0 128 store i32 %result, i32 addrspace(1)* %out, align 4 129 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 130 ret void 131 } 132 133 ; SI-LABEL: {{^}}atomic_swap_shl_base_lds_0: 134 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 135 ; SI: ds_wrxchg_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 136 ; SI: s_endpgm 137 define void @atomic_swap_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 138 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 139 %idx.0 = add nsw i32 %tid.x, 2 140 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 141 %val = atomicrmw xchg i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 142 store i32 %val, i32 addrspace(1)* %out, align 4 143 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 144 ret void 145 } 146 147 ; SI-LABEL: {{^}}atomic_add_shl_base_lds_0: 148 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 149 ; SI: ds_add_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 150 ; SI: s_endpgm 151 define void @atomic_add_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 152 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 153 %idx.0 = add nsw i32 %tid.x, 2 154 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 155 %val = atomicrmw add i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 156 store i32 %val, i32 addrspace(1)* %out, align 4 157 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 158 ret void 159 } 160 161 ; SI-LABEL: {{^}}atomic_sub_shl_base_lds_0: 162 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 163 ; SI: ds_sub_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 164 ; SI: s_endpgm 165 define void @atomic_sub_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 166 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 167 %idx.0 = add nsw i32 %tid.x, 2 168 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 169 %val = atomicrmw sub i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 170 store i32 %val, i32 addrspace(1)* %out, align 4 171 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 172 ret void 173 } 174 175 ; SI-LABEL: {{^}}atomic_and_shl_base_lds_0: 176 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 177 ; SI: ds_and_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 178 ; SI: s_endpgm 179 define void @atomic_and_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 180 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 181 %idx.0 = add nsw i32 %tid.x, 2 182 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 183 %val = atomicrmw and i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 184 store i32 %val, i32 addrspace(1)* %out, align 4 185 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 186 ret void 187 } 188 189 ; SI-LABEL: {{^}}atomic_or_shl_base_lds_0: 190 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 191 ; SI: ds_or_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 192 ; SI: s_endpgm 193 define void @atomic_or_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 194 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 195 %idx.0 = add nsw i32 %tid.x, 2 196 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 197 %val = atomicrmw or i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 198 store i32 %val, i32 addrspace(1)* %out, align 4 199 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 200 ret void 201 } 202 203 ; SI-LABEL: {{^}}atomic_xor_shl_base_lds_0: 204 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 205 ; SI: ds_xor_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 206 ; SI: s_endpgm 207 define void @atomic_xor_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 208 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 209 %idx.0 = add nsw i32 %tid.x, 2 210 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 211 %val = atomicrmw xor i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 212 store i32 %val, i32 addrspace(1)* %out, align 4 213 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 214 ret void 215 } 216 217 ; define void @atomic_nand_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 218 ; %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 219 ; %idx.0 = add nsw i32 %tid.x, 2 220 ; %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 221 ; %val = atomicrmw nand i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 222 ; store i32 %val, i32 addrspace(1)* %out, align 4 223 ; store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 224 ; ret void 225 ; } 226 227 ; SI-LABEL: {{^}}atomic_min_shl_base_lds_0: 228 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 229 ; SI: ds_min_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 230 ; SI: s_endpgm 231 define void @atomic_min_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 232 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 233 %idx.0 = add nsw i32 %tid.x, 2 234 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 235 %val = atomicrmw min i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 236 store i32 %val, i32 addrspace(1)* %out, align 4 237 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 238 ret void 239 } 240 241 ; SI-LABEL: {{^}}atomic_max_shl_base_lds_0: 242 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 243 ; SI: ds_max_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 244 ; SI: s_endpgm 245 define void @atomic_max_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 246 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 247 %idx.0 = add nsw i32 %tid.x, 2 248 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 249 %val = atomicrmw max i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 250 store i32 %val, i32 addrspace(1)* %out, align 4 251 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 252 ret void 253 } 254 255 ; SI-LABEL: {{^}}atomic_umin_shl_base_lds_0: 256 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 257 ; SI: ds_min_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 258 ; SI: s_endpgm 259 define void @atomic_umin_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 260 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 261 %idx.0 = add nsw i32 %tid.x, 2 262 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 263 %val = atomicrmw umin i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 264 store i32 %val, i32 addrspace(1)* %out, align 4 265 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 266 ret void 267 } 268 269 ; SI-LABEL: {{^}}atomic_umax_shl_base_lds_0: 270 ; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}} 271 ; SI: ds_max_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8 272 ; SI: s_endpgm 273 define void @atomic_umax_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 { 274 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 275 %idx.0 = add nsw i32 %tid.x, 2 276 %arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds2, i32 0, i32 %idx.0 277 %val = atomicrmw umax i32 addrspace(3)* %arrayidx0, i32 3 seq_cst 278 store i32 %val, i32 addrspace(1)* %out, align 4 279 store i32 %idx.0, i32 addrspace(1)* %add_use, align 4 280 ret void 281 } 282 283 attributes #0 = { nounwind } 284 attributes #1 = { nounwind readnone } 285
