Lines Matching refs:ir_builder_
59 ir_builder_(module_->getContext()),
61 &ir_emitter_context->buffer_assignment(), &ir_builder_, module_,
64 ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags(
74 .EmitReadArrayElement(index, &ir_builder_);
78 *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
121 /*alignment=*/1, GetBasePointer(*operand), &ir_builder_, module_));
151 llvm_ir::EmitTuple(GetIrArray(*tuple, *tuple), base_ptrs, &ir_builder_,
173 ir_builder_.CreateCall(emitted_function, arguments);
193 llvm::Value* source = ir_builder_.CreateLoad(source_address, "source");
200 {output_address->getType()}, &ir_builder_);
205 ir_builder_.CreateAtomicRMW(llvm::AtomicRMWInst::Add, output_address,
219 ir_builder_.CreateAtomicRMW(opcode, output_address, source,
230 ir_builder_.CreateAtomicRMW(opcode, output_address, source,
303 llvm::Type* atomic_type = ir_builder_.getIntNTy(atomic_size);
310 llvm::Value* cas_old_output_address = ir_builder_.CreateAlloca(
312 llvm::Value* cas_new_output_address = ir_builder_.CreateAlloca(
316 llvm::BasicBlock* loop_preheader_bb = ir_builder_.GetInsertBlock();
328 ir_builder_.CreatePtrToInt(output_address, address_int_type);
330 llvm::Value* offset = ir_builder_.CreateAnd(atomic_memory_address, mask);
332 atomic_memory_address = ir_builder_.CreateAnd(atomic_memory_address, mask);
334 ir_builder_.CreateIntToPtr(atomic_memory_address, atomic_address_type);
335 binop_output_address = ir_builder_.CreateAdd(
336 ir_builder_.CreatePtrToInt(cas_new_output_address, address_int_type),
339 ir_builder_.CreateIntToPtr(binop_output_address, element_address_type);
342 ir_builder_.CreateBitCast(output_address, atomic_address_type);
344 ir_builder_.CreateBitCast(cas_new_output_address, element_address_type);
350 ir_builder_.CreateLoad(atomic_memory_address, "cas_old_output");
351 ir_builder_.CreateStore(cas_old_output, cas_old_output_address);
354 ir_builder_.GetInsertPoint(), "atomic_op_loop_exit");
356 llvm::BasicBlock::Create(ir_builder_.getContext(), "atomic_op_loop_body",
357 ir_builder_.GetInsertBlock()->getParent());
358 ir_builder_.SetInsertPoint(loop_body_bb);
366 ir_builder_.CreateLoad(cas_old_output_address, "cas_old_output");
367 ir_builder_.CreateStore(cas_old_output, cas_new_output_address);
374 ir_builder_.CreateLoad(cas_new_output_address, "cas_new_output");
379 llvm::Value* ret_value = ir_builder_.CreateAtomicCmpXchg(
386 ir_builder_.CreateStore(
387 ir_builder_.CreateExtractValue(ret_value, 0, "cas_old_output"),
391 ir_builder_.CreateCondBr(
392 ir_builder_.CreateExtractValue(ret_value, 1, "success"), loop_exit_bb,
397 SetToFirstInsertPoint(loop_exit_bb, &ir_builder_);
431 GetBasePointer(*on_false), &ir_builder_, module_);
454 lhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_);
456 rhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_);
460 return ir_builder_.CreateExtractValue(x, {0});
463 return ir_builder_.CreateExtractValue(x, {1});
465 llvm::Value* real_result = ir_builder_.CreateFSub(
466 ir_builder_.CreateFMul(real(lhs_value), real(rhs_value)),
467 ir_builder_.CreateFMul(imag(lhs_value), imag(rhs_value)));
468 llvm::Value* imag_result = ir_builder_.CreateFAdd(
469 ir_builder_.CreateFMul(real(lhs_value), imag(rhs_value)),
470 ir_builder_.CreateFMul(imag(lhs_value), real(rhs_value)));
472 result = ir_builder_.CreateInsertValue(result, real_result, {0});
473 result = ir_builder_.CreateInsertValue(result, imag_result, {1});
475 result = ir_builder_.CreateFMul(lhs_value, rhs_value);
477 target_array.EmitWriteArrayElement(/*index=*/{}, result, &ir_builder_);
504 llvm_ir::ForLoopNest loop_nest(IrName(dot), &ir_builder_);
528 &ir_builder_);
542 ir_builder_.SetInsertPoint(
545 lhs_array.EmitReadArrayElement(lhs_index, &ir_builder_);
547 rhs_array.EmitReadArrayElement(rhs_index, &ir_builder_);
548 llvm::Value* accum = ir_builder_.CreateLoad(accum_address);
551 #define REAL(x) ir_builder_.CreateExtractValue(x, {0})
552 #define IMAG(x) ir_builder_.CreateExtractValue(x, {1})
553 llvm::Value* product_real = ir_builder_.CreateFSub(
554 ir_builder_.CreateFMul(REAL(lhs_element), REAL(rhs_element)),
555 ir_builder_.CreateFMul(IMAG(lhs_element), IMAG(rhs_element)));
556 llvm::Value* product_imag = ir_builder_.CreateFAdd(
557 ir_builder_.CreateFMul(REAL(lhs_element), IMAG(rhs_element)),
558 ir_builder_.CreateFMul(IMAG(lhs_element), REAL(rhs_element)));
559 updated_accum = ir_builder_.CreateInsertValue(
560 accum, ir_builder_.CreateFAdd(REAL(accum), product_real), {0});
561 updated_accum = ir_builder_.CreateInsertValue(
562 updated_accum, ir_builder_.CreateFAdd(IMAG(accum), product_imag), {1});
566 llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element);
567 updated_accum = ir_builder_.CreateFAdd(accum, product);
569 ir_builder_.CreateStore(updated_accum, accum_address);
586 SetToFirstInsertPoint(reduction_loop->GetExitBasicBlock(), &ir_builder_);
589 ir_builder_.CreateLoad(
591 &ir_builder_);
595 ir_builder_.SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
637 ir_builder_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
639 ir_builder_.CreateStore(
640 ir_builder_.CreateLoad(GetBasePointer(*init_value)),
649 llvm_ir::ForLoopNest loops(IrName(reduce, "inner"), &ir_builder_);
654 SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
674 .EmitArrayElementAddress(input_index, &ir_builder_);
678 SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
679 return ir_builder_.CreateLoad(accumulator_addr);
693 &ir_builder_, GetNestedComputer());
728 .EmitReadArrayElement(index, &ir_builder_);
734 GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
737 GetIrArray(*random, *random), &ir_builder_)
800 "return_buffer", &ir_builder_);
804 parameter_element->getType(), "parameter_buffer", &ir_builder_));
805 ir_builder_.CreateStore(parameter_element, parameter_buffers.back());
809 return ir_builder_.CreateLoad(return_buffer);
