Lines Matching full:emitter
35 def LoadAndDuplicateOffsets(emitter, registers, lanes, offsets):
40 emitter.EmitVLoadA('1.32',
41 [emitter.AllLanes(registers.Low(register)),
42 emitter.AllLanes(registers.High(register))],
43 emitter.DereferenceIncrement(offsets, 32))
50 def GenerateQntLanes(emitter,
60 emitter, registers, qnt_lanes, offsets)
80 emitter.EmitAdd(input_register, last_input_register, stride)
81 emitter.EmitAdd(output_register, last_output_register, destination_stride)
87 def DuplicateRegister(emitter, registers, value):
89 emitter.EmitVDup('32', register, value)
93 def GenerateQuantize(emitter,
102 emitter.EmitVAdd('i32', lane[0], lane[0], lane[1])
105 emitter.EmitVMul('i32', lane[0], lane[0], multiplicative_offset)
108 emitter.EmitVAdd('i32', lane[0], lane[0], rounding_offset)
111 emitter.EmitVShl('s32', lane[0], lane[0], shift)
114 emitter.EmitVQmovn('s32', lane[2], lane[0])
117 emitter.EmitVQmovun('s16', registers.Low(lane_temp), lane_temp)
120 def GenerateLoadQuantizeStore(emitter,
133 emitter.EmitVLoadA('1.32',
138 emitter.DereferenceIncrement(lane.source, 64))
141 emitter.EmitPld(lane.source)
148 GenerateQuantize(emitter,
157 emitter.EmitVStore('1.8',
159 emitter.DereferenceIncrement(lane.output, alignment))
165 def GenerateLoadLeftovers(emitter, registers, leftovers, lanes):
169 emitter.EmitVLoad('1.32',
170 emitter.Lane(registers.Low(lane.load_1), 0),
171 emitter.Dereference(lane.source, None))
174 emitter.EmitVLoad('1.32',
176 emitter.Dereference(lane.source, 64))
179 emitter.EmitVLoad('1.32',
181 emitter.DereferenceIncrement(lane.source, 64))
183 emitter.EmitVLoad('1.32',
184 emitter.Lane(registers.High(lane.load_1), 0),
185 emitter.Dereference(lane.source, None))
188 emitter.EmitVLoadA('1.32',
191 emitter.Dereference(lane.source, 64))
194 emitter.EmitVLoadA('1.32',
197 emitter.DereferenceIncrement(lane.source, 64))
199 emitter.EmitVLoad('1.32',
200 emitter.Lane(registers.Low(lane.load_2), 0),
201 emitter.Dereference(lane.source, None))
204 emitter.EmitVLoadA('1.32',
208 emitter.Dereference(lane.source, 64))
211 emitter.EmitVLoadA('1.32',
215 emitter.DereferenceIncrement(lane.source, 64))
217 emitter.EmitVLoad('1.32',
218 emitter.Lane(registers.High(lane.load_2), 0),
219 emitter.Dereference(lane.source, None))
224 def GenerateStoreLeftovers(emitter, registers, leftovers, lane_temps, lanes):
232 emitter.EmitVStore('1.8', emitter.Lane(lane[0], 0),
233 emitter.Dereference(lane[1], None))
236 emitter.EmitVStore('1.16', emitter.Lane(lane[0], 0),
237 emitter.Dereference(lane[1], None))
240 emitter.EmitVStore('1.16', emitter.Lane(lane[0], 0),
241 emitter.DereferenceIncrement(lane[1], None))
243 emitter.EmitVStore('1.8', emitter.Lane(lane[0], 2),
244 emitter.Dereference(lane[1], None))
247 emitter.EmitVStore('1.32', emitter.Lane(lane[0], 0),
248 emitter.Dereference(lane[1], None))
251 emitter.EmitVStore('1.32', emitter.Lane(lane[0], 0),
252 emitter.DereferenceIncrement(lane[1], None))
254 emitter.EmitVStore('1.8', emitter.Lane(lane[0], 4),
255 emitter.Dereference(lane[1], None))
258 emitter.EmitVStore('1.32', emitter.Lane(lane[0], 0),
259 emitter.DereferenceIncrement(lane[1], None))
261 emitter.EmitVStore('1.16', emitter.Lane(lane[0], 2),
262 emitter.Dereference(lane[1], None))
265 emitter.EmitVStore('1.32', emitter.Lane(lane[0], 0),
266 emitter.DereferenceIncrement(lane[1], None))
268 emitter.EmitVStore('1.16', emitter.Lane(lane[0], 2),
269 emitter.DereferenceIncrement(lane[1], None))
271 emitter.EmitVStore('1.8', emitter.Lane(lane[0], 6),
272 emitter.DereferenceIncrement(lane[1], None))
277 def GenerateLeftoverLoadQuantizeStore(emitter,
289 GenerateLoadLeftovers(emitter, registers, leftovers, lanes)
298 GenerateQuantize(emitter,
306 GenerateStoreLeftovers(emitter, registers, leftovers, lane_temps, lanes)
309 def GenerateQntNx8(emitter, qnt_lanes, leftovers, aligned):
318 emitter.EmitFunctionBeginA(name,
329 emitter.EmitAssert('count %% 8 == %d' % leftovers)
330 emitter.EmitAssert('count >= 8')
331 emitter.EmitAssert('reinterpret_cast<std::uintptr_t>(source) % 8 == 0')
333 emitter.EmitAssert('reinterpret_cast<std::uintptr_t>(destination) % 8 == 0')
335 emitter.EmitAssert(
337 emitter.EmitAsmBegin()
344 emitter, registers, registers.MapParameter('multiplicative_offset'))
346 emitter, registers, registers.MapParameter('rounding_offset'))
347 shift = DuplicateRegister(emitter, registers, registers.MapParameter('shift'))
350 emitter, registers, qnt_lanes,
358 emitter.EmitSubs(count, count, emitter.ImmediateConstant(leftovers))
359 emitter.EmitBeqFront(2)
361 emitter.EmitNewline()
362 emitter.EmitNumericalLabel(1)
363 emitter.EmitSubs(count, count, emitter.ImmediateConstant(8))
365 GenerateLoadQuantizeStore(emitter,
373 emitter.EmitNewline()
374 emitter.EmitBneBack(1)
377 emitter.EmitNumericalLabel(2)
378 GenerateLeftoverLoadQuantizeStore(emitter,
386 emitter.EmitAsmEnd(registers.MappedParameters(),
389 emitter.EmitFunctionEnd()
392 def GenerateFunctions(emitter):
396 GenerateQntNx8(emitter, lanes, leftovers, aligned)
397 emitter.EmitNewline()
