Lines Matching full:bits
44 dequant = 2^(scale_factor+1) * ((raw * 2.0 + 1.0) / ((2^bits) - 1) - 1)
46 2 <= bits <= 16
47 0 <= raw < (2^bits)-1 (the -1 is because quantized values with all 1's are
55 dequant_long_scaled table for 16 bits is as accurate as possible, since it has
63 dequant_long_long[bits] = round(2^31 * 1/((2^bits - 1) / 1.38...) for 2 <=
64 bits <= 16
71 dequant_long_scaled[bits] <= 2^31 / ((2^bits - 1)) for 2 <= bits <= 16
77 d = 2 * raw + 1 1 <= d <= 2^bits - 2
79 d' = d * dequant_long[bits]
81 d * dequant_long_scaled[bits] <= (2^bits - 2) * (2^31 /
82 (2^bits - 1))
83 d * dequant_long_scaled[bits] <= 2^31 * (2^bits - 2)/(2^bits -
91 d' =~ 2^31 * (raw * 2.0 + 1.0) / (2^bits - 1) / 1.38...
93 result = d' - 2^31/1.38... =~ 2^31 * ((raw * 2.0 + 1.0) / (2^bits - 1) - 1) /
123 /** Scales x by y bits to the right, adding a rounding factor.
133 INLINE float dequant_float(uint32_t raw, OI_UINT scale_factor, OI_UINT bits) {
135 ((raw * 2.0f + 1.0f) / ((1 << bits) - 1.0f) - 1.0f);
141 * values with all 1-bits set can, theoretically, trigger this assert. This
152 OI_UINT bits) {
157 OI_ASSERT(bits <= 16);
159 if (bits <= 1) {
164 d *= dequant_long_scaled[bits];
172 float_result = dequant_float(raw, scale_factor, bits);
191 OI_UINT bits) {
196 OI_ASSERT(bits <= 16);
198 if (bits <= 1) {
201 if (bits == 16) {
207 d *= dequant_long_unscaled[bits];
