Automated sync from github.com/tensorflow/tensorflow (#2445)
BUG=automated sync from upstream
NO_CHECK_TFLITE_FILES=automated sync from upstream
diff --git a/tensorflow/lite/kernels/internal/common.cc b/tensorflow/lite/kernels/internal/common.cc
index 1654ab8..fabb020 100644
--- a/tensorflow/lite/kernels/internal/common.cc
+++ b/tensorflow/lite/kernels/internal/common.cc
@@ -17,6 +17,53 @@
namespace tflite {
+// Single-rounding MultiplyByQuantizedMultiplier
+#if TFLITE_SINGLE_ROUNDING
+int32_t MultiplyByQuantizedMultiplier(int32_t x, int32_t quantized_multiplier,
+ int shift) {
+ TFLITE_DCHECK(quantized_multiplier >= 0);
+ TFLITE_DCHECK(shift >= -31 && shift <= 30);
+
+ const int64_t total_shift = 31 - shift;
+ const int64_t round = static_cast<int64_t>(1) << (total_shift - 1);
+ int64_t result = x * static_cast<int64_t>(quantized_multiplier) + round;
+ result = result >> total_shift;
+
+ TFLITE_DCHECK(result >= std::numeric_limits<int32_t>::min() &&
+ result <= std::numeric_limits<int32_t>::max());
+ return static_cast<int32_t>(result);
+}
+
+int32_t MultiplyByQuantizedMultiplier(int64_t x, int32_t quantized_multiplier,
+ int shift) {
+ // Inputs:
+ // - quantized_multiplier has fixed point at bit 31
+ // - shift is -31 to +7 (negative for right shift)
+ //
+ // Assumptions: The following input ranges are assumed
+ // - quantize_scale>=0 (the usual range is (1<<30) to (1>>31)-1)
+ // - scaling is chosen so final scaled result fits in int32_t
+ // - input x is in the range -(1<<47) <= x < (1<<47)
+ TFLITE_DCHECK(quantized_multiplier >= 0);
+ TFLITE_DCHECK(shift >= -31 && shift < 8);
+ TFLITE_DCHECK(x >= -(static_cast<int64_t>(1) << 47) &&
+ x < (static_cast<int64_t>(1) << 47));
+
+ const int32_t reduced_multiplier =
+ (quantized_multiplier < 0x7FFF0000)
+ ? ((quantized_multiplier + (1 << 15)) >> 16)
+ : 0x7FFF;
+ const int64_t total_shift = 15 - shift;
+ const int64_t round = static_cast<int64_t>(1) << (total_shift - 1);
+ int64_t result = x * static_cast<int64_t>(reduced_multiplier) + round;
+ result = result >> total_shift;
+
+ TFLITE_DCHECK(result >= std::numeric_limits<int32_t>::min() &&
+ result <= std::numeric_limits<int32_t>::max());
+ return static_cast<int32_t>(result);
+}
+// Double-rounding MultiplyByQuantizedMultiplier
+#else
int32_t MultiplyByQuantizedMultiplier(int32_t x, int32_t quantized_multiplier,
int shift) {
using gemmlowp::RoundingDivideByPOT;
@@ -51,5 +98,6 @@
int32_t result = x >> total_shift;
return result;
}
+#endif // TFLITE_SINGLE_ROUNDING
} // namespace tflite
diff --git a/tensorflow/lite/kernels/internal/common.h b/tensorflow/lite/kernels/internal/common.h
index 14d8599..9761a8c 100644
--- a/tensorflow/lite/kernels/internal/common.h
+++ b/tensorflow/lite/kernels/internal/common.h
@@ -257,24 +257,14 @@
#endif
}
+TFLITE_NOINLINE int32_t MultiplyByQuantizedMultiplier(
+ int32_t x, int32_t quantized_multiplier, int shift);
+
+TFLITE_NOINLINE int32_t MultiplyByQuantizedMultiplier(
+ int64_t x, int32_t quantized_multiplier, int shift);
+
// Single-rounding MultiplyByQuantizedMultiplier
#if TFLITE_SINGLE_ROUNDING
-inline int32_t MultiplyByQuantizedMultiplier(int32_t x,
- int32_t quantized_multiplier,
- int shift) {
- TFLITE_DCHECK(quantized_multiplier >= 0);
- TFLITE_DCHECK(shift >= -31 && shift <= 30);
-
- const int64_t total_shift = 31 - shift;
- const int64_t round = static_cast<int64_t>(1) << (total_shift - 1);
- int64_t result = x * static_cast<int64_t>(quantized_multiplier) + round;
- result = result >> total_shift;
-
- TFLITE_DCHECK(result >= std::numeric_limits<int32_t>::min() &&
- result <= std::numeric_limits<int32_t>::max());
- return static_cast<int32_t>(result);
-}
-
inline int32_t MultiplyByQuantizedMultiplierSmallerThanOneExp(
int32_t x, int32_t quantized_multiplier, int shift) {
TFLITE_DCHECK_LE(shift, 0);
@@ -287,36 +277,6 @@
return MultiplyByQuantizedMultiplier(x, quantized_multiplier, shift);
}
-inline int32_t MultiplyByQuantizedMultiplier(int64_t x,
- int32_t quantized_multiplier,
- int shift) {
- // Inputs:
- // - quantized_multiplier has fixed point at bit 31
- // - shift is -31 to +7 (negative for right shift)
- //
- // Assumptions: The following input ranges are assumed
- // - quantize_scale>=0 (the usual range is (1<<30) to (1>>31)-1)
- // - scaling is chosen so final scaled result fits in int32_t
- // - input x is in the range -(1<<47) <= x < (1<<47)
- TFLITE_DCHECK(quantized_multiplier >= 0);
- TFLITE_DCHECK(shift >= -31 && shift < 8);
- TFLITE_DCHECK(x >= -(static_cast<int64_t>(1) << 47) &&
- x < (static_cast<int64_t>(1) << 47));
-
- const int32_t reduced_multiplier =
- (quantized_multiplier < 0x7FFF0000)
- ? ((quantized_multiplier + (1 << 15)) >> 16)
- : 0x7FFF;
- const int64_t total_shift = 15 - shift;
- const int64_t round = static_cast<int64_t>(1) << (total_shift - 1);
- int64_t result = x * static_cast<int64_t>(reduced_multiplier) + round;
- result = result >> total_shift;
-
- TFLITE_DCHECK(result >= std::numeric_limits<int32_t>::min() &&
- result <= std::numeric_limits<int32_t>::max());
- return static_cast<int32_t>(result);
-}
-
#ifdef USE_NEON
inline int32x4x4_t MultiplyByQuantizedMultiplier4Rows(
int32x4x4_t input_val, int32_t quantized_multiplier, int shift) {
@@ -366,12 +326,6 @@
quantized_multiplier);
}
-TFLITE_NOINLINE int32_t MultiplyByQuantizedMultiplier(
- int32_t x, int32_t quantized_multiplier, int shift);
-
-TFLITE_NOINLINE int32_t MultiplyByQuantizedMultiplier(
- int64_t x, int32_t quantized_multiplier, int shift);
-
#ifdef USE_NEON
// Round uses ARM's rounding shift right.
inline int32x4x4_t MultiplyByQuantizedMultiplier4Rows(
diff --git a/tensorflow/lite/kernels/internal/portable_tensor_utils.h b/tensorflow/lite/kernels/internal/portable_tensor_utils.h
index 03bfdc8..d37fe6e 100644
--- a/tensorflow/lite/kernels/internal/portable_tensor_utils.h
+++ b/tensorflow/lite/kernels/internal/portable_tensor_utils.h
@@ -241,7 +241,8 @@
const int32_t* __restrict__ indices, int m_rows, int m_cols,
const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector,
int n_batch, const int32_t input_offset, const int32_t output_multiplier,
- const int32_t output_shift, const int32_t output_offset,
+ int32_t output_shift, const int32_t* per_channel_scale,
+ const int32_t* per_channel_shift, int32_t output_offset,
const int32_t output_activation_min, const int32_t output_activation_max,
int8_t* __restrict__ result);
diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
index b519d81..7d40df4 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
@@ -157,7 +157,7 @@
*result += dotprod * batch_scaling_factor;
++result;
} // for row
- } // for batch
+ } // for batch
}
void PortableMatrixBatchVectorMultiplyAccumulate(
@@ -200,7 +200,7 @@
*result += dotprod * scale;
++result;
} // for row
- } // for batch
+ } // for batch
}
void PortableSparseMatrixBatchVectorMultiplyAccumulate1x4(
@@ -232,7 +232,8 @@
const int32_t* __restrict__ indices, int m_rows, int m_cols,
const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector,
int n_batch, const int32_t input_offset, const int32_t output_multiplier,
- const int32_t output_shift, const int32_t output_offset,
+ const int32_t output_shift, const int32_t* per_channel_scale,
+ const int32_t* per_channel_shift, const int32_t output_offset,
const int32_t output_activation_min, const int32_t output_activation_max,
int8_t* __restrict__ result) {
const int kBlockSize = 16;
@@ -252,8 +253,10 @@
}
}
const int32_t bias_value = bias_vector != nullptr ? bias_vector[row] : 0;
- dot_prod = MultiplyByQuantizedMultiplier(dot_prod + bias_value,
- output_multiplier, output_shift);
+ dot_prod = MultiplyByQuantizedMultiplier(
+ dot_prod + bias_value,
+ per_channel_scale ? per_channel_scale[row] : output_multiplier,
+ per_channel_shift ? per_channel_shift[row] : output_shift);
dot_prod += output_offset;
result[batch * m_rows + row] =
static_cast<int8_t>(ActivationFunctionWithMinMax(
@@ -319,14 +322,14 @@
for (int c = 0; c < kBlockSize; c++) {
dotprod += (*row_ptr++) * (*vector_block_ptr++);
} // for block
- } // for num_nonzero_blocks
+ } // for num_nonzero_blocks
float scaling_factor = batch_scaling_factor;
if (per_channel_scale) {
scaling_factor *= per_channel_scale[row];
}
result[batch * m_rows + row] += dotprod * scaling_factor;
} // for row
- } // for batch
+ } // for batch
}
template <typename T>
diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
index ab0185e..7c623f7 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
@@ -116,14 +116,16 @@
const int32_t* __restrict__ indices, int m_rows, int m_cols,
const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector,
int n_batch, const int32_t input_offset, const int32_t output_multiplier,
- const int32_t output_shift, const int32_t output_offset,
+ const int32_t output_shift, const int32_t* per_channel_scale,
+ const int32_t* per_channel_shift, const int32_t output_offset,
const int32_t output_activation_min, const int32_t output_activation_max,
int8_t* __restrict__ result) {
PortableSparseMatrixBatchVectorMultiplyAccumulate1x16(
matrix, segments, indices, m_rows, m_cols, vector, bias_vector, n_batch,
- input_offset, output_multiplier, output_shift, output_offset,
- output_activation_min, output_activation_max, result);
+ input_offset, output_multiplier, output_shift, per_channel_scale,
+ per_channel_shift, output_offset, output_activation_min,
+ output_activation_max, result);
}
void SparseMatrixBatchVectorMultiplyAccumulate(
diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils_impl.h b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils_impl.h
index 6a1f18e..11765ec 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils_impl.h
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils_impl.h
@@ -92,7 +92,8 @@
const int32_t* __restrict__ indices, int m_rows, int m_cols,
const int8_t* __restrict__ vector, const int32_t* __restrict__ bias_vector,
int n_batch, const int32_t input_offset, const int32_t output_multiplier,
- const int32_t output_shift, const int32_t output_offset,
+ int32_t output_shift, const int32_t* per_channel_scale,
+ const int32_t* per_channel_shift, int32_t output_offset,
const int32_t output_activation_min, const int32_t output_activation_max,
int8_t* __restrict__ result);
