diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h index adb2eac7a..7c4a42bea 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h @@ -2188,7 +2188,7 @@ EIGEN_ALWAYS_INLINE void gemm_cols( gemm_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); template -EIGEN_STRONG_INLINE void gemm_extra_cols( +EIGEN_ALWAYS_INLINE void gemm_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, @@ -2622,7 +2622,7 @@ EIGEN_ALWAYS_INLINE void gemm_complex_cols( gemm_complex_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); template -EIGEN_STRONG_INLINE void gemm_complex_extra_cols( +EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h b/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h index 35e7f673e..1ac66292e 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h @@ -30,6 +30,23 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row( const Packet& pAlpha, const Packet& pMask); +template +EIGEN_ALWAYS_INLINE void gemm_extra_cols( + const DataMapper& res, + const Scalar* blockA, + const Scalar* blockB, + Index depth, + Index strideA, + Index offsetA, + Index strideB, + Index offsetB, + Index col, + Index rows, + Index cols, + Index remaining_rows, + const Packet& pAlpha, + const Packet& pMask); + template EIGEN_ALWAYS_INLINE Packet bmask(const Index remaining_rows); @@ -50,7 +67,7 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row( const Packet& pMask); template -EIGEN_STRONG_INLINE void gemm_complex_extra_cols( +EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols( const DataMapper& res, const Scalar* blockA, const Scalar* blockB, diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h b/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h index b01dc92a4..e4013a747 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h @@ -429,49 +429,6 @@ EIGEN_ALWAYS_INLINE void gemmMMA_cols( } } -#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ - switch(value) { \ - default: \ - MICRO_EXTRA_UNROLL(1) \ - break; \ - case 2: \ - if (is_col || (sizeof(Scalar) == sizeof(float))) { \ - MICRO_EXTRA_UNROLL(2) \ - } \ - break; \ - case 3: \ - if (is_col || (sizeof(Scalar) == sizeof(float))) { \ - MICRO_EXTRA_UNROLL(3) \ - } \ - break; \ - } - -#define MICRO_EXTRA_COLS(N) \ - gemmMMA_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); - -template -EIGEN_STRONG_INLINE void gemmMMA_extra_cols( - const DataMapper& res, - const Scalar* blockA, - const Scalar* blockB, - Index depth, - Index strideA, - Index offsetA, - Index strideB, - Index offsetB, - Index col, - Index rows, - Index cols, - Index remaining_rows, - const Packet& pAlpha, - const Packet& pMask) -{ - MICRO_EXTRA(MICRO_EXTRA_COLS, cols-col, true) -} - -#undef MICRO_EXTRA -#undef MICRO_EXTRA_COLS - template void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { @@ -493,7 +450,7 @@ void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, if (col != cols) { - gemmMMA_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); + gemm_extra_cols(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask); } }