From b00e48a867eab40bca914b7673f2fd43f1114831 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 13:45:56 +0200 Subject: [PATCH 1/9] Improve slice-vectorization logic for redux (significant speed-up for reduxion of blocks) --- Eigen/src/Core/Redux.h | 13 +++++++++++-- test/vectorization_logic.cpp | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index e449ef3ac..25e374c77 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h @@ -32,14 +32,20 @@ public: PacketSize = unpacket_traits::size, InnerMaxSize = int(Evaluator::IsRowMajor) ? Evaluator::MaxColsAtCompileTime - : Evaluator::MaxRowsAtCompileTime + : Evaluator::MaxRowsAtCompileTime, + OuterMaxSize = int(Evaluator::IsRowMajor) + ? Evaluator::MaxRowsAtCompileTime + : Evaluator::MaxColsAtCompileTime, + SliceVectorizedWork = int(InnerMaxSize)==Dynamic ? Dynamic + : int(OuterMaxSize)==Dynamic ? (int(InnerMaxSize)>=int(PacketSize) ? Dynamic : 0) + : (int(InnerMaxSize)/int(PacketSize)) * int(OuterMaxSize) }; enum { MightVectorize = (int(Evaluator::Flags)&ActualPacketAccessBit) && (functor_traits::PacketAccess), MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&LinearAccessBit), - MaySliceVectorize = bool(MightVectorize) && int(InnerMaxSize)>=3*PacketSize + MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork)==Dynamic || int(SliceVectorizedWork)>=3) }; public: @@ -69,12 +75,15 @@ public: EIGEN_DEBUG_VAR(Evaluator::Flags) std::cerr.unsetf(std::ios::hex); EIGEN_DEBUG_VAR(InnerMaxSize) + EIGEN_DEBUG_VAR(OuterMaxSize) + EIGEN_DEBUG_VAR(SliceVectorizedWork) EIGEN_DEBUG_VAR(PacketSize) EIGEN_DEBUG_VAR(MightVectorize) EIGEN_DEBUG_VAR(MayLinearVectorize) EIGEN_DEBUG_VAR(MaySliceVectorize) EIGEN_DEBUG_VAR(Traversal) EIGEN_DEBUG_VAR(UnrollingLimit) + EIGEN_DEBUG_VAR(Unrolling) std::cerr << std::endl; } diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index c15f75103..2d0a8ceb6 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp @@ -230,8 +230,13 @@ struct vectorization_logic VERIFY(test_redux(Matrix44(), LinearVectorizedTraversal,NoUnrolling)); - VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?4:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:4>(1,2), - DefaultTraversal,CompleteUnrolling)); + if(PacketSize>1) { + VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?4:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:4>(1,2), + SliceVectorizedTraversal,CompleteUnrolling)); + + VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?2:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:2>(1,2), + DefaultTraversal,CompleteUnrolling)); + } VERIFY(test_redux(Matrix44c().template block<2*PacketSize,1>(1,2), LinearVectorizedTraversal,CompleteUnrolling)); @@ -375,8 +380,13 @@ struct vectorization_logic_half VERIFY(test_redux(Matrix35(), LinearVectorizedTraversal,CompleteUnrolling)); - VERIFY(test_redux(Matrix57().template block(1,0), - DefaultTraversal,CompleteUnrolling)); + VERIFY(test_redux(Matrix57().template block(1,0), + SliceVectorizedTraversal,CompleteUnrolling)); + + if(PacketSize>1) { + VERIFY(test_redux(Matrix57().template block(1,0), + DefaultTraversal,CompleteUnrolling)); + } VERIFY((test_assign< Map, AlignedMax, InnerStride<3*PacketSize> >, From 91716f03a782992137f2e833f5fbd8fff6000ea6 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 14:32:24 +0200 Subject: [PATCH 2/9] Fix vectorization logic unit test for AVX512 --- test/vectorization_logic.cpp | 37 +++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index 2d0a8ceb6..8efee92a1 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp @@ -37,6 +37,7 @@ using internal::demangle_unrolling; template bool test_assign(const Dst&, const Src&, int traversal, int unrolling) { + EIGEN_PREDICATE_SAME_MATRIX_SIZE(Dst,Src); typedef internal::copy_using_evaluator_traits,internal::evaluator, internal::assign_op > traits; bool res = traits::Traversal==traversal; if(unrolling==InnerUnrolling+CompleteUnrolling) @@ -61,6 +62,7 @@ bool test_assign(const Dst&, const Src&, int traversal, int unrolling) template bool test_assign(int traversal, int unrolling) { + EIGEN_PREDICATE_SAME_MATRIX_SIZE(Dst,Src); typedef internal::copy_using_evaluator_traits,internal::evaluator, internal::assign_op > traits; bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) @@ -117,26 +119,26 @@ struct vectorization_logic typedef Matrix VectorX; typedef Matrix MatrixXX; typedef Matrix Matrix11; - typedef Matrix Matrix22; + typedef Matrix Matrix22; typedef Matrix Matrix44; typedef Matrix Matrix44u; typedef Matrix Matrix44c; typedef Matrix Matrix44r; typedef Matrix Matrix1; typedef Matrix Matrix1u; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix Matrix3; #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT @@ -202,7 +204,7 @@ struct vectorization_logic VERIFY(test_assign(Matrix11(), Matrix11()+Matrix11(),InnerVectorizedTraversal,CompleteUnrolling)); - VERIFY(test_assign(Matrix11(),Matrix().template block(2,3)+Matrix().template block(8,4), + VERIFY(test_assign(Matrix11(),Matrix().template block(2,3)+Matrix().template block(3,2), (EIGEN_UNALIGNED_VECTORIZE) ? InnerVectorizedTraversal : DefaultTraversal, CompleteUnrolling|InnerUnrolling)); VERIFY(test_assign(Vector1(),Matrix11()*Vector1(), @@ -294,19 +296,19 @@ struct vectorization_logic_half // typedef Matrix Matrix44r; typedef Matrix Matrix1; typedef Matrix Matrix1u; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix Matrix3; #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT @@ -359,7 +361,8 @@ struct vectorization_logic_half NoUnrolling)); VERIFY(test_assign(Matrix11(),Matrix().template block(2,3)+Matrix().template block(8,4), - EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : DefaultTraversal,InnerUnrolling+CompleteUnrolling)); + VERIFY(test_assign(Vector1(),Matrix11()*Vector1(), InnerVectorizedTraversal,CompleteUnrolling)); @@ -391,10 +394,10 @@ struct vectorization_logic_half VERIFY((test_assign< Map, AlignedMax, InnerStride<3*PacketSize> >, Matrix - >(DefaultTraversal,CompleteUnrolling))); + >(DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling))); VERIFY((test_assign(Matrix57(), Matrix()*Matrix(), - InnerVectorizedTraversal, InnerUnrolling|CompleteUnrolling))); + InnerVectorizedTraversal, InnerUnrolling+CompleteUnrolling))); #endif } }; From 371068992a509ec43f9f147db98b460ad26db2a4 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 14:32:39 +0200 Subject: [PATCH 3/9] Add more debug output --- Eigen/src/Core/AssignEvaluator.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 362d905d2..83cec500f 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h @@ -172,6 +172,8 @@ public: EIGEN_DEBUG_VAR(MaySliceVectorize) std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl; EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost) + EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost) + EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime) EIGEN_DEBUG_VAR(UnrollingLimit) EIGEN_DEBUG_VAR(MayUnrollCompletely) EIGEN_DEBUG_VAR(MayUnrollInner) From 03a0cb2b72b6be3ad28aa932acefabb97b5ba0fd Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 14:40:26 +0200 Subject: [PATCH 4/9] fix unalignedcount for avx512 --- test/unalignedcount.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/unalignedcount.cpp b/test/unalignedcount.cpp index 069fc1bb9..52cdd9e1d 100644 --- a/test/unalignedcount.cpp +++ b/test/unalignedcount.cpp @@ -30,7 +30,14 @@ static int nb_storeu; EIGEN_DECLARE_TEST(unalignedcount) { - #if defined(EIGEN_VECTORIZE_AVX) + #if defined(EIGEN_VECTORIZE_AVX512) + VectorXf a(48), b(48); + VERIFY_ALIGNED_UNALIGNED_COUNT(a += b, 6, 0, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) += b.segment(0,48), 3, 3, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) -= b.segment(0,48), 3, 3, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) *= 3.5, 3, 0, 3, 0); + VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,48) /= 3.5, 3, 0, 3, 0); + #elif defined(EIGEN_VECTORIZE_AVX) VectorXf a(40), b(40); VERIFY_ALIGNED_UNALIGNED_COUNT(a += b, 10, 0, 5, 0); VERIFY_ALIGNED_UNALIGNED_COUNT(a.segment(0,40) += b.segment(0,40), 5, 5, 5, 0); From 4291f167eeb00d01adac7208849ef4de4a8ebae4 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 14:53:43 +0200 Subject: [PATCH 5/9] Add missing plugins to DynamicSparseMatrix -- fix sparse_extra_3 --- .../Eigen/src/SparseExtra/DynamicSparseMatrix.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h b/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h index 3f1ff14ad..42c99e467 100644 --- a/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h +++ b/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h @@ -228,6 +228,9 @@ template EIGEN_DEPRECATED inline DynamicSparseMatrix() : m_innerSize(0), m_data(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif eigen_assert(innerSize()==0 && outerSize()==0); } @@ -235,6 +238,9 @@ template EIGEN_DEPRECATED inline DynamicSparseMatrix(Index rows, Index cols) : m_innerSize(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif resize(rows, cols); } @@ -243,12 +249,18 @@ template EIGEN_DEPRECATED explicit inline DynamicSparseMatrix(const SparseMatrixBase& other) : m_innerSize(0) { - Base::operator=(other.derived()); + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif + Base::operator=(other.derived()); } inline DynamicSparseMatrix(const DynamicSparseMatrix& other) : Base(), m_innerSize(0) { + #ifdef EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN + #endif *this = other.derived(); } From e3c82890474fa3ab4b49a0c97b8b4eccce93a77a Mon Sep 17 00:00:00 2001 From: Christoph Hertzberg Date: Fri, 21 Sep 2018 21:15:51 +0200 Subject: [PATCH 6/9] Replace unused PREDICATE by corresponding STATIC_ASSERT --- test/vectorization_logic.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index 8efee92a1..e2146eef3 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp @@ -37,7 +37,7 @@ using internal::demangle_unrolling; template bool test_assign(const Dst&, const Src&, int traversal, int unrolling) { - EIGEN_PREDICATE_SAME_MATRIX_SIZE(Dst,Src); + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src); typedef internal::copy_using_evaluator_traits,internal::evaluator, internal::assign_op > traits; bool res = traits::Traversal==traversal; if(unrolling==InnerUnrolling+CompleteUnrolling) @@ -62,7 +62,7 @@ bool test_assign(const Dst&, const Src&, int traversal, int unrolling) template bool test_assign(int traversal, int unrolling) { - EIGEN_PREDICATE_SAME_MATRIX_SIZE(Dst,Src); + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src); typedef internal::copy_using_evaluator_traits,internal::evaluator, internal::assign_op > traits; bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) From c696dbcaa6e17cdfa6c9ff37dadf89cf4b707504 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 23:02:33 +0200 Subject: [PATCH 7/9] Fiw shadowing of last and all --- .../src/Tensor/TensorContractionMapper.h | 6 +-- .../CXX11/src/Tensor/TensorDeviceThreadPool.h | 12 ++--- .../Eigen/CXX11/src/Tensor/TensorExecutor.h | 48 +++++++++---------- .../Eigen/CXX11/src/Tensor/TensorPadding.h | 40 ++++++++-------- .../CXX11/src/Tensor/TensorReductionGpu.h | 4 +- .../Eigen/CXX11/src/ThreadPool/EventCount.h | 8 ++-- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h index dbb0f76bb..2d3b69128 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h @@ -255,7 +255,7 @@ class BaseTensorContractionMapper : public SimpleTensorContractionMapper indexPair = this->computeIndexPair(i, j, packet_size - 1); const Index first = indexPair.first; - const Index last = indexPair.second; + const Index lastIdx = indexPair.second; // We can always do optimized packet reads from left hand side right now, because // the vertical matrix dimension on the left hand side is never contracting. @@ -263,7 +263,7 @@ class BaseTensorContractionMapper : public SimpleTensorContractionMapper::value <= 1 || !inner_dim_reordered) && - (last - first) == (packet_size - 1)) { + (lastIdx - first) == (packet_size - 1)) { return this->m_tensor.template packet(first); } @@ -276,7 +276,7 @@ class BaseTensorContractionMapper : public SimpleTensorContractionMapperm_tensor.coeff(internal_pair.first); data[k + 1] = this->m_tensor.coeff(internal_pair.second); } - data[packet_size - 1] = this->m_tensor.coeff(last); + data[packet_size - 1] = this->m_tensor.coeff(lastIdx); return pload(data); } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h index 6fc6688d3..1612c004b 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h @@ -213,17 +213,17 @@ struct ThreadPoolDevice { // block_count leaves that do actual computations. Barrier barrier(static_cast(block_count)); std::function handleRange; - handleRange = [=, &handleRange, &barrier, &f](Index first, Index last) { - if (last - first <= block_size) { + handleRange = [=, &handleRange, &barrier, &f](Index firstIdx, Index lastIdx) { + if (lastIdx - firstIdx <= block_size) { // Single block or less, execute directly. - f(first, last); + f(firstIdx, lastIdx); barrier.Notify(); return; } // Split into halves and submit to the pool. - Index mid = first + divup((last - first) / 2, block_size) * block_size; - pool_->Schedule([=, &handleRange]() { handleRange(mid, last); }); - handleRange(first, mid); + Index mid = firstIdx + divup((lastIdx - firstIdx) / 2, block_size) * block_size; + pool_->Schedule([=, &handleRange]() { handleRange(mid, lastIdx); }); + handleRange(firstIdx, mid); }; handleRange(0, n); barrier.Wait(); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h index bfe1f97b8..1c44541bd 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h @@ -165,11 +165,11 @@ class TensorExecutor struct EvalRange { - static void run(Evaluator* evaluator_in, const StorageIndex first, - const StorageIndex last) { + static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, + const StorageIndex lastIdx) { Evaluator evaluator = *evaluator_in; - eigen_assert(last >= first); - for (StorageIndex i = first; i < last; ++i) { + eigen_assert(lastIdx >= firstIdx); + for (StorageIndex i = firstIdx; i < lastIdx; ++i) { evaluator.evalScalar(i); } } @@ -182,14 +182,14 @@ struct EvalRange { static const int PacketSize = unpacket_traits::size; - static void run(Evaluator* evaluator_in, const StorageIndex first, - const StorageIndex last) { + static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, + const StorageIndex lastIdx) { Evaluator evaluator = *evaluator_in; - eigen_assert(last >= first); - StorageIndex i = first; - if (last - first >= PacketSize) { - eigen_assert(first % PacketSize == 0); - StorageIndex last_chunk_offset = last - 4 * PacketSize; + eigen_assert(lastIdx >= firstIdx); + StorageIndex i = firstIdx; + if (lastIdx - firstIdx >= PacketSize) { + eigen_assert(firstIdx % PacketSize == 0); + StorageIndex last_chunk_offset = lastIdx - 4 * PacketSize; // Give compiler a strong possibility to unroll the loop. But don't insist // on unrolling, because if the function is expensive compiler should not // unroll the loop at the expense of inlining. @@ -198,12 +198,12 @@ struct EvalRange { evaluator.evalPacket(i + j * PacketSize); } } - last_chunk_offset = last - PacketSize; + last_chunk_offset = lastIdx - PacketSize; for (; i <= last_chunk_offset; i += PacketSize) { evaluator.evalPacket(i); } } - for (; i < last; ++i) { + for (; i < lastIdx; ++i) { evaluator.evalScalar(i); } } @@ -234,8 +234,8 @@ class TensorExecutor { const StorageIndex size = array_prod(evaluator.dimensions()); device.parallelFor(size, evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize, - [&evaluator](StorageIndex first, StorageIndex last) { - EvalRange::run(&evaluator, first, last); + [&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { + EvalRange::run(&evaluator, firstIdx, lastIdx); }); } evaluator.cleanup(); @@ -292,8 +292,8 @@ class TensorExecutor= -1 && thread_idx < num_threads); Scalar* thread_buf = reinterpret_cast( static_cast(buf) + aligned_blocksize * (thread_idx + 1)); - for (StorageIndex i = first; i < last; ++i) { + for (StorageIndex i = firstIdx; i < lastIdx; ++i) { auto block = block_mapper.GetBlockForIndex(i, thread_buf); evaluator.evalBlock(&block); } @@ -330,8 +330,8 @@ class TensorExecutor { template struct EigenMetaKernelEval { static __device__ EIGEN_ALWAYS_INLINE - void run(Evaluator& eval, StorageIndex first, StorageIndex last, StorageIndex step_size) { - for (StorageIndex i = first; i < last; i += step_size) { + void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx, StorageIndex step_size) { + for (StorageIndex i = firstIdx; i < lastIdx; i += step_size) { eval.evalScalar(i); } } @@ -340,17 +340,17 @@ struct EigenMetaKernelEval { template struct EigenMetaKernelEval { static __device__ EIGEN_ALWAYS_INLINE - void run(Evaluator& eval, StorageIndex first, StorageIndex last, StorageIndex step_size) { + void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx, StorageIndex step_size) { const StorageIndex PacketSize = unpacket_traits::size; - const StorageIndex vectorized_size = (last / PacketSize) * PacketSize; + const StorageIndex vectorized_size = (lastIdx / PacketSize) * PacketSize; const StorageIndex vectorized_step_size = step_size * PacketSize; // Use the vector path - for (StorageIndex i = first * PacketSize; i < vectorized_size; + for (StorageIndex i = firstIdx * PacketSize; i < vectorized_size; i += vectorized_step_size) { eval.evalPacket(i); } - for (StorageIndex i = vectorized_size + first; i < last; i += step_size) { + for (StorageIndex i = vectorized_size + firstIdx; i < lastIdx; i += step_size) { eval.evalScalar(i); } } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h index 59c1704ed..4837f2200 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h @@ -273,21 +273,21 @@ struct TensorEvaluator, Device const Index initialIndex = index; Index inputIndex = 0; for (int i = NumDims - 1; i > 0; --i) { - const Index first = index; - const Index last = index + PacketSize - 1; + const Index firstIdx = index; + const Index lastIdx = index + PacketSize - 1; const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i]; const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i]; const Index lastPaddedRight = m_outputStrides[i+1]; - if (!isLeftPaddingCompileTimeZero(i) && last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if (!isRightPaddingCompileTimeZero(i) && first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (first >= lastPaddedLeft && last < firstPaddedRight)) { + else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. const Index idx = index / m_outputStrides[i]; inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; @@ -299,21 +299,21 @@ struct TensorEvaluator, Device } } - const Index last = index + PacketSize - 1; - const Index first = index; + const Index lastIdx = index + PacketSize - 1; + const Index firstIdx = index; const Index lastPaddedLeft = m_padding[0].first; const Index firstPaddedRight = (m_dimensions[0] - m_padding[0].second); const Index lastPaddedRight = m_outputStrides[1]; - if (!isLeftPaddingCompileTimeZero(0) && last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(0) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if (!isRightPaddingCompileTimeZero(0) && first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(0) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if ((isLeftPaddingCompileTimeZero(0) && isRightPaddingCompileTimeZero(0)) || (first >= lastPaddedLeft && last < firstPaddedRight)) { + else if ((isLeftPaddingCompileTimeZero(0) && isRightPaddingCompileTimeZero(0)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. inputIndex += (index - m_padding[0].first); return m_impl.template packet(inputIndex); @@ -331,21 +331,21 @@ struct TensorEvaluator, Device Index inputIndex = 0; for (int i = 0; i < NumDims - 1; ++i) { - const Index first = index; - const Index last = index + PacketSize - 1; + const Index firstIdx = index; + const Index lastIdx = index + PacketSize - 1; const Index lastPaddedLeft = m_padding[i].first * m_outputStrides[i+1]; const Index firstPaddedRight = (m_dimensions[i] - m_padding[i].second) * m_outputStrides[i+1]; const Index lastPaddedRight = m_outputStrides[i]; - if (!isLeftPaddingCompileTimeZero(i) && last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(i) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if (!isRightPaddingCompileTimeZero(i) && first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(i) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (first >= lastPaddedLeft && last < firstPaddedRight)) { + else if ((isLeftPaddingCompileTimeZero(i) && isRightPaddingCompileTimeZero(i)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. const Index idx = index / m_outputStrides[i+1]; inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; @@ -357,21 +357,21 @@ struct TensorEvaluator, Device } } - const Index last = index + PacketSize - 1; - const Index first = index; + const Index lastIdx = index + PacketSize - 1; + const Index firstIdx = index; const Index lastPaddedLeft = m_padding[NumDims-1].first; const Index firstPaddedRight = (m_dimensions[NumDims-1] - m_padding[NumDims-1].second); const Index lastPaddedRight = m_outputStrides[NumDims-1]; - if (!isLeftPaddingCompileTimeZero(NumDims-1) && last < lastPaddedLeft) { + if (!isLeftPaddingCompileTimeZero(NumDims-1) && lastIdx < lastPaddedLeft) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if (!isRightPaddingCompileTimeZero(NumDims-1) && first >= firstPaddedRight && last < lastPaddedRight) { + else if (!isRightPaddingCompileTimeZero(NumDims-1) && firstIdx >= firstPaddedRight && lastIdx < lastPaddedRight) { // all the coefficient are in the padding zone. return internal::pset1(m_paddingValue); } - else if ((isLeftPaddingCompileTimeZero(NumDims-1) && isRightPaddingCompileTimeZero(NumDims-1)) || (first >= lastPaddedLeft && last < firstPaddedRight)) { + else if ((isLeftPaddingCompileTimeZero(NumDims-1) && isRightPaddingCompileTimeZero(NumDims-1)) || (firstIdx >= lastPaddedLeft && lastIdx < firstPaddedRight)) { // all the coefficient are between the 2 padding zones. inputIndex += (index - m_padding[NumDims-1].first); return m_impl.template packet(inputIndex); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h index 7504c1598..88940e6e6 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h @@ -208,8 +208,8 @@ __global__ void ReductionInitFullReduxKernelHalfFloat(Reducer reducer, const Sel eigen_assert(blockDim.x == 1); eigen_assert(gridDim.x == 1); if (num_coeffs % 2 != 0) { - half last = input.m_impl.coeff(num_coeffs-1); - *scratch = __halves2half2(last, reducer.initialize()); + half lastCoeff = input.m_impl.coeff(num_coeffs-1); + *scratch = __halves2half2(lastCoeff, reducer.initialize()); } else { *scratch = reducer.template initializePacket(); } diff --git a/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h b/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h index 22c952ae1..7a71f89fd 100644 --- a/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h +++ b/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h @@ -128,7 +128,7 @@ class EventCount { // Notify wakes one or all waiting threads. // Must be called after changing the associated wait predicate. - void Notify(bool all) { + void Notify(bool notifyAll) { std::atomic_thread_fence(std::memory_order_seq_cst); uint64_t state = state_.load(std::memory_order_acquire); for (;;) { @@ -137,7 +137,7 @@ class EventCount { return; uint64_t waiters = (state & kWaiterMask) >> kWaiterShift; uint64_t newstate; - if (all) { + if (notifyAll) { // Reset prewait counter and empty wait list. newstate = (state & kEpochMask) + (kEpochInc * waiters) + kStackMask; } else if (waiters) { @@ -157,10 +157,10 @@ class EventCount { } if (state_.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { - if (!all && waiters) return; // unblocked pre-wait thread + if (!notifyAll && waiters) return; // unblocked pre-wait thread if ((state & kStackMask) == kStackMask) return; Waiter* w = &waiters_[state & kStackMask]; - if (!all) w->next.store(nullptr, std::memory_order_relaxed); + if (!notifyAll) w->next.store(nullptr, std::memory_order_relaxed); Unpark(w); return; } From bac36d09963591aa66d8cfd9b39c375cf07efebd Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 21 Sep 2018 23:03:45 +0200 Subject: [PATCH 8/9] Demangle Travseral and Unrolling in Redux --- Eigen/src/Core/Redux.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 25e374c77..0aee855df 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h @@ -81,10 +81,9 @@ public: EIGEN_DEBUG_VAR(MightVectorize) EIGEN_DEBUG_VAR(MayLinearVectorize) EIGEN_DEBUG_VAR(MaySliceVectorize) - EIGEN_DEBUG_VAR(Traversal) + std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl; EIGEN_DEBUG_VAR(UnrollingLimit) - - EIGEN_DEBUG_VAR(Unrolling) + std::cerr << "Unrolling" << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl; std::cerr << std::endl; } #endif @@ -411,7 +410,7 @@ DenseBase::redux(const Func& func) const typedef typename internal::redux_evaluator ThisEvaluator; ThisEvaluator thisEval(derived()); - + // The initial expression is passed to the reducer as an additional argument instead of // passing it as a member of redux_evaluator to help return internal::redux_impl::run(thisEval, func, derived()); From 795e12393b862b44721b5c67eabbbe920a7ce28e Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Sat, 22 Sep 2018 16:44:33 +0200 Subject: [PATCH 9/9] Fix logic in diagonal*dense product in a corner case. The problem was for: diag(1x1) * mat(1,n) --- Eigen/src/Core/ProductEvaluators.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index 2787987e7..0762d9e8b 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h @@ -803,13 +803,21 @@ public: MatrixFlags = evaluator::Flags, DiagFlags = evaluator::Flags, - _StorageOrder = MatrixFlags & RowMajorBit ? RowMajor : ColMajor, + + _StorageOrder = (Derived::MaxRowsAtCompileTime==1 && Derived::MaxColsAtCompileTime!=1) ? RowMajor + : (Derived::MaxColsAtCompileTime==1 && Derived::MaxRowsAtCompileTime!=1) ? ColMajor + : MatrixFlags & RowMajorBit ? RowMajor : ColMajor, + _SameStorageOrder = _StorageOrder == (MatrixFlags & RowMajorBit ? RowMajor : ColMajor), + _ScalarAccessOnDiag = !((int(_StorageOrder) == ColMajor && int(ProductOrder) == OnTheLeft) ||(int(_StorageOrder) == RowMajor && int(ProductOrder) == OnTheRight)), _SameTypes = is_same::value, // FIXME currently we need same types, but in the future the next rule should be the one //_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagFlags)&PacketAccessBit))), - _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), + _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) + && _SameTypes + && (_SameStorageOrder || (MatrixFlags&LinearAccessBit)==LinearAccessBit) + && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0, Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0), Alignment = evaluator::Alignment, @@ -870,10 +878,10 @@ struct product_evaluator, ProductTag, DiagonalSha typedef Product XprType; typedef typename XprType::PlainObject PlainObject; + typedef typename Lhs::DiagonalVectorType DiagonalType; + - enum { - StorageOrder = int(Rhs::Flags) & RowMajorBit ? RowMajor : ColMajor - }; + enum { StorageOrder = Base::_StorageOrder }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.rhs(), xpr.lhs().diagonal()) @@ -917,7 +925,7 @@ struct product_evaluator, ProductTag, DenseShape, typedef Product XprType; typedef typename XprType::PlainObject PlainObject; - enum { StorageOrder = int(Lhs::Flags) & RowMajorBit ? RowMajor : ColMajor }; + enum { StorageOrder = Base::_StorageOrder }; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) : Base(xpr.lhs(), xpr.rhs().diagonal())