diff --git a/Eigen/Core b/Eigen/Core index 0cf101636..0189654a6 100644 --- a/Eigen/Core +++ b/Eigen/Core @@ -355,6 +355,12 @@ using std::ptrdiff_t; #include "src/Core/ArrayBase.h" #include "src/Core/ArrayWrapper.h" +#ifdef EIGEN_ENABLE_EVALUATORS +#include "src/Core/Product.h" +#include "src/Core/CoreEvaluators.h" +#include "src/Core/AssignEvaluator.h" +#endif + #ifdef EIGEN_USE_BLAS #include "src/Core/products/GeneralMatrixMatrix_MKL.h" #include "src/Core/products/GeneralMatrixVector_MKL.h" diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h new file mode 100644 index 000000000..006a87d47 --- /dev/null +++ b/Eigen/src/Core/AssignEvaluator.h @@ -0,0 +1,682 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2011 Benoit Jacob +// Copyright (C) 2011 Gael Guennebaud +// Copyright (C) 2011 Jitse Niesen +// +// Eigen is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// Alternatively, you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation; either version 2 of +// the License, or (at your option) any later version. +// +// Eigen is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +// FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License and a copy of the GNU General Public License along with +// Eigen. If not, see . + +#ifndef EIGEN_ASSIGN_EVALUATOR_H +#define EIGEN_ASSIGN_EVALUATOR_H + +// This implementation is based on Assign.h + +namespace internal { + +/*************************************************************************** +* Part 1 : the logic deciding a strategy for traversal and unrolling * +***************************************************************************/ + +// copy_using_evaluator_traits is based on assign_traits +// (actually, it's identical) + +template +struct copy_using_evaluator_traits +{ +public: + enum { + DstIsAligned = Derived::Flags & AlignedBit, + DstHasDirectAccess = Derived::Flags & DirectAccessBit, + SrcIsAligned = OtherDerived::Flags & AlignedBit, + JointAlignment = bool(DstIsAligned) && bool(SrcIsAligned) ? Aligned : Unaligned + }; + +private: + enum { + InnerSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::SizeAtCompileTime) + : int(Derived::Flags)&RowMajorBit ? int(Derived::ColsAtCompileTime) + : int(Derived::RowsAtCompileTime), + InnerMaxSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::MaxSizeAtCompileTime) + : int(Derived::Flags)&RowMajorBit ? int(Derived::MaxColsAtCompileTime) + : int(Derived::MaxRowsAtCompileTime), + MaxSizeAtCompileTime = Derived::SizeAtCompileTime, + PacketSize = packet_traits::size + }; + + enum { + StorageOrdersAgree = (int(Derived::IsRowMajor) == int(OtherDerived::IsRowMajor)), + MightVectorize = StorageOrdersAgree + && (int(Derived::Flags) & int(OtherDerived::Flags) & ActualPacketAccessBit), + MayInnerVectorize = MightVectorize && int(InnerSize)!=Dynamic && int(InnerSize)%int(PacketSize)==0 + && int(DstIsAligned) && int(SrcIsAligned), + MayLinearize = StorageOrdersAgree && (int(Derived::Flags) & int(OtherDerived::Flags) & LinearAccessBit), + MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess + && (DstIsAligned || MaxSizeAtCompileTime == Dynamic), + /* If the destination isn't aligned, we have to do runtime checks and we don't unroll, + so it's only good for large enough sizes. */ + MaySliceVectorize = MightVectorize && DstHasDirectAccess + && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=3*PacketSize) + /* slice vectorization can be slow, so we only want it if the slices are big, which is + indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block + in a fixed-size matrix */ + }; + +public: + enum { + Traversal = int(MayInnerVectorize) ? int(InnerVectorizedTraversal) + : int(MayLinearVectorize) ? int(LinearVectorizedTraversal) + : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) + : int(MayLinearize) ? int(LinearTraversal) + : int(DefaultTraversal), + Vectorized = int(Traversal) == InnerVectorizedTraversal + || int(Traversal) == LinearVectorizedTraversal + || int(Traversal) == SliceVectorizedTraversal + }; + +private: + enum { + UnrollingLimit = EIGEN_UNROLLING_LIMIT * (Vectorized ? int(PacketSize) : 1), + MayUnrollCompletely = int(Derived::SizeAtCompileTime) != Dynamic + && int(OtherDerived::CoeffReadCost) != Dynamic + && int(Derived::SizeAtCompileTime) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit), + MayUnrollInner = int(InnerSize) != Dynamic + && int(OtherDerived::CoeffReadCost) != Dynamic + && int(InnerSize) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit) + }; + +public: + enum { + Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal)) + ? ( + int(MayUnrollCompletely) ? int(CompleteUnrolling) + : int(MayUnrollInner) ? int(InnerUnrolling) + : int(NoUnrolling) + ) + : int(Traversal) == int(LinearVectorizedTraversal) + ? ( bool(MayUnrollCompletely) && bool(DstIsAligned) ? int(CompleteUnrolling) + : int(NoUnrolling) ) + : int(Traversal) == int(LinearTraversal) + ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling) + : int(NoUnrolling) ) + : int(NoUnrolling) + }; + +#ifdef EIGEN_DEBUG_ASSIGN + static void debug() + { + EIGEN_DEBUG_VAR(DstIsAligned) + EIGEN_DEBUG_VAR(SrcIsAligned) + EIGEN_DEBUG_VAR(JointAlignment) + EIGEN_DEBUG_VAR(InnerSize) + EIGEN_DEBUG_VAR(InnerMaxSize) + EIGEN_DEBUG_VAR(PacketSize) + EIGEN_DEBUG_VAR(StorageOrdersAgree) + EIGEN_DEBUG_VAR(MightVectorize) + EIGEN_DEBUG_VAR(MayLinearize) + EIGEN_DEBUG_VAR(MayInnerVectorize) + EIGEN_DEBUG_VAR(MayLinearVectorize) + EIGEN_DEBUG_VAR(MaySliceVectorize) + EIGEN_DEBUG_VAR(Traversal) + EIGEN_DEBUG_VAR(UnrollingLimit) + EIGEN_DEBUG_VAR(MayUnrollCompletely) + EIGEN_DEBUG_VAR(MayUnrollInner) + EIGEN_DEBUG_VAR(Unrolling) + } +#endif +}; + +/*************************************************************************** +* Part 2 : meta-unrollers +***************************************************************************/ + +/************************ +*** Default traversal *** +************************/ + +template +struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling +{ + typedef typename DstEvaluatorType::XprType DstXprType; + + enum { + outer = Index / DstXprType::InnerSizeAtCompileTime, + inner = Index % DstXprType::InnerSizeAtCompileTime + }; + + EIGEN_STRONG_INLINE static void run(DstEvaluatorType &dstEvaluator, + SrcEvaluatorType &srcEvaluator) + { + dstEvaluator.copyCoeffByOuterInner(outer, inner, srcEvaluator); + copy_using_evaluator_DefaultTraversal_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType&, SrcEvaluatorType&) { } +}; + +template +struct copy_using_evaluator_DefaultTraversal_InnerUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType &dstEvaluator, + SrcEvaluatorType &srcEvaluator, + int outer) + { + dstEvaluator.copyCoeffByOuterInner(outer, Index, srcEvaluator); + copy_using_evaluator_DefaultTraversal_InnerUnrolling + + ::run(dstEvaluator, srcEvaluator, outer); + } +}; + +template +struct copy_using_evaluator_DefaultTraversal_InnerUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType&, SrcEvaluatorType&, int) { } +}; + +/*********************** +*** Linear traversal *** +***********************/ + +template +struct copy_using_evaluator_LinearTraversal_CompleteUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType &dstEvaluator, + SrcEvaluatorType &srcEvaluator) + { + dstEvaluator.copyCoeff(Index, srcEvaluator); + copy_using_evaluator_LinearTraversal_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_LinearTraversal_CompleteUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType&, SrcEvaluatorType&) { } +}; + +/************************** +*** Inner vectorization *** +**************************/ + +template +struct copy_using_evaluator_innervec_CompleteUnrolling +{ + typedef typename DstEvaluatorType::XprType DstXprType; + typedef typename SrcEvaluatorType::XprType SrcXprType; + + enum { + outer = Index / DstXprType::InnerSizeAtCompileTime, + inner = Index % DstXprType::InnerSizeAtCompileTime, + JointAlignment = copy_using_evaluator_traits::JointAlignment + }; + + EIGEN_STRONG_INLINE static void run(DstEvaluatorType &dstEvaluator, + SrcEvaluatorType &srcEvaluator) + { + dstEvaluator.template copyPacketByOuterInner(outer, inner, srcEvaluator); + enum { NextIndex = Index + packet_traits::size }; + copy_using_evaluator_innervec_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_innervec_CompleteUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType&, SrcEvaluatorType&) { } +}; + +template +struct copy_using_evaluator_innervec_InnerUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType &dstEvaluator, + SrcEvaluatorType &srcEvaluator, + int outer) + { + dstEvaluator.template copyPacketByOuterInner(outer, Index, srcEvaluator); + typedef typename DstEvaluatorType::XprType DstXprType; + enum { NextIndex = Index + packet_traits::size }; + copy_using_evaluator_innervec_InnerUnrolling + + ::run(dstEvaluator, srcEvaluator, outer); + } +}; + +template +struct copy_using_evaluator_innervec_InnerUnrolling +{ + EIGEN_STRONG_INLINE static void run(DstEvaluatorType&, SrcEvaluatorType&, int) { } +}; + +/*************************************************************************** +* Part 3 : implementation of all cases +***************************************************************************/ + +// copy_using_evaluator_impl is based on assign_impl + +template::Traversal, + int Unrolling = copy_using_evaluator_traits::Unrolling> +struct copy_using_evaluator_impl; + +/************************ +*** Default traversal *** +************************/ + +template +struct copy_using_evaluator_impl +{ + static void run(DstXprType& dst, const SrcXprType& src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + typedef typename DstXprType::Index Index; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + for(Index outer = 0; outer < dst.outerSize(); ++outer) { + for(Index inner = 0; inner < dst.innerSize(); ++inner) { + dstEvaluator.copyCoeffByOuterInner(outer, inner, srcEvaluator); + } + } + } +}; + +template +struct copy_using_evaluator_impl +{ + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + copy_using_evaluator_DefaultTraversal_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_impl +{ + typedef typename DstXprType::Index Index; + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + const Index outerSize = dst.outerSize(); + for(Index outer = 0; outer < outerSize; ++outer) + copy_using_evaluator_DefaultTraversal_InnerUnrolling + + ::run(dstEvaluator, srcEvaluator, outer); + } +}; + +/*************************** +*** Linear vectorization *** +***************************/ + +template +struct unaligned_copy_using_evaluator_impl +{ + // if IsAligned = true, then do nothing + template + static EIGEN_STRONG_INLINE void run(const SrcEvaluatorType&, DstEvaluatorType&, + typename SrcEvaluatorType::Index, typename SrcEvaluatorType::Index) {} +}; + +template <> +struct unaligned_copy_using_evaluator_impl +{ + // MSVC must not inline this functions. If it does, it fails to optimize the + // packet access path. +#ifdef _MSC_VER + template + static EIGEN_DONT_INLINE void run(DstEvaluatorType &dstEvaluator, + const SrcEvaluatorType &srcEvaluator, + typename DstEvaluatorType::Index start, + typename DstEvaluatorType::Index end) +#else + template + static EIGEN_STRONG_INLINE void run(DstEvaluatorType &dstEvaluator, + const SrcEvaluatorType &srcEvaluator, + typename DstEvaluatorType::Index start, + typename DstEvaluatorType::Index end) +#endif + { + for (typename DstEvaluatorType::Index index = start; index < end; ++index) + dstEvaluator.copyCoeff(index, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_impl +{ + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + typedef typename DstXprType::Index Index; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + const Index size = dst.size(); + typedef packet_traits PacketTraits; + enum { + packetSize = PacketTraits::size, + dstIsAligned = int(copy_using_evaluator_traits::DstIsAligned), + dstAlignment = PacketTraits::AlignedOnScalar ? Aligned : dstIsAligned, + srcAlignment = copy_using_evaluator_traits::JointAlignment + }; + const Index alignedStart = dstIsAligned ? 0 : first_aligned(&dstEvaluator.coeffRef(0), size); + const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize; + + unaligned_copy_using_evaluator_impl::run(dstEvaluator, srcEvaluator, 0, alignedStart); + + for(Index index = alignedStart; index < alignedEnd; index += packetSize) + { + dstEvaluator.template copyPacket(index, srcEvaluator); + } + + unaligned_copy_using_evaluator_impl<>::run(dstEvaluator, srcEvaluator, alignedEnd, size); + } +}; + +template +struct copy_using_evaluator_impl +{ + typedef typename DstXprType::Index Index; + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + enum { size = DstXprType::SizeAtCompileTime, + packetSize = packet_traits::size, + alignedSize = (size/packetSize)*packetSize }; + + copy_using_evaluator_innervec_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + copy_using_evaluator_DefaultTraversal_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +/************************** +*** Inner vectorization *** +**************************/ + +template +struct copy_using_evaluator_impl +{ + inline static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + typedef typename DstXprType::Index Index; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + const Index innerSize = dst.innerSize(); + const Index outerSize = dst.outerSize(); + const Index packetSize = packet_traits::size; + for(Index outer = 0; outer < outerSize; ++outer) + for(Index inner = 0; inner < innerSize; inner+=packetSize) { + dstEvaluator.template copyPacketByOuterInner(outer, inner, srcEvaluator); + } + } +}; + +template +struct copy_using_evaluator_impl +{ + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + copy_using_evaluator_innervec_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_impl +{ + typedef typename DstXprType::Index Index; + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + const Index outerSize = dst.outerSize(); + for(Index outer = 0; outer < outerSize; ++outer) + copy_using_evaluator_innervec_InnerUnrolling + + ::run(dstEvaluator, srcEvaluator, outer); + } +}; + +/*********************** +*** Linear traversal *** +***********************/ + +template +struct copy_using_evaluator_impl +{ + inline static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + typedef typename DstXprType::Index Index; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + const Index size = dst.size(); + for(Index i = 0; i < size; ++i) + dstEvaluator.copyCoeff(i, srcEvaluator); + } +}; + +template +struct copy_using_evaluator_impl +{ + EIGEN_STRONG_INLINE static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + copy_using_evaluator_LinearTraversal_CompleteUnrolling + + ::run(dstEvaluator, srcEvaluator); + } +}; + +/************************** +*** Slice vectorization *** +***************************/ + +template +struct copy_using_evaluator_impl +{ + inline static void run(DstXprType &dst, const SrcXprType &src) + { + typedef typename evaluator::type DstEvaluatorType; + typedef typename evaluator::type SrcEvaluatorType; + typedef typename DstXprType::Index Index; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + typedef packet_traits PacketTraits; + enum { + packetSize = PacketTraits::size, + alignable = PacketTraits::AlignedOnScalar, + dstAlignment = alignable ? Aligned : int(copy_using_evaluator_traits::DstIsAligned) , + srcAlignment = copy_using_evaluator_traits::JointAlignment + }; + const Index packetAlignedMask = packetSize - 1; + const Index innerSize = dst.innerSize(); + const Index outerSize = dst.outerSize(); + const Index alignedStep = alignable ? (packetSize - dst.outerStride() % packetSize) & packetAlignedMask : 0; + Index alignedStart = ((!alignable) || copy_using_evaluator_traits::DstIsAligned) ? 0 + : first_aligned(&dstEvaluator.coeffRef(0,0), innerSize); + + for(Index outer = 0; outer < outerSize; ++outer) + { + const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask); + // do the non-vectorizable part of the assignment + for(Index inner = 0; inner(outer, inner, srcEvaluator); + } + + // do the non-vectorizable part of the assignment + for(Index inner = alignedEnd; inner((alignedStart+alignedStep)%packetSize, innerSize); + } + } +}; + +/*************************************************************************** +* Part 4 : Entry points +***************************************************************************/ + +// Based on DenseBase::LazyAssign() + +template +const DstXprType& copy_using_evaluator(const DstXprType& dst, const SrcXprType& src) +{ +#ifdef EIGEN_DEBUG_ASSIGN + internal::copy_using_evaluator_traits::debug(); +#endif + copy_using_evaluator_impl::run(const_cast(dst), src); + return dst; +} + +// Based on DenseBase::swap() +// TODO: Chech whether we need to do something special for swapping two +// Arrays or Matrices. + +template +void swap_using_evaluator(const DstXprType& dst, const SrcXprType& src) +{ + copy_using_evaluator(SwapWrapper(const_cast(dst)), src); +} + +// Based on MatrixBase::operator+= (in CwiseBinaryOp.h) +template +void add_assign_using_evaluator(const MatrixBase& dst, const MatrixBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + +// Based on ArrayBase::operator+= +template +void add_assign_using_evaluator(const ArrayBase& dst, const ArrayBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + +// TODO: Add add_assign_using_evaluator for EigenBase ? + +template +void subtract_assign_using_evaluator(const MatrixBase& dst, const MatrixBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + +template +void subtract_assign_using_evaluator(const ArrayBase& dst, const ArrayBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + +template +void multiply_assign_using_evaluator(const ArrayBase& dst, const ArrayBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + +template +void divide_assign_using_evaluator(const ArrayBase& dst, const ArrayBase& src) +{ + typedef typename DstXprType::Scalar Scalar; + SelfCwiseBinaryOp, DstXprType, SrcXprType> tmp(dst.const_cast_derived()); + copy_using_evaluator(tmp, src.derived()); +} + + +} // namespace internal + +#endif // EIGEN_ASSIGN_EVALUATOR_H diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h new file mode 100644 index 000000000..c060913fb --- /dev/null +++ b/Eigen/src/Core/CoreEvaluators.h @@ -0,0 +1,1157 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2011 Benoit Jacob +// Copyright (C) 2011 Gael Guennebaud +// Copyright (C) 2011 Jitse Niesen +// +// Eigen is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 3 of the License, or (at your option) any later version. +// +// Alternatively, you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation; either version 2 of +// the License, or (at your option) any later version. +// +// Eigen is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +// FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License and a copy of the GNU General Public License along with +// Eigen. If not, see . + + +#ifndef EIGEN_COREEVALUATORS_H +#define EIGEN_COREEVALUATORS_H + +namespace internal { + +template +struct evaluator_impl {}; + +template +struct evaluator +{ + typedef evaluator_impl type; +}; + +// TODO: Think about const-correctness + +template +struct evaluator +{ + typedef evaluator_impl type; +}; + +// ---------- base class for all writable evaluators ---------- + +template +struct evaluator_impl_base +{ + typedef typename ExpressionType::Index Index; + + template + void copyCoeff(Index row, Index col, const OtherEvaluatorType& other) + { + derived().coeffRef(row, col) = other.coeff(row, col); + } + + template + void copyCoeffByOuterInner(Index outer, Index inner, const OtherEvaluatorType& other) + { + Index row = rowIndexByOuterInner(outer, inner); + Index col = colIndexByOuterInner(outer, inner); + derived().copyCoeff(row, col, other); + } + + template + void copyCoeff(Index index, const OtherEvaluatorType& other) + { + derived().coeffRef(index) = other.coeff(index); + } + + template + void copyPacket(Index row, Index col, const OtherEvaluatorType& other) + { + derived().template writePacket(row, col, + other.template packet(row, col)); + } + + template + void copyPacketByOuterInner(Index outer, Index inner, const OtherEvaluatorType& other) + { + Index row = rowIndexByOuterInner(outer, inner); + Index col = colIndexByOuterInner(outer, inner); + derived().template copyPacket(row, col, other); + } + + template + void copyPacket(Index index, const OtherEvaluatorType& other) + { + derived().template writePacket(index, + other.template packet(index)); + } + + Index rowIndexByOuterInner(Index outer, Index inner) const + { + return int(ExpressionType::RowsAtCompileTime) == 1 ? 0 + : int(ExpressionType::ColsAtCompileTime) == 1 ? inner + : int(ExpressionType::Flags)&RowMajorBit ? outer + : inner; + } + + Index colIndexByOuterInner(Index outer, Index inner) const + { + return int(ExpressionType::ColsAtCompileTime) == 1 ? 0 + : int(ExpressionType::RowsAtCompileTime) == 1 ? inner + : int(ExpressionType::Flags)&RowMajorBit ? inner + : outer; + } + + evaluator_impl& derived() + { + return *static_cast*>(this); + } +}; + +// -------------------- Transpose -------------------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef Transpose XprType; + + evaluator_impl(const XprType& t) : m_argImpl(t.nestedExpression()) {} + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_argImpl.coeff(col, row); + } + + CoeffReturnType coeff(Index index) const + { + return m_argImpl.coeff(index); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(col, row); + } + + typename XprType::Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(index); + } + + template + PacketReturnType packet(Index row, Index col) const + { + return m_argImpl.template packet(col, row); + } + + template + PacketReturnType packet(Index index) const + { + return m_argImpl.template packet(index); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + m_argImpl.template writePacket(col, row, x); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + m_argImpl.template writePacket(index, x); + } + +protected: + typename evaluator::type m_argImpl; +}; + +// -------------------- Matrix and Array -------------------- +// +// evaluator_impl is a common base class for the +// Matrix and Array evaluators. + +template +struct evaluator_impl > + : evaluator_impl_base +{ + typedef PlainObjectBase PlainObjectType; + + evaluator_impl(const PlainObjectType& m) : m_plainObject(m) {} + + typedef typename PlainObjectType::Index Index; + typedef typename PlainObjectType::Scalar Scalar; + typedef typename PlainObjectType::CoeffReturnType CoeffReturnType; + typedef typename PlainObjectType::PacketScalar PacketScalar; + typedef typename PlainObjectType::PacketReturnType PacketReturnType; + + CoeffReturnType coeff(Index i, Index j) const + { + return m_plainObject.coeff(i, j); + } + + CoeffReturnType coeff(Index index) const + { + return m_plainObject.coeff(index); + } + + Scalar& coeffRef(Index i, Index j) + { + return m_plainObject.const_cast_derived().coeffRef(i, j); + } + + Scalar& coeffRef(Index index) + { + return m_plainObject.const_cast_derived().coeffRef(index); + } + + template + PacketReturnType packet(Index row, Index col) const + { + return m_plainObject.template packet(row, col); + } + + template + PacketReturnType packet(Index index) const + { + return m_plainObject.template packet(index); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + m_plainObject.const_cast_derived().template writePacket(row, col, x); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + m_plainObject.const_cast_derived().template writePacket(index, x); + } + +protected: + const PlainObjectType &m_plainObject; +}; + +template +struct evaluator_impl > + : evaluator_impl > > +{ + typedef Matrix XprType; + + evaluator_impl(const XprType& m) + : evaluator_impl >(m) + { } +}; + +template +struct evaluator_impl > + : evaluator_impl > > +{ + typedef Array XprType; + + evaluator_impl(const XprType& m) + : evaluator_impl >(m) + { } +}; + +// -------------------- CwiseNullaryOp -------------------- + +template +struct evaluator_impl > +{ + typedef CwiseNullaryOp XprType; + + evaluator_impl(const XprType& n) + : m_functor(n.functor()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_functor(row, col); + } + + CoeffReturnType coeff(Index index) const + { + return m_functor(index); + } + + template + PacketScalar packet(Index row, Index col) const + { + return m_functor.packetOp(row, col); + } + + template + PacketScalar packet(Index index) const + { + return m_functor.packetOp(index); + } + +protected: + const NullaryOp m_functor; +}; + +// -------------------- CwiseUnaryOp -------------------- + +template +struct evaluator_impl > +{ + typedef CwiseUnaryOp XprType; + + evaluator_impl(const XprType& op) + : m_functor(op.functor()), + m_argImpl(op.nestedExpression()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_functor(m_argImpl.coeff(row, col)); + } + + CoeffReturnType coeff(Index index) const + { + return m_functor(m_argImpl.coeff(index)); + } + + template + PacketScalar packet(Index row, Index col) const + { + return m_functor.packetOp(m_argImpl.template packet(row, col)); + } + + template + PacketScalar packet(Index index) const + { + return m_functor.packetOp(m_argImpl.template packet(index)); + } + +protected: + const UnaryOp m_functor; + typename evaluator::type m_argImpl; +}; + +// -------------------- CwiseBinaryOp -------------------- + +template +struct evaluator_impl > +{ + typedef CwiseBinaryOp XprType; + + evaluator_impl(const XprType& xpr) + : m_functor(xpr.functor()), + m_lhsImpl(xpr.lhs()), + m_rhsImpl(xpr.rhs()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_functor(m_lhsImpl.coeff(row, col), m_rhsImpl.coeff(row, col)); + } + + CoeffReturnType coeff(Index index) const + { + return m_functor(m_lhsImpl.coeff(index), m_rhsImpl.coeff(index)); + } + + template + PacketScalar packet(Index row, Index col) const + { + return m_functor.packetOp(m_lhsImpl.template packet(row, col), + m_rhsImpl.template packet(row, col)); + } + + template + PacketScalar packet(Index index) const + { + return m_functor.packetOp(m_lhsImpl.template packet(index), + m_rhsImpl.template packet(index)); + } + +protected: + const BinaryOp m_functor; + typename evaluator::type m_lhsImpl; + typename evaluator::type m_rhsImpl; +}; + +// -------------------- CwiseUnaryView -------------------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef CwiseUnaryView XprType; + + evaluator_impl(const XprType& op) + : m_unaryOp(op.functor()), + m_argImpl(op.nestedExpression()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_unaryOp(m_argImpl.coeff(row, col)); + } + + CoeffReturnType coeff(Index index) const + { + return m_unaryOp(m_argImpl.coeff(index)); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_unaryOp(m_argImpl.coeffRef(row, col)); + } + + Scalar& coeffRef(Index index) + { + return m_unaryOp(m_argImpl.coeffRef(index)); + } + +protected: + const UnaryOp m_unaryOp; + typename evaluator::type m_argImpl; +}; + +// -------------------- Product -------------------- + +template +struct evaluator_impl > : public evaluator::PlainObject>::type +{ + typedef Product XprType; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator::type evaluator_base; + +// enum { +// EvaluateLhs = ; +// EvaluateRhs = ; +// }; + + evaluator_impl(const XprType& product) : evaluator_base(m_result) + { + // here we process the left and right hand sides with a specialized evaluator + // perhaps this step should be done by the TreeOptimizer to get a canonical tree and reduce evaluator instanciations + // typename product_operand_evaluator::type m_lhsImpl(product.lhs()); + // typename product_operand_evaluator::type m_rhsImpl(product.rhs()); + + // TODO do not rely on previous product mechanism !! + m_result.resize(product.rows(), product.cols()); + m_result.noalias() = product.lhs() * product.rhs(); + } + +protected: + PlainObject m_result; +}; + +// -------------------- Map -------------------- + +template +struct evaluator_impl > + : evaluator_impl_base +{ + typedef MapBase MapType; + typedef Derived XprType; + + typedef typename XprType::PointerType PointerType; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + evaluator_impl(const XprType& map) + : m_data(const_cast(map.data())), + m_rowStride(map.rowStride()), + m_colStride(map.colStride()) + { } + + enum { + RowsAtCompileTime = XprType::RowsAtCompileTime + }; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_data[col * m_colStride + row * m_rowStride]; + } + + CoeffReturnType coeff(Index index) const + { + return coeff(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_data[col * m_colStride + row * m_rowStride]; + } + + Scalar& coeffRef(Index index) + { + return coeffRef(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + template + PacketReturnType packet(Index row, Index col) const + { + PointerType ptr = m_data + row * m_rowStride + col * m_colStride; + return internal::ploadt(ptr); + } + + template + PacketReturnType packet(Index index) const + { + return packet(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + PointerType ptr = m_data + row * m_rowStride + col * m_colStride; + return internal::pstoret(ptr, x); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + return writePacket(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0, + x); + } + +protected: + PointerType m_data; + int m_rowStride; + int m_colStride; +}; + +template +struct evaluator_impl > + : public evaluator_impl > > +{ + typedef Map XprType; + + evaluator_impl(const XprType& map) + : evaluator_impl >(map) + { } +}; + +// -------------------- Block -------------------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef Block XprType; + + evaluator_impl(const XprType& block) + : m_argImpl(block.nestedExpression()), + m_startRow(block.startRow()), + m_startCol(block.startCol()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + enum { + RowsAtCompileTime = XprType::RowsAtCompileTime + }; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_argImpl.coeff(m_startRow + row, m_startCol + col); + } + + CoeffReturnType coeff(Index index) const + { + return coeff(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(m_startRow + row, m_startCol + col); + } + + Scalar& coeffRef(Index index) + { + return coeffRef(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + template + PacketReturnType packet(Index row, Index col) const + { + return m_argImpl.template packet(m_startRow + row, m_startCol + col); + } + + template + PacketReturnType packet(Index index) const + { + return packet(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + return m_argImpl.template writePacket(m_startRow + row, m_startCol + col, x); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + return writePacket(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0, + x); + } + +protected: + typename evaluator::type m_argImpl; + + // TODO: Get rid of m_startRow, m_startCol if known at compile time + Index m_startRow; + Index m_startCol; +}; + +// TODO: This evaluator does not actually use the child evaluator; +// all action is via the data() as returned by the Block expression. + +template +struct evaluator_impl > + : evaluator_impl > > +{ + typedef Block XprType; + + evaluator_impl(const XprType& block) + : evaluator_impl >(block) + { } +}; + + +// -------------------- Select -------------------- + +template +struct evaluator_impl > +{ + typedef Select XprType; + + evaluator_impl(const XprType& select) + : m_conditionImpl(select.conditionMatrix()), + m_thenImpl(select.thenMatrix()), + m_elseImpl(select.elseMatrix()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + if (m_conditionImpl.coeff(row, col)) + return m_thenImpl.coeff(row, col); + else + return m_elseImpl.coeff(row, col); + } + + CoeffReturnType coeff(Index index) const + { + if (m_conditionImpl.coeff(index)) + return m_thenImpl.coeff(index); + else + return m_elseImpl.coeff(index); + } + +protected: + typename evaluator::type m_conditionImpl; + typename evaluator::type m_thenImpl; + typename evaluator::type m_elseImpl; +}; + + +// -------------------- Replicate -------------------- + +template +struct evaluator_impl > +{ + typedef Replicate XprType; + + evaluator_impl(const XprType& replicate) + : m_argImpl(replicate.nestedExpression()), + m_rows(replicate.nestedExpression().rows()), + m_cols(replicate.nestedExpression().cols()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketReturnType PacketReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + // try to avoid using modulo; this is a pure optimization strategy + const Index actual_row = internal::traits::RowsAtCompileTime==1 ? 0 + : RowFactor==1 ? row + : row % m_rows; + const Index actual_col = internal::traits::ColsAtCompileTime==1 ? 0 + : ColFactor==1 ? col + : col % m_cols; + + return m_argImpl.coeff(actual_row, actual_col); + } + + template + PacketReturnType packet(Index row, Index col) const + { + const Index actual_row = internal::traits::RowsAtCompileTime==1 ? 0 + : RowFactor==1 ? row + : row % m_rows; + const Index actual_col = internal::traits::ColsAtCompileTime==1 ? 0 + : ColFactor==1 ? col + : col % m_cols; + + return m_argImpl.template packet(actual_row, actual_col); + } + +protected: + typename evaluator::type m_argImpl; + Index m_rows; // TODO: Get rid of this if known at compile time + Index m_cols; +}; + + +// -------------------- PartialReduxExpr -------------------- +// +// This is a wrapper around the expression object. +// TODO: Find out how to write a proper evaluator without duplicating +// the row() and col() member functions. + +template< typename ArgType, typename MemberOp, int Direction> +struct evaluator_impl > +{ + typedef PartialReduxExpr XprType; + + evaluator_impl(const XprType expr) + : m_expr(expr) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_expr.coeff(row, col); + } + + CoeffReturnType coeff(Index index) const + { + return m_expr.coeff(index); + } + +protected: + const XprType m_expr; +}; + + +// -------------------- MatrixWrapper and ArrayWrapper -------------------- +// +// evaluator_impl_wrapper_base is a common base class for the +// MatrixWrapper and ArrayWrapper evaluators. + +template +struct evaluator_impl_wrapper_base + : evaluator_impl_base +{ + evaluator_impl_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} + + typedef typename ArgType::Index Index; + typedef typename ArgType::Scalar Scalar; + typedef typename ArgType::CoeffReturnType CoeffReturnType; + typedef typename ArgType::PacketScalar PacketScalar; + typedef typename ArgType::PacketReturnType PacketReturnType; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_argImpl.coeff(row, col); + } + + CoeffReturnType coeff(Index index) const + { + return m_argImpl.coeff(index); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(row, col); + } + + Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(index); + } + + template + PacketReturnType packet(Index row, Index col) const + { + return m_argImpl.template packet(row, col); + } + + template + PacketReturnType packet(Index index) const + { + return m_argImpl.template packet(index); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + m_argImpl.template writePacket(row, col, x); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + m_argImpl.template writePacket(index, x); + } + +protected: + typename evaluator::type m_argImpl; +}; + +template +struct evaluator_impl > + : evaluator_impl_wrapper_base +{ + typedef MatrixWrapper XprType; + + evaluator_impl(const XprType& wrapper) + : evaluator_impl_wrapper_base(wrapper.nestedExpression()) + { } +}; + +template +struct evaluator_impl > + : evaluator_impl_wrapper_base +{ + typedef ArrayWrapper XprType; + + evaluator_impl(const XprType& wrapper) + : evaluator_impl_wrapper_base(wrapper.nestedExpression()) + { } +}; + + +// -------------------- Reverse -------------------- + +// defined in Reverse.h: +template struct reverse_packet_cond; + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef Reverse XprType; + + evaluator_impl(const XprType& reverse) + : m_argImpl(reverse.nestedExpression()), + m_rows(reverse.nestedExpression().rows()), + m_cols(reverse.nestedExpression().cols()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + enum { + PacketSize = internal::packet_traits::size, + IsRowMajor = XprType::IsRowMajor, + IsColMajor = !IsRowMajor, + ReverseRow = (Direction == Vertical) || (Direction == BothDirections), + ReverseCol = (Direction == Horizontal) || (Direction == BothDirections), + OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, + OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1, + ReversePacket = (Direction == BothDirections) + || ((Direction == Vertical) && IsColMajor) + || ((Direction == Horizontal) && IsRowMajor) + }; + typedef internal::reverse_packet_cond reverse_packet; + + CoeffReturnType coeff(Index row, Index col) const + { + return m_argImpl.coeff(ReverseRow ? m_rows - row - 1 : row, + ReverseCol ? m_cols - col - 1 : col); + } + + CoeffReturnType coeff(Index index) const + { + return m_argImpl.coeff(m_rows * m_cols - index - 1); + } + + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(ReverseRow ? m_rows - row - 1 : row, + ReverseCol ? m_cols - col - 1 : col); + } + + Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(m_rows * m_cols - index - 1); + } + + template + PacketScalar packet(Index row, Index col) const + { + return reverse_packet::run(m_argImpl.template packet( + ReverseRow ? m_rows - row - OffsetRow : row, + ReverseCol ? m_cols - col - OffsetCol : col)); + } + + template + PacketScalar packet(Index index) const + { + return preverse(m_argImpl.template packet(m_rows * m_cols - index - PacketSize)); + } + + template + void writePacket(Index row, Index col, const PacketScalar& x) + { + m_argImpl.template writePacket( + ReverseRow ? m_rows - row - OffsetRow : row, + ReverseCol ? m_cols - col - OffsetCol : col, + reverse_packet::run(x)); + } + + template + void writePacket(Index index, const PacketScalar& x) + { + m_argImpl.template writePacket(m_rows * m_cols - index - PacketSize, preverse(x)); + } + +protected: + typename evaluator::type m_argImpl; + Index m_rows; // TODO: Don't use if known at compile time or not needed + Index m_cols; +}; + + +// -------------------- Diagonal -------------------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef Diagonal XprType; + + evaluator_impl(const XprType& diagonal) + : m_argImpl(diagonal.nestedExpression()), + m_index(diagonal.index()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + CoeffReturnType coeff(Index row, Index) const + { + return m_argImpl.coeff(row + rowOffset(), row + colOffset()); + } + + CoeffReturnType coeff(Index index) const + { + return m_argImpl.coeff(index + rowOffset(), index + colOffset()); + } + + Scalar& coeffRef(Index row, Index) + { + return m_argImpl.coeffRef(row + rowOffset(), row + colOffset()); + } + + Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(index + rowOffset(), index + colOffset()); + } + +protected: + typename evaluator::type m_argImpl; + Index m_index; // TODO: Don't use if known at compile time + +private: + EIGEN_STRONG_INLINE Index rowOffset() const { return m_index>0 ? 0 : -m_index; } + EIGEN_STRONG_INLINE Index colOffset() const { return m_index>0 ? m_index : 0; } +}; + + +// ---------- SwapWrapper ---------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef SwapWrapper XprType; + + evaluator_impl(const XprType& swapWrapper) + : m_argImpl(swapWrapper.expression()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::Packet Packet; + + // This function and the next one are needed by assign to correctly align loads/stores + // TODO make Assign use .data() + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(row, col); + } + + inline Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(index); + } + + template + void copyCoeff(Index row, Index col, const OtherEvaluatorType& other) + { + OtherEvaluatorType& nonconst_other = const_cast(other); + Scalar tmp = m_argImpl.coeff(row, col); + m_argImpl.coeffRef(row, col) = nonconst_other.coeff(row, col); + nonconst_other.coeffRef(row, col) = tmp; + } + + template + void copyCoeff(Index index, const OtherEvaluatorType& other) + { + OtherEvaluatorType& nonconst_other = const_cast(other); + Scalar tmp = m_argImpl.coeff(index); + m_argImpl.coeffRef(index) = nonconst_other.coeff(index); + nonconst_other.coeffRef(index) = tmp; + } + + template + void copyPacket(Index row, Index col, const OtherEvaluatorType& other) + { + OtherEvaluatorType& nonconst_other = const_cast(other); + Packet tmp = m_argImpl.template packet(row, col); + m_argImpl.template writePacket + (row, col, nonconst_other.template packet(row, col)); + nonconst_other.template writePacket(row, col, tmp); + } + + template + void copyPacket(Index index, const OtherEvaluatorType& other) + { + OtherEvaluatorType& nonconst_other = const_cast(other); + Packet tmp = m_argImpl.template packet(index); + m_argImpl.template writePacket + (index, nonconst_other.template packet(index)); + nonconst_other.template writePacket(index, tmp); + } + +protected: + typename evaluator::type m_argImpl; +}; + + +// ---------- SelfCwiseBinaryOp ---------- + +template +struct evaluator_impl > + : evaluator_impl_base > +{ + typedef SelfCwiseBinaryOp XprType; + + evaluator_impl(const XprType& selfCwiseBinaryOp) + : m_argImpl(selfCwiseBinaryOp.expression()), + m_functor(selfCwiseBinaryOp.functor()) + { } + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::Packet Packet; + + // This function and the next one are needed by assign to correctly align loads/stores + // TODO make Assign use .data() + Scalar& coeffRef(Index row, Index col) + { + return m_argImpl.coeffRef(row, col); + } + + inline Scalar& coeffRef(Index index) + { + return m_argImpl.coeffRef(index); + } + + template + void copyCoeff(Index row, Index col, const OtherEvaluatorType& other) + { + Scalar& tmp = m_argImpl.coeffRef(row, col); + tmp = m_functor(tmp, other.coeff(row, col)); + } + + template + void copyCoeff(Index index, const OtherEvaluatorType& other) + { + Scalar& tmp = m_argImpl.coeffRef(index); + tmp = m_functor(tmp, other.coeff(index)); + } + + template + void copyPacket(Index row, Index col, const OtherEvaluatorType& other) + { + const Packet res = m_functor.packetOp(m_argImpl.template packet(row, col), + other.template packet(row, col)); + m_argImpl.template writePacket(row, col, res); + } + + template + void copyPacket(Index index, const OtherEvaluatorType& other) + { + const Packet res = m_functor.packetOp(m_argImpl.template packet(index), + other.template packet(index)); + m_argImpl.template writePacket(index, res); + } + +protected: + typename evaluator::type m_argImpl; + const BinaryOp m_functor; +}; + + +} // namespace internal + +#endif // EIGEN_COREEVALUATORS_H diff --git a/Eigen/src/Core/Product.h b/Eigen/src/Core/Product.h index 53eb0fbae..9bea26886 100644 --- a/Eigen/src/Core/Product.h +++ b/Eigen/src/Core/Product.h @@ -110,4 +110,18 @@ class ProductImpl : public internal::dense_xpr_base +const Product +prod(const Lhs& lhs, const Rhs& rhs) +{ + return Product(lhs,rhs); +} + #endif // EIGEN_PRODUCT_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6f8fc4ae3..c71534df9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -195,6 +195,7 @@ ei_add_test(nullary) ei_add_test(nesting_ops "${CMAKE_CXX_FLAGS_DEBUG}") ei_add_test(zerosized) ei_add_test(dontalign) +ei_add_test(evaluators) ei_add_test(sizeoverflow) ei_add_test(prec_inverse_4x4) ei_add_test(vectorwiseop) diff --git a/test/evaluators.cpp b/test/evaluators.cpp new file mode 100644 index 000000000..5c8e500bc --- /dev/null +++ b/test/evaluators.cpp @@ -0,0 +1,264 @@ +#define EIGEN_ENABLE_EVALUATORS +#include "main.h" + +using internal::copy_using_evaluator; +using namespace std; + +#define VERIFY_IS_APPROX_EVALUATOR(DEST,EXPR) VERIFY_IS_APPROX(copy_using_evaluator(DEST,(EXPR)), (EXPR).eval()); +#define VERIFY_IS_APPROX_EVALUATOR2(DEST,EXPR,REF) VERIFY_IS_APPROX(copy_using_evaluator(DEST,(EXPR)), (REF).eval()); + +void test_evaluators() +{ + // Testing Matrix evaluator and Transpose + Vector2d v = Vector2d::Random(); + const Vector2d v_const(v); + Vector2d v2; + RowVector2d w; + + VERIFY_IS_APPROX_EVALUATOR(v2, v); + VERIFY_IS_APPROX_EVALUATOR(v2, v_const); + + // Testing Transpose + VERIFY_IS_APPROX_EVALUATOR(w, v.transpose()); // Transpose as rvalue + VERIFY_IS_APPROX_EVALUATOR(w, v_const.transpose()); + + copy_using_evaluator(w.transpose(), v); // Transpose as lvalue + VERIFY_IS_APPROX(w,v.transpose().eval()); + + copy_using_evaluator(w.transpose(), v_const); + VERIFY_IS_APPROX(w,v_const.transpose().eval()); + + // Testing Array evaluator + ArrayXXf a(2,3); + ArrayXXf b(3,2); + a << 1,2,3, 4,5,6; + const ArrayXXf a_const(a); + + VERIFY_IS_APPROX_EVALUATOR(b, a.transpose()); + + VERIFY_IS_APPROX_EVALUATOR(b, a_const.transpose()); + + // Testing CwiseNullaryOp evaluator + copy_using_evaluator(w, RowVector2d::Random()); + VERIFY((w.array() >= -1).all() && (w.array() <= 1).all()); // not easy to test ... + + VERIFY_IS_APPROX_EVALUATOR(w, RowVector2d::Zero()); + + VERIFY_IS_APPROX_EVALUATOR(w, RowVector2d::Constant(3)); + + // mix CwiseNullaryOp and transpose + VERIFY_IS_APPROX_EVALUATOR(w, Vector2d::Zero().transpose()); + + { + int s = internal::random(1,100); + MatrixXf a(s,s), b(s,s), c(s,s), d(s,s); + a.setRandom(); + b.setRandom(); + c.setRandom(); + d.setRandom(); + VERIFY_IS_APPROX_EVALUATOR(d, (a + b)); + VERIFY_IS_APPROX_EVALUATOR(d, (a + b).transpose()); + VERIFY_IS_APPROX_EVALUATOR2(d, prod(a,b).transpose(), (a*b).transpose()); + VERIFY_IS_APPROX_EVALUATOR2(d, prod(a,b) + prod(b,c), a*b + b*c); + +// copy_using_evaluator(d, a.transpose() + (a.transpose() * (b+b))); +// cout << d << endl; + } + + // this does not work because Random is eval-before-nested: + // copy_using_evaluator(w, Vector2d::Random().transpose()); + + // test CwiseUnaryOp + VERIFY_IS_APPROX_EVALUATOR(v2, 3 * v); + VERIFY_IS_APPROX_EVALUATOR(w, (3 * v).transpose()); + VERIFY_IS_APPROX_EVALUATOR(b, (a + 3).transpose()); + VERIFY_IS_APPROX_EVALUATOR(b, (2 * a_const + 3).transpose()); + + // test CwiseBinaryOp + VERIFY_IS_APPROX_EVALUATOR(v2, v + Vector2d::Ones()); + VERIFY_IS_APPROX_EVALUATOR(w, (v + Vector2d::Ones()).transpose().cwiseProduct(RowVector2d::Constant(3))); + + // dynamic matrices and arrays + MatrixXd mat1(6,6), mat2(6,6); + VERIFY_IS_APPROX_EVALUATOR(mat1, MatrixXd::Identity(6,6)); + VERIFY_IS_APPROX_EVALUATOR(mat2, mat1); + copy_using_evaluator(mat2.transpose(), mat1); + VERIFY_IS_APPROX(mat2.transpose(), mat1); + + ArrayXXd arr1(6,6), arr2(6,6); + VERIFY_IS_APPROX_EVALUATOR(arr1, ArrayXXd::Constant(6,6, 3.0)); + VERIFY_IS_APPROX_EVALUATOR(arr2, arr1); + + // test direct traversal + Matrix3f m3; + Array33f a3; + VERIFY_IS_APPROX_EVALUATOR(m3, Matrix3f::Identity()); // matrix, nullary + // TODO: find a way to test direct traversal with array + VERIFY_IS_APPROX_EVALUATOR(m3.transpose(), Matrix3f::Identity().transpose()); // transpose + VERIFY_IS_APPROX_EVALUATOR(m3, 2 * Matrix3f::Identity()); // unary + VERIFY_IS_APPROX_EVALUATOR(m3, Matrix3f::Identity() + m3); // binary + VERIFY_IS_APPROX_EVALUATOR(m3.block(0,0,2,2), Matrix3f::Identity().block(1,1,2,2)); // block + + // test linear traversal + VERIFY_IS_APPROX_EVALUATOR(m3, Matrix3f::Zero()); // matrix, nullary + VERIFY_IS_APPROX_EVALUATOR(a3, Array33f::Zero()); // array + VERIFY_IS_APPROX_EVALUATOR(m3.transpose(), Matrix3f::Zero().transpose()); // transpose + VERIFY_IS_APPROX_EVALUATOR(m3, 2 * Matrix3f::Zero()); // unary + VERIFY_IS_APPROX_EVALUATOR(m3, Matrix3f::Zero() + m3); // binary + + // test inner vectorization + Matrix4f m4, m4src = Matrix4f::Random(); + Array44f a4, a4src = Matrix4f::Random(); + VERIFY_IS_APPROX_EVALUATOR(m4, m4src); // matrix + VERIFY_IS_APPROX_EVALUATOR(a4, a4src); // array + VERIFY_IS_APPROX_EVALUATOR(m4.transpose(), m4src.transpose()); // transpose + // TODO: find out why Matrix4f::Zero() does not allow inner vectorization + VERIFY_IS_APPROX_EVALUATOR(m4, 2 * m4src); // unary + VERIFY_IS_APPROX_EVALUATOR(m4, m4src + m4src); // binary + + // test linear vectorization + MatrixXf mX(6,6), mXsrc = MatrixXf::Random(6,6); + ArrayXXf aX(6,6), aXsrc = ArrayXXf::Random(6,6); + VERIFY_IS_APPROX_EVALUATOR(mX, mXsrc); // matrix + VERIFY_IS_APPROX_EVALUATOR(aX, aXsrc); // array + VERIFY_IS_APPROX_EVALUATOR(mX.transpose(), mXsrc.transpose()); // transpose + VERIFY_IS_APPROX_EVALUATOR(mX, MatrixXf::Zero(6,6)); // nullary + VERIFY_IS_APPROX_EVALUATOR(mX, 2 * mXsrc); // unary + VERIFY_IS_APPROX_EVALUATOR(mX, mXsrc + mXsrc); // binary + + // test blocks and slice vectorization + VERIFY_IS_APPROX_EVALUATOR(m4, (mXsrc.block<4,4>(1,0))); + VERIFY_IS_APPROX_EVALUATOR(aX, ArrayXXf::Constant(10, 10, 3.0).block(2, 3, 6, 6)); + + Matrix4f m4ref = m4; + copy_using_evaluator(m4.block(1, 1, 2, 3), m3.bottomRows(2)); + m4ref.block(1, 1, 2, 3) = m3.bottomRows(2); + VERIFY_IS_APPROX(m4, m4ref); + + mX.setIdentity(20,20); + MatrixXf mXref = MatrixXf::Identity(20,20); + mXsrc = MatrixXf::Random(9,12); + copy_using_evaluator(mX.block(4, 4, 9, 12), mXsrc); + mXref.block(4, 4, 9, 12) = mXsrc; + VERIFY_IS_APPROX(mX, mXref); + + // test Map + const float raw[3] = {1,2,3}; + float buffer[3] = {0,0,0}; + Vector3f v3; + Array3f a3f; + VERIFY_IS_APPROX_EVALUATOR(v3, Map(raw)); + VERIFY_IS_APPROX_EVALUATOR(a3f, Map(raw)); + Vector3f::Map(buffer) = 2*v3; + VERIFY(buffer[0] == 2); + VERIFY(buffer[1] == 4); + VERIFY(buffer[2] == 6); + + // test CwiseUnaryView + mat1.setRandom(); + mat2.setIdentity(); + MatrixXcd matXcd(6,6), matXcd_ref(6,6); + copy_using_evaluator(matXcd.real(), mat1); + copy_using_evaluator(matXcd.imag(), mat2); + matXcd_ref.real() = mat1; + matXcd_ref.imag() = mat2; + VERIFY_IS_APPROX(matXcd, matXcd_ref); + + // test Select + VERIFY_IS_APPROX_EVALUATOR(aX, (aXsrc > 0).select(aXsrc, -aXsrc)); + + // test Replicate + mXsrc = MatrixXf::Random(6, 6); + VectorXf vX = VectorXf::Random(6); + mX.resize(6, 6); + VERIFY_IS_APPROX_EVALUATOR(mX, mXsrc.colwise() + vX); + matXcd.resize(12, 12); + VERIFY_IS_APPROX_EVALUATOR(matXcd, matXcd_ref.replicate(2,2)); + VERIFY_IS_APPROX_EVALUATOR(matXcd, (matXcd_ref.replicate<2,2>())); + + // test partial reductions + VectorXd vec1(6); + VERIFY_IS_APPROX_EVALUATOR(vec1, mat1.rowwise().sum()); + VERIFY_IS_APPROX_EVALUATOR(vec1, mat1.colwise().sum().transpose()); + + // test MatrixWrapper and ArrayWrapper + mat1.setRandom(6,6); + arr1.setRandom(6,6); + VERIFY_IS_APPROX_EVALUATOR(mat2, arr1.matrix()); + VERIFY_IS_APPROX_EVALUATOR(arr2, mat1.array()); + VERIFY_IS_APPROX_EVALUATOR(mat2, (arr1 + 2).matrix()); + VERIFY_IS_APPROX_EVALUATOR(arr2, mat1.array() + 2); + mat2.array() = arr1 * arr1; + VERIFY_IS_APPROX(mat2, (arr1 * arr1).matrix()); + arr2.matrix() = MatrixXd::Identity(6,6); + VERIFY_IS_APPROX(arr2, MatrixXd::Identity(6,6).array()); + + // test Reverse + VERIFY_IS_APPROX_EVALUATOR(arr2, arr1.reverse()); + VERIFY_IS_APPROX_EVALUATOR(arr2, arr1.colwise().reverse()); + VERIFY_IS_APPROX_EVALUATOR(arr2, arr1.rowwise().reverse()); + arr2.reverse() = arr1; + VERIFY_IS_APPROX(arr2, arr1.reverse()); + + // test Diagonal + VERIFY_IS_APPROX_EVALUATOR(vec1, mat1.diagonal()); + vec1.resize(5); + VERIFY_IS_APPROX_EVALUATOR(vec1, mat1.diagonal(1)); + VERIFY_IS_APPROX_EVALUATOR(vec1, mat1.diagonal<-1>()); + vec1.setRandom(); + + mat2 = mat1; + copy_using_evaluator(mat1.diagonal(1), vec1); + mat2.diagonal(1) = vec1; + VERIFY_IS_APPROX(mat1, mat2); + + copy_using_evaluator(mat1.diagonal<-1>(), mat1.diagonal(1)); + mat2.diagonal<-1>() = mat2.diagonal(1); + VERIFY_IS_APPROX(mat1, mat2); + + { + // test swapping + MatrixXd mat1, mat2, mat1ref, mat2ref; + mat1ref = mat1 = MatrixXd::Random(6, 6); + mat2ref = mat2 = 2 * mat1 + MatrixXd::Identity(6, 6); + swap_using_evaluator(mat1, mat2); + mat1ref.swap(mat2ref); + VERIFY_IS_APPROX(mat1, mat1ref); + VERIFY_IS_APPROX(mat2, mat2ref); + + swap_using_evaluator(mat1.block(0, 0, 3, 3), mat2.block(3, 3, 3, 3)); + mat1ref.block(0, 0, 3, 3).swap(mat2ref.block(3, 3, 3, 3)); + VERIFY_IS_APPROX(mat1, mat1ref); + VERIFY_IS_APPROX(mat2, mat2ref); + + swap_using_evaluator(mat1.row(2), mat2.col(3).transpose()); + mat1.row(2).swap(mat2.col(3).transpose()); + VERIFY_IS_APPROX(mat1, mat1ref); + VERIFY_IS_APPROX(mat2, mat2ref); + } + + { + // test compound assignment + const Matrix4d mat_const = Matrix4d::Random(); + Matrix4d mat, mat_ref; + mat = mat_ref = Matrix4d::Identity(); + add_assign_using_evaluator(mat, mat_const); + mat_ref += mat_const; + VERIFY_IS_APPROX(mat, mat_ref); + + subtract_assign_using_evaluator(mat.row(1), 2*mat.row(2)); + mat_ref.row(1) -= 2*mat_ref.row(2); + VERIFY_IS_APPROX(mat, mat_ref); + + const ArrayXXf arr_const = ArrayXXf::Random(5,3); + ArrayXXf arr, arr_ref; + arr = arr_ref = ArrayXXf::Constant(5, 3, 0.5); + multiply_assign_using_evaluator(arr, arr_const); + arr_ref *= arr_const; + VERIFY_IS_APPROX(arr, arr_ref); + + divide_assign_using_evaluator(arr.row(1), arr.row(2) + 1); + arr_ref.row(1) /= (arr_ref.row(2) + 1); + VERIFY_IS_APPROX(arr, arr_ref); + } +}