diff --git a/Eigen/Core b/Eigen/Core index 468ae0c76..e2c9c69cd 100644 --- a/Eigen/Core +++ b/Eigen/Core @@ -316,7 +316,6 @@ using std::ptrdiff_t; #include "src/Core/Product.h" #include "src/Core/CoreEvaluators.h" #include "src/Core/AssignEvaluator.h" -#include "src/Core/ProductEvaluators.h" #endif #ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874 @@ -382,6 +381,10 @@ using std::ptrdiff_t; #include "src/Core/BandMatrix.h" #include "src/Core/CoreIterators.h" +#ifdef EIGEN_ENABLE_EVALUATORS +#include "src/Core/ProductEvaluators.h" +#endif + #include "src/Core/BooleanRedux.h" #include "src/Core/Select.h" #include "src/Core/VectorwiseOp.h" diff --git a/Eigen/src/Core/Product.h b/Eigen/src/Core/Product.h index 5d3789be7..52586e5c0 100644 --- a/Eigen/src/Core/Product.h +++ b/Eigen/src/Core/Product.h @@ -12,8 +12,7 @@ namespace Eigen { -template class Product; -template class ProductImpl; +template class ProductImpl; /** \class Product * \ingroup Core_Module @@ -24,13 +23,17 @@ template class ProductImpl; * \param Rhs the type of the right-hand side expression * * This class represents an expression of the product of two arbitrary matrices. + * + * The other template parameters are: + * \tparam Option can be DefaultProduct or LazyProduct + * \tparam ProductTag can be InnerProduct, OuterProduct, GemvProduct, GemmProduct. It is used to ease expression manipulations. * */ // Use ProductReturnType to get correct traits, in particular vectorization flags namespace internal { -template -struct traits > +template +struct traits > : traits::Type> { // We want A+B*C to be of type Product and not Product @@ -42,14 +45,15 @@ struct traits > } // end namespace internal -template -class Product : public ProductImpl::StorageKind, - typename internal::traits::StorageKind>::ret> +template +class Product : public ProductImpl::StorageKind, + typename internal::traits::StorageKind>::ret> { public: typedef typename ProductImpl< - Lhs, Rhs, + Lhs, Rhs, Option, ProductTag, typename internal::promote_storage_type::ret>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Product) @@ -78,13 +82,13 @@ class Product : public ProductImpl -class ProductImpl : public internal::dense_xpr_base >::type +template +class ProductImpl : public internal::dense_xpr_base >::type { typedef Product Derived; public: - typedef typename internal::dense_xpr_base >::type Base; + typedef typename internal::dense_xpr_base >::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Derived) }; @@ -102,6 +106,15 @@ prod(const Lhs& lhs, const Rhs& rhs) return Product(lhs,rhs); } +/** \internal used to test the evaluator only + */ +template +const Product +lazyprod(const Lhs& lhs, const Rhs& rhs) +{ + return Product(lhs,rhs); +} + } // end namespace Eigen #endif // EIGEN_PRODUCT_H diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index 855914f2e..42dd3c7ac 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h @@ -17,94 +17,172 @@ namespace Eigen { namespace internal { -// We can evaluate the product either all at once, like GeneralProduct and its evalTo() function, or -// traverse the matrix coefficient by coefficient, like CoeffBasedProduct. Use the existing logic -// in ProductReturnType to decide. + +// Helper class to perform a dense product with the destination at hand. +// Depending on the sizes of the factors, there are different evaluation strategies +// as controlled by internal::product_type. +template::value> +struct dense_product_impl; -template -struct product_evaluator_dispatcher; -template -struct evaluator_impl > - : product_evaluator_dispatcher, typename ProductReturnType::Type> +// The evaluator for default dense products creates a temporary and call dense_product_impl +template +struct evaluator_impl > + : public evaluator::PlainObject>::type { - typedef Product XprType; - typedef product_evaluator_dispatcher::Type> Base; - - evaluator_impl(const XprType& xpr) : Base(xpr) - { } -}; - -template -struct product_evaluator_traits_dispatcher; - -template -struct evaluator_traits > - : product_evaluator_traits_dispatcher, typename ProductReturnType::Type> -{ - static const int AssumeAliasing = 1; -}; - -// Case 1: Evaluate all at once -// -// We can view the GeneralProduct class as a part of the product evaluator. -// Four sub-cases: InnerProduct, OuterProduct, GemmProduct and GemvProduct. -// InnerProduct is special because GeneralProduct does not have an evalTo() method in this case. - -template -struct product_evaluator_traits_dispatcher, GeneralProduct > -{ - static const int HasEvalTo = 0; -}; - -template -struct product_evaluator_dispatcher, GeneralProduct > - : public evaluator::PlainObject>::type -{ - typedef Product XprType; + typedef Product XprType; typedef typename XprType::PlainObject PlainObject; - typedef typename evaluator::type evaluator_base; + typedef typename evaluator::type Base; - // TODO: Computation is too early (?) - product_evaluator_dispatcher(const XprType& xpr) : evaluator_base(m_result) + evaluator_impl(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) { - m_result.coeffRef(0,0) = (xpr.lhs().transpose().cwiseProduct(xpr.rhs())).sum(); + ::new (static_cast(this)) Base(m_result); + dense_product_impl::evalTo(m_result, xpr.lhs(), xpr.rhs()); } protected: PlainObject m_result; }; -// For the other three subcases, simply call the evalTo() method of GeneralProduct -// TODO: GeneralProduct should take evaluators, not expression objects. -template -struct product_evaluator_traits_dispatcher, GeneralProduct > +template +struct dense_product_impl { - static const int HasEvalTo = 1; -}; - -template -struct product_evaluator_dispatcher, GeneralProduct > -{ - typedef Product XprType; - typedef typename XprType::PlainObject PlainObject; - typedef typename evaluator::type evaluator_base; - - product_evaluator_dispatcher(const XprType& xpr) : m_xpr(xpr) - { } - - template - void evalTo(DstEvaluatorType /* not used */, DstXprType& dst) const + template + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - dst.resize(m_xpr.rows(), m_xpr.cols()); - GeneralProduct(m_xpr.lhs(), m_xpr.rhs()).evalTo(dst); + dst.coeffRef(0,0) = (lhs.transpose().cwiseProduct(rhs)).sum(); } -protected: - const XprType& m_xpr; + template + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + dst.coeffRef(0,0) += (lhs.transpose().cwiseProduct(rhs)).sum(); + } + + template + static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst.coeffRef(0,0) -= (lhs.transpose().cwiseProduct(rhs)).sum(); } }; + + +template +struct dense_product_impl +{ + typedef typename Product::Scalar Scalar; + + template + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + // TODO bypass GeneralProduct class + GeneralProduct(lhs,rhs).evalTo(dst); + } + + template + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + // TODO bypass GeneralProduct class + GeneralProduct(lhs,rhs).addTo(dst); + } + + template + static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + // TODO bypass GeneralProduct class + GeneralProduct(lhs,rhs).subTo(dst); + } + + template + static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { + // TODO bypass GeneralProduct class + GeneralProduct(lhs,rhs).scaleAndAddTo(dst, alpha); + } + +}; + + +// This base class provides default implementations for evalTo, addTo, subTo, in terms of scaleAndAddTo +template +struct dense_product_impl_base +{ + typedef typename Product::Scalar Scalar; + + template + static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst.setZero(); scaleAndAddTo(dst, lhs, rhs, Scalar(1)); } + + template + static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { scaleAndAddTo(dst,lhs, rhs, Scalar(1)); } + + template + static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); } + + template + static void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { Derived::scaleAndAddTo(dst,lhs,rhs,alpha); } + +}; + +template +struct dense_product_impl : dense_product_impl_base > +{ + typedef typename Product::Scalar Scalar; + enum { Side = Lhs::IsVectorAtCompileTime ? OnTheLeft : OnTheRight }; + typedef typename internal::conditional::type MatrixType; + + template + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { + internal::gemv_selector::HasUsableDirectAccess) + >::run(GeneralProduct(lhs,rhs), dst, alpha); + } +}; + +template +struct dense_product_impl : dense_product_impl_base > +{ + typedef typename Product::Scalar Scalar; + + template + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { + // TODO bypass GeneralProduct class + GeneralProduct(lhs,rhs).scaleAndAddTo(dst, alpha); + } +}; + +template +struct dense_product_impl +{ + typedef typename Product::Scalar Scalar; + + template + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst = lazyprod(lhs,rhs); } + + template + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst += lazyprod(lhs,rhs); } + + template + static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst -= lazyprod(lhs,rhs); } + + template + static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { dst += alpha * lazyprod(lhs,rhs); } +}; + +template +struct dense_product_impl : dense_product_impl {}; + // Case 2: Evaluate coeff by coeff // // This is mostly taken from CoeffBasedProduct.h @@ -117,20 +195,14 @@ struct etor_product_coeff_impl; template struct etor_product_packet_impl; -template -struct product_evaluator_traits_dispatcher, CoeffBasedProduct > +template +struct evaluator_impl > + : evaluator_impl_base > { - static const int HasEvalTo = 0; -}; + typedef Product XprType; + typedef CoeffBasedProduct CoeffBasedProductType; -template -struct product_evaluator_dispatcher, CoeffBasedProduct > - : evaluator_impl_base > -{ - typedef Product XprType; - typedef CoeffBasedProduct CoeffBasedProductType; - - product_evaluator_dispatcher(const XprType& xpr) + evaluator_impl(const XprType& xpr) : m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()), m_innerDim(xpr.lhs().cols()) @@ -150,11 +222,13 @@ struct product_evaluator_dispatcher, CoeffBasedProduct::InnerSize, CoeffReadCost = traits::CoeffReadCost, Unroll = CoeffReadCost != Dynamic && CoeffReadCost <= EIGEN_UNROLLING_LIMIT, - CanVectorizeInner = traits::CanVectorizeInner + CanVectorizeInner = traits::CanVectorizeInner, + Flags = CoeffBasedProductType::Flags }; typedef typename evaluator::type LhsEtorType; typedef typename evaluator::type RhsEtorType; + typedef etor_product_coeff_impl CoeffImpl; @@ -183,8 +257,8 @@ struct product_evaluator_dispatcher, CoeffBasedProduct PacketImpl; + Unroll ? InnerSize-1 : Dynamic, + LhsEtorType, RhsEtorType, PacketScalar, LoadMode> PacketImpl; PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res); return res; } @@ -197,6 +271,7 @@ protected: Index m_innerDim; }; + /*************************************************************************** * Normal product .coeff() implementation (with meta-unrolling) ***************************************************************************/ @@ -275,7 +350,6 @@ struct etor_product_coeff_impl::run(row, col, lhs, rhs, innerDim, pres); - etor_product_coeff_impl::run(row, col, lhs, rhs, innerDim, res); res = predux(pres); } }; diff --git a/Eigen/src/Core/util/Constants.h b/Eigen/src/Core/util/Constants.h index 05107fdfe..3178ff06e 100644 --- a/Eigen/src/Core/util/Constants.h +++ b/Eigen/src/Core/util/Constants.h @@ -425,7 +425,7 @@ namespace Architecture /** \internal \ingroup enums * Enum used as template parameter in GeneralProduct. */ -enum { CoeffBasedProductMode, LazyCoeffBasedProductMode, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; +enum { DefaultProduct=0, CoeffBasedProductMode, LazyCoeffBasedProductMode, LazyProduct, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; /** \internal \ingroup enums * Enum used in experimental parallel implementation. */ diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h index 0a2144c69..459422524 100644 --- a/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Eigen/src/Core/util/ForwardDeclarations.h @@ -87,11 +87,20 @@ template class CwiseNullaryOp; template class CwiseUnaryOp; template class CwiseUnaryView; template class CwiseBinaryOp; -template class SelfCwiseBinaryOp; +template class SelfCwiseBinaryOp; // TODO deprecated template class ProductBase; -template class Product; -template class GeneralProduct; -template class CoeffBasedProduct; + +namespace internal { + template struct product_tag; +} + +template::ret + > class Product; + +template class GeneralProduct; // TODO deprecated +template class CoeffBasedProduct; // TODO deprecated template class DiagonalBase; template class DiagonalWrapper;