From 5f6fbaa0e72d85a3904c7796490ada89d8acd4e4 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 19 Dec 2008 15:38:39 +0000 Subject: [PATCH] * fix a vectorization issue in Product * use _mm_malloc/_mm_free on other platforms than linux of MSVC (eg., cygwin, OSX) * replace a lot of inline keywords by EIGEN_STRONG_INLINE to compensate for poor MSVC inlining --- Eigen/src/Core/Assign.h | 46 +++++----- Eigen/src/Core/CacheFriendlyProduct.h | 2 +- Eigen/src/Core/Coeffs.h | 52 ++++++------ Eigen/src/Core/CwiseBinaryOp.h | 32 +++---- Eigen/src/Core/CwiseNullaryOp.h | 67 ++++++++------- Eigen/src/Core/CwiseUnaryOp.h | 36 ++++---- Eigen/src/Core/Functors.h | 78 ++++++++--------- Eigen/src/Core/Matrix.h | 48 +++++------ Eigen/src/Core/MatrixBase.h | 11 +-- Eigen/src/Core/Product.h | 55 ++++++------ Eigen/src/Core/arch/SSE/PacketMath.h | 116 +++++++++++++------------- Eigen/src/Core/util/Macros.h | 21 +++-- Eigen/src/Core/util/Memory.h | 34 ++++---- 13 files changed, 302 insertions(+), 296 deletions(-) diff --git a/Eigen/src/Core/Assign.h b/Eigen/src/Core/Assign.h index bcb3e9e7e..7e8102997 100644 --- a/Eigen/src/Core/Assign.h +++ b/Eigen/src/Core/Assign.h @@ -112,7 +112,7 @@ struct ei_assign_novec_CompleteUnrolling : Index / Derived1::RowsAtCompileTime }; - inline static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { dst.copyCoeff(row, col, src); ei_assign_novec_CompleteUnrolling::run(dst, src); @@ -122,13 +122,13 @@ struct ei_assign_novec_CompleteUnrolling template struct ei_assign_novec_CompleteUnrolling { - inline static void run(Derived1 &, const Derived2 &) {} + EIGEN_STRONG_INLINE static void run(Derived1 &, const Derived2 &) {} }; template struct ei_assign_novec_InnerUnrolling { - inline static void run(Derived1 &dst, const Derived2 &src, int row_or_col) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src, int row_or_col) { const bool rowMajor = int(Derived1::Flags)&RowMajorBit; const int row = rowMajor ? row_or_col : Index; @@ -141,7 +141,7 @@ struct ei_assign_novec_InnerUnrolling template struct ei_assign_novec_InnerUnrolling { - inline static void run(Derived1 &, const Derived2 &, int) {} + EIGEN_STRONG_INLINE static void run(Derived1 &, const Derived2 &, int) {} }; /************************** @@ -161,7 +161,7 @@ struct ei_assign_innervec_CompleteUnrolling SrcAlignment = ei_assign_traits::SrcAlignment }; - inline static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { dst.template copyPacket(row, col, src); ei_assign_innervec_CompleteUnrolling struct ei_assign_innervec_CompleteUnrolling { - inline static void run(Derived1 &, const Derived2 &) {} + EIGEN_STRONG_INLINE static void run(Derived1 &, const Derived2 &) {} }; template struct ei_assign_innervec_InnerUnrolling { - inline static void run(Derived1 &dst, const Derived2 &src, int row_or_col) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src, int row_or_col) { const int row = int(Derived1::Flags)&RowMajorBit ? row_or_col : Index; const int col = int(Derived1::Flags)&RowMajorBit ? Index : row_or_col; @@ -191,7 +191,7 @@ struct ei_assign_innervec_InnerUnrolling template struct ei_assign_innervec_InnerUnrolling { - inline static void run(Derived1 &, const Derived2 &, int) {} + EIGEN_STRONG_INLINE static void run(Derived1 &, const Derived2 &, int) {} }; /*************************************************************************** @@ -210,7 +210,7 @@ struct ei_assign_impl; template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + inline static void run(Derived1 &dst, const Derived2 &src) { const int innerSize = dst.innerSize(); const int outerSize = dst.outerSize(); @@ -228,7 +228,7 @@ struct ei_assign_impl template struct ei_assign_impl { - inline static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { ei_assign_novec_CompleteUnrolling ::run(dst, src); @@ -238,7 +238,7 @@ struct ei_assign_impl template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { const bool rowMajor = int(Derived1::Flags)&RowMajorBit; const int innerSize = rowMajor ? Derived1::ColsAtCompileTime : Derived1::RowsAtCompileTime; @@ -256,7 +256,7 @@ struct ei_assign_impl template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + inline static void run(Derived1 &dst, const Derived2 &src) { const int innerSize = dst.innerSize(); const int outerSize = dst.outerSize(); @@ -275,7 +275,7 @@ struct ei_assign_impl template struct ei_assign_impl { - inline static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { ei_assign_innervec_CompleteUnrolling ::run(dst, src); @@ -285,7 +285,7 @@ struct ei_assign_impl template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { const bool rowMajor = int(Derived1::Flags)&RowMajorBit; const int innerSize = rowMajor ? Derived1::ColsAtCompileTime : Derived1::RowsAtCompileTime; @@ -303,7 +303,7 @@ struct ei_assign_impl template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + inline static void run(Derived1 &dst, const Derived2 &src) { const int size = dst.size(); const int packetSize = ei_packet_traits::size; @@ -327,7 +327,7 @@ struct ei_assign_impl template struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + EIGEN_STRONG_INLINE static void run(Derived1 &dst, const Derived2 &src) { const int size = Derived1::SizeAtCompileTime; const int packetSize = ei_packet_traits::size; @@ -345,7 +345,7 @@ struct ei_assign_impl struct ei_assign_impl { - static void run(Derived1 &dst, const Derived2 &src) + inline static void run(Derived1 &dst, const Derived2 &src) { const int packetSize = ei_packet_traits::size; const int packetAlignedMask = packetSize - 1; @@ -397,7 +397,7 @@ struct ei_assign_impl template template -inline Derived& MatrixBase +EIGEN_STRONG_INLINE Derived& MatrixBase ::lazyAssign(const MatrixBase& other) { EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Derived,OtherDerived) @@ -417,24 +417,24 @@ struct ei_assign_selector; template struct ei_assign_selector { - static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.derived()); } + EIGEN_STRONG_INLINE static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.derived()); } }; template struct ei_assign_selector { - static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.eval()); } + EIGEN_STRONG_INLINE static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.eval()); } }; template struct ei_assign_selector { - static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose()); } + EIGEN_STRONG_INLINE static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose()); } }; template struct ei_assign_selector { - static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose().eval()); } + EIGEN_STRONG_INLINE static Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose().eval()); } }; template template -inline Derived& MatrixBase +EIGEN_STRONG_INLINE Derived& MatrixBase ::operator=(const MatrixBase& other) { return ei_assign_selector::run(derived(), other.derived()); diff --git a/Eigen/src/Core/CacheFriendlyProduct.h b/Eigen/src/Core/CacheFriendlyProduct.h index 928b50cfa..0daccaf5f 100644 --- a/Eigen/src/Core/CacheFriendlyProduct.h +++ b/Eigen/src/Core/CacheFriendlyProduct.h @@ -494,7 +494,7 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_colmajor_times_vector( /* process remaining coeffs (or all if there is no explicit vectorization) */ for (int j=alignedSize; j -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::coeff(int row, int col) const { ei_internal_assert(row >= 0 && row < rows() @@ -53,7 +53,7 @@ inline const typename ei_traits::Scalar MatrixBase * \sa operator()(int,int), operator[](int) const */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::operator()(int row, int col) const { ei_assert(row >= 0 && row < rows() @@ -76,7 +76,7 @@ inline const typename ei_traits::Scalar MatrixBase * \sa operator()(int,int), coeff(int, int) const, coeffRef(int) */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::coeffRef(int row, int col) { ei_internal_assert(row >= 0 && row < rows() @@ -89,7 +89,7 @@ inline typename ei_traits::Scalar& MatrixBase * \sa operator()(int,int) const, operator[](int) */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::operator()(int row, int col) { ei_assert(row >= 0 && row < rows() @@ -112,7 +112,7 @@ inline typename ei_traits::Scalar& MatrixBase * \sa operator[](int) const, coeffRef(int), coeff(int,int) const */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::coeff(int index) const { ei_internal_assert(index >= 0 && index < size()); @@ -127,7 +127,7 @@ inline const typename ei_traits::Scalar MatrixBase * z() const, w() const */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::operator[](int index) const { ei_assert(index >= 0 && index < size()); @@ -144,7 +144,7 @@ inline const typename ei_traits::Scalar MatrixBase * z() const, w() const */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::operator()(int index) const { ei_assert(index >= 0 && index < size()); @@ -166,7 +166,7 @@ inline const typename ei_traits::Scalar MatrixBase * \sa operator[](int), coeff(int) const, coeffRef(int,int) */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::coeffRef(int index) { ei_internal_assert(index >= 0 && index < size()); @@ -180,7 +180,7 @@ inline typename ei_traits::Scalar& MatrixBase * \sa operator[](int) const, operator()(int,int), x(), y(), z(), w() */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::operator[](int index) { ei_assert(index >= 0 && index < size()); @@ -196,7 +196,7 @@ inline typename ei_traits::Scalar& MatrixBase * \sa operator[](int) const, operator()(int,int), x(), y(), z(), w() */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::operator()(int index) { ei_assert(index >= 0 && index < size()); @@ -205,42 +205,42 @@ inline typename ei_traits::Scalar& MatrixBase /** equivalent to operator[](0). */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::x() const { return (*this)[0]; } /** equivalent to operator[](1). */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::y() const { return (*this)[1]; } /** equivalent to operator[](2). */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::z() const { return (*this)[2]; } /** equivalent to operator[](3). */ template -inline const typename ei_traits::Scalar MatrixBase +EIGEN_STRONG_INLINE const typename ei_traits::Scalar MatrixBase ::w() const { return (*this)[3]; } /** equivalent to operator[](0). */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::x() { return (*this)[0]; } /** equivalent to operator[](1). */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::y() { return (*this)[1]; } /** equivalent to operator[](2). */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::z() { return (*this)[2]; } /** equivalent to operator[](3). */ template -inline typename ei_traits::Scalar& MatrixBase +EIGEN_STRONG_INLINE typename ei_traits::Scalar& MatrixBase ::w() { return (*this)[3]; } /** \returns the packet of coefficients starting at the given row and column. It is your responsibility @@ -253,7 +253,7 @@ inline typename ei_traits::Scalar& MatrixBase */ template template -inline typename ei_packet_traits::Scalar>::type +EIGEN_STRONG_INLINE typename ei_packet_traits::Scalar>::type MatrixBase::packet(int row, int col) const { ei_internal_assert(row >= 0 && row < rows() @@ -271,7 +271,7 @@ MatrixBase::packet(int row, int col) const */ template template -inline void MatrixBase::writePacket +EIGEN_STRONG_INLINE void MatrixBase::writePacket (int row, int col, const typename ei_packet_traits::Scalar>::type& x) { ei_internal_assert(row >= 0 && row < rows() @@ -289,7 +289,7 @@ inline void MatrixBase::writePacket */ template template -inline typename ei_packet_traits::Scalar>::type +EIGEN_STRONG_INLINE typename ei_packet_traits::Scalar>::type MatrixBase::packet(int index) const { ei_internal_assert(index >= 0 && index < size()); @@ -306,7 +306,7 @@ MatrixBase::packet(int index) const */ template template -inline void MatrixBase::writePacket +EIGEN_STRONG_INLINE void MatrixBase::writePacket (int index, const typename ei_packet_traits::Scalar>::type& x) { ei_internal_assert(index >= 0 && index < size()); @@ -322,7 +322,7 @@ inline void MatrixBase::writePacket */ template template -inline void MatrixBase::copyCoeff(int row, int col, const MatrixBase& other) +EIGEN_STRONG_INLINE void MatrixBase::copyCoeff(int row, int col, const MatrixBase& other) { ei_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); @@ -338,7 +338,7 @@ inline void MatrixBase::copyCoeff(int row, int col, const MatrixBase template -inline void MatrixBase::copyCoeff(int index, const MatrixBase& other) +EIGEN_STRONG_INLINE void MatrixBase::copyCoeff(int index, const MatrixBase& other) { ei_internal_assert(index >= 0 && index < size()); derived().coeffRef(index) = other.derived().coeff(index); @@ -353,7 +353,7 @@ inline void MatrixBase::copyCoeff(int index, const MatrixBase template -inline void MatrixBase::copyPacket(int row, int col, const MatrixBase& other) +EIGEN_STRONG_INLINE void MatrixBase::copyPacket(int row, int col, const MatrixBase& other) { ei_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); @@ -370,7 +370,7 @@ inline void MatrixBase::copyPacket(int row, int col, const MatrixBase template -inline void MatrixBase::copyPacket(int index, const MatrixBase& other) +EIGEN_STRONG_INLINE void MatrixBase::copyPacket(int index, const MatrixBase& other) { ei_internal_assert(index >= 0 && index < size()); derived().template writePacket(index, diff --git a/Eigen/src/Core/CwiseBinaryOp.h b/Eigen/src/Core/CwiseBinaryOp.h index e6ce04717..834a9b820 100644 --- a/Eigen/src/Core/CwiseBinaryOp.h +++ b/Eigen/src/Core/CwiseBinaryOp.h @@ -89,7 +89,7 @@ class CwiseBinaryOp : ei_no_assignment_operator, class InnerIterator; - inline CwiseBinaryOp(const Lhs& lhs, const Rhs& rhs, const BinaryOp& func = BinaryOp()) + EIGEN_STRONG_INLINE CwiseBinaryOp(const Lhs& lhs, const Rhs& rhs, const BinaryOp& func = BinaryOp()) : m_lhs(lhs), m_rhs(rhs), m_functor(func) { // we require Lhs and Rhs to have the same scalar type. Currently there is no example of a binary functor @@ -106,27 +106,27 @@ class CwiseBinaryOp : ei_no_assignment_operator, ei_assert(lhs.rows() == rhs.rows() && lhs.cols() == rhs.cols()); } - inline int rows() const { return m_lhs.rows(); } - inline int cols() const { return m_lhs.cols(); } + EIGEN_STRONG_INLINE int rows() const { return m_lhs.rows(); } + EIGEN_STRONG_INLINE int cols() const { return m_lhs.cols(); } - inline const Scalar coeff(int row, int col) const + EIGEN_STRONG_INLINE const Scalar coeff(int row, int col) const { return m_functor(m_lhs.coeff(row, col), m_rhs.coeff(row, col)); } template - inline PacketScalar packet(int row, int col) const + EIGEN_STRONG_INLINE PacketScalar packet(int row, int col) const { return m_functor.packetOp(m_lhs.template packet(row, col), m_rhs.template packet(row, col)); } - inline const Scalar coeff(int index) const + EIGEN_STRONG_INLINE const Scalar coeff(int index) const { return m_functor(m_lhs.coeff(index), m_rhs.coeff(index)); } template - inline PacketScalar packet(int index) const + EIGEN_STRONG_INLINE PacketScalar packet(int index) const { return m_functor.packetOp(m_lhs.template packet(index), m_rhs.template packet(index)); } @@ -145,7 +145,7 @@ class CwiseBinaryOp : ei_no_assignment_operator, */ template template -inline const CwiseBinaryOp::Scalar>, +EIGEN_STRONG_INLINE const CwiseBinaryOp::Scalar>, Derived, OtherDerived> MatrixBase::operator-(const MatrixBase &other) const { @@ -159,7 +159,7 @@ MatrixBase::operator-(const MatrixBase &other) const */ template template -inline Derived & +EIGEN_STRONG_INLINE Derived & MatrixBase::operator-=(const MatrixBase &other) { return *this = *this - other; @@ -175,7 +175,7 @@ MatrixBase::operator-=(const MatrixBase &other) */ template template -inline const CwiseBinaryOp::Scalar>, Derived, OtherDerived> +EIGEN_STRONG_INLINE const CwiseBinaryOp::Scalar>, Derived, OtherDerived> MatrixBase::operator+(const MatrixBase &other) const { return CwiseBinaryOp, Derived, OtherDerived>(derived(), other.derived()); @@ -187,7 +187,7 @@ MatrixBase::operator+(const MatrixBase &other) const */ template template -inline Derived & +EIGEN_STRONG_INLINE Derived & MatrixBase::operator+=(const MatrixBase& other) { return *this = *this + other; @@ -202,7 +202,7 @@ MatrixBase::operator+=(const MatrixBase& other) */ template template -inline const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_product_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_product_op) Cwise::operator*(const MatrixBase &other) const { return EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_product_op)(_expression(), other.derived()); @@ -217,7 +217,7 @@ Cwise::operator*(const MatrixBase &other) const */ template template -inline const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_quotient_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_quotient_op) Cwise::operator/(const MatrixBase &other) const { return EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_quotient_op)(_expression(), other.derived()); @@ -232,7 +232,7 @@ Cwise::operator/(const MatrixBase &other) const */ template template -inline const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_min_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_min_op) Cwise::min(const MatrixBase &other) const { return EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_min_op)(_expression(), other.derived()); @@ -247,7 +247,7 @@ Cwise::min(const MatrixBase &other) const */ template template -inline const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_max_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_max_op) Cwise::max(const MatrixBase &other) const { return EIGEN_CWISE_BINOP_RETURN_TYPE(ei_scalar_max_op)(_expression(), other.derived()); @@ -268,7 +268,7 @@ Cwise::max(const MatrixBase &other) const */ template template -inline const CwiseBinaryOp +EIGEN_STRONG_INLINE const CwiseBinaryOp MatrixBase::binaryExpr(const MatrixBase &other, const CustomBinaryOp& func) const { return CwiseBinaryOp(derived(), other.derived(), func); diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index 32cc12d4e..a92e3bb48 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h @@ -75,21 +75,21 @@ class CwiseNullaryOp : ei_no_assignment_operator, && (ColsAtCompileTime == Dynamic || ColsAtCompileTime == cols)); } - int rows() const { return m_rows.value(); } - int cols() const { return m_cols.value(); } + EIGEN_STRONG_INLINE int rows() const { return m_rows.value(); } + EIGEN_STRONG_INLINE int cols() const { return m_cols.value(); } - const Scalar coeff(int rows, int cols) const + EIGEN_STRONG_INLINE const Scalar coeff(int rows, int cols) const { return m_functor(rows, cols); } template - PacketScalar packet(int, int) const + EIGEN_STRONG_INLINE PacketScalar packet(int, int) const { return m_functor.packetOp(); } - const Scalar coeff(int index) const + EIGEN_STRONG_INLINE const Scalar coeff(int index) const { if(RowsAtCompileTime == 1) return m_functor(0, index); @@ -98,7 +98,7 @@ class CwiseNullaryOp : ei_no_assignment_operator, } template - PacketScalar packet(int) const + EIGEN_STRONG_INLINE PacketScalar packet(int) const { return m_functor.packetOp(); } @@ -125,7 +125,7 @@ class CwiseNullaryOp : ei_no_assignment_operator, */ template template -const CwiseNullaryOp +EIGEN_STRONG_INLINE const CwiseNullaryOp MatrixBase::NullaryExpr(int rows, int cols, const CustomNullaryOp& func) { return CwiseNullaryOp(rows, cols, func); @@ -148,7 +148,7 @@ MatrixBase::NullaryExpr(int rows, int cols, const CustomNullaryOp& func */ template template -const CwiseNullaryOp +EIGEN_STRONG_INLINE const CwiseNullaryOp MatrixBase::NullaryExpr(int size, const CustomNullaryOp& func) { ei_assert(IsVectorAtCompileTime); @@ -167,7 +167,7 @@ MatrixBase::NullaryExpr(int size, const CustomNullaryOp& func) */ template template -const CwiseNullaryOp +EIGEN_STRONG_INLINE const CwiseNullaryOp MatrixBase::NullaryExpr(const CustomNullaryOp& func) { return CwiseNullaryOp(RowsAtCompileTime, ColsAtCompileTime, func); @@ -187,7 +187,7 @@ MatrixBase::NullaryExpr(const CustomNullaryOp& func) * \sa class CwiseNullaryOp */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Constant(int rows, int cols, const Scalar& value) { return NullaryExpr(rows, cols, ei_scalar_constant_op(value)); @@ -209,7 +209,7 @@ MatrixBase::Constant(int rows, int cols, const Scalar& value) * \sa class CwiseNullaryOp */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Constant(int size, const Scalar& value) { return NullaryExpr(size, ei_scalar_constant_op(value)); @@ -225,7 +225,7 @@ MatrixBase::Constant(int size, const Scalar& value) * \sa class CwiseNullaryOp */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Constant(const Scalar& value) { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) @@ -248,7 +248,7 @@ bool MatrixBase::isApproxToConstant * \sa class CwiseNullaryOp, Zero(), Ones() */ template -Derived& MatrixBase::setConstant(const Scalar& value) +EIGEN_STRONG_INLINE Derived& MatrixBase::setConstant(const Scalar& value) { return derived() = Constant(rows(), cols(), value); } @@ -272,7 +272,7 @@ Derived& MatrixBase::setConstant(const Scalar& value) * \sa Zero(), Zero(int) */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Zero(int rows, int cols) { return Constant(rows, cols, Scalar(0)); @@ -295,7 +295,7 @@ MatrixBase::Zero(int rows, int cols) * \sa Zero(), Zero(int,int) */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Zero(int size) { return Constant(size, Scalar(0)); @@ -312,7 +312,7 @@ MatrixBase::Zero(int size) * \sa Zero(int), Zero(int,int) */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Zero() { return Constant(Scalar(0)); @@ -327,8 +327,7 @@ MatrixBase::Zero() * \sa class CwiseNullaryOp, Zero() */ template -bool MatrixBase::isZero -(RealScalar prec) const +bool MatrixBase::isZero(RealScalar prec) const { for(int j = 0; j < cols(); ++j) for(int i = 0; i < rows(); ++i) @@ -345,7 +344,7 @@ bool MatrixBase::isZero * \sa class CwiseNullaryOp, Zero() */ template -Derived& MatrixBase::setZero() +EIGEN_STRONG_INLINE Derived& MatrixBase::setZero() { return setConstant(Scalar(0)); } @@ -369,7 +368,7 @@ Derived& MatrixBase::setZero() * \sa Ones(), Ones(int), isOnes(), class Ones */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Ones(int rows, int cols) { return Constant(rows, cols, Scalar(1)); @@ -392,7 +391,7 @@ MatrixBase::Ones(int rows, int cols) * \sa Ones(), Ones(int,int), isOnes(), class Ones */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Ones(int size) { return Constant(size, Scalar(1)); @@ -409,7 +408,7 @@ MatrixBase::Ones(int size) * \sa Ones(int), Ones(int,int), isOnes(), class Ones */ template -const typename MatrixBase::ConstantReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ConstantReturnType MatrixBase::Ones() { return Constant(Scalar(1)); @@ -438,7 +437,7 @@ bool MatrixBase::isOnes * \sa class CwiseNullaryOp, Ones() */ template -Derived& MatrixBase::setOnes() +EIGEN_STRONG_INLINE Derived& MatrixBase::setOnes() { return setConstant(Scalar(1)); } @@ -462,7 +461,7 @@ Derived& MatrixBase::setOnes() * \sa Identity(), setIdentity(), isIdentity() */ template -inline const typename MatrixBase::IdentityReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::IdentityReturnType MatrixBase::Identity(int rows, int cols) { return NullaryExpr(rows, cols, ei_scalar_identity_op()); @@ -479,7 +478,7 @@ MatrixBase::Identity(int rows, int cols) * \sa Identity(int,int), setIdentity(), isIdentity() */ template -inline const typename MatrixBase::IdentityReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::IdentityReturnType MatrixBase::Identity() { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) @@ -521,7 +520,7 @@ bool MatrixBase::isIdentity template=16)> struct ei_setIdentity_impl { - static inline Derived& run(Derived& m) + static EIGEN_STRONG_INLINE Derived& run(Derived& m) { return m = Derived::Identity(m.rows(), m.cols()); } @@ -530,7 +529,7 @@ struct ei_setIdentity_impl template struct ei_setIdentity_impl { - static inline Derived& run(Derived& m) + static EIGEN_STRONG_INLINE Derived& run(Derived& m) { m.setZero(); const int size = std::min(m.rows(), m.cols()); @@ -547,7 +546,7 @@ struct ei_setIdentity_impl * \sa class CwiseNullaryOp, Identity(), Identity(int,int), isIdentity() */ template -inline Derived& MatrixBase::setIdentity() +EIGEN_STRONG_INLINE Derived& MatrixBase::setIdentity() { return ei_setIdentity_impl::run(derived()); } @@ -559,7 +558,7 @@ inline Derived& MatrixBase::setIdentity() * \sa MatrixBase::Unit(int), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::Unit(int size, int i) +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::Unit(int size, int i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(size,size), i); @@ -574,7 +573,7 @@ const typename MatrixBase::BasisReturnType MatrixBase::Unit(in * \sa MatrixBase::Unit(int,int), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::Unit(int i) +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::Unit(int i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(),i); @@ -587,7 +586,7 @@ const typename MatrixBase::BasisReturnType MatrixBase::Unit(in * \sa MatrixBase::Unit(int,int), MatrixBase::Unit(int), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::UnitX() +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitX() { return Derived::Unit(0); } /** \returns an expression of the Y axis unit vector (0,1{,0}^*) @@ -597,7 +596,7 @@ const typename MatrixBase::BasisReturnType MatrixBase::UnitX() * \sa MatrixBase::Unit(int,int), MatrixBase::Unit(int), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::UnitY() +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitY() { return Derived::Unit(1); } /** \returns an expression of the Z axis unit vector (0,0,1{,0}^*) @@ -607,7 +606,7 @@ const typename MatrixBase::BasisReturnType MatrixBase::UnitY() * \sa MatrixBase::Unit(int,int), MatrixBase::Unit(int), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::UnitZ() +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitZ() { return Derived::Unit(2); } /** \returns an expression of the W axis unit vector (0,0,0,1) @@ -617,7 +616,7 @@ const typename MatrixBase::BasisReturnType MatrixBase::UnitZ() * \sa MatrixBase::Unit(int,int), MatrixBase::Unit(int), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template -const typename MatrixBase::BasisReturnType MatrixBase::UnitW() +EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitW() { return Derived::Unit(3); } #endif // EIGEN_CWISE_NULLARY_OP_H diff --git a/Eigen/src/Core/CwiseUnaryOp.h b/Eigen/src/Core/CwiseUnaryOp.h index e5520bc64..701bc34b0 100644 --- a/Eigen/src/Core/CwiseUnaryOp.h +++ b/Eigen/src/Core/CwiseUnaryOp.h @@ -74,27 +74,27 @@ class CwiseUnaryOp : ei_no_assignment_operator, inline CwiseUnaryOp(const MatrixType& mat, const UnaryOp& func = UnaryOp()) : m_matrix(mat), m_functor(func) {} - inline int rows() const { return m_matrix.rows(); } - inline int cols() const { return m_matrix.cols(); } + EIGEN_STRONG_INLINE int rows() const { return m_matrix.rows(); } + EIGEN_STRONG_INLINE int cols() const { return m_matrix.cols(); } - inline const Scalar coeff(int row, int col) const + EIGEN_STRONG_INLINE const Scalar coeff(int row, int col) const { return m_functor(m_matrix.coeff(row, col)); } template - inline PacketScalar packet(int row, int col) const + EIGEN_STRONG_INLINE PacketScalar packet(int row, int col) const { return m_functor.packetOp(m_matrix.template packet(row, col)); } - inline const Scalar coeff(int index) const + EIGEN_STRONG_INLINE const Scalar coeff(int index) const { return m_functor(m_matrix.coeff(index)); } template - inline PacketScalar packet(int index) const + EIGEN_STRONG_INLINE PacketScalar packet(int index) const { return m_functor.packetOp(m_matrix.template packet(index)); } @@ -119,7 +119,7 @@ class CwiseUnaryOp : ei_no_assignment_operator, */ template template -inline const CwiseUnaryOp +EIGEN_STRONG_INLINE const CwiseUnaryOp MatrixBase::unaryExpr(const CustomUnaryOp& func) const { return CwiseUnaryOp(derived(), func); @@ -128,7 +128,7 @@ MatrixBase::unaryExpr(const CustomUnaryOp& func) const /** \returns an expression of the opposite of \c *this */ template -inline const CwiseUnaryOp::Scalar>,Derived> +EIGEN_STRONG_INLINE const CwiseUnaryOp::Scalar>,Derived> MatrixBase::operator-() const { return derived(); @@ -142,7 +142,7 @@ MatrixBase::operator-() const * \sa abs2() */ template -inline const EIGEN_CWISE_UNOP_RETURN_TYPE(ei_scalar_abs_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_UNOP_RETURN_TYPE(ei_scalar_abs_op) Cwise::abs() const { return _expression(); @@ -156,7 +156,7 @@ Cwise::abs() const * \sa abs(), square() */ template -inline const EIGEN_CWISE_UNOP_RETURN_TYPE(ei_scalar_abs2_op) +EIGEN_STRONG_INLINE const EIGEN_CWISE_UNOP_RETURN_TYPE(ei_scalar_abs2_op) Cwise::abs2() const { return _expression(); @@ -166,7 +166,7 @@ Cwise::abs2() const * * \sa adjoint() */ template -inline typename MatrixBase::ConjugateReturnType +EIGEN_STRONG_INLINE typename MatrixBase::ConjugateReturnType MatrixBase::conjugate() const { return ConjugateReturnType(derived()); @@ -176,14 +176,14 @@ MatrixBase::conjugate() const * * \sa imag() */ template -inline const typename MatrixBase::RealReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::RealReturnType MatrixBase::real() const { return derived(); } /** \returns an expression of the imaginary part of \c *this. * * \sa real() */ template -inline const typename MatrixBase::ImagReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ImagReturnType MatrixBase::imag() const { return derived(); } /** \returns an expression of *this with the \a Scalar type casted to @@ -195,7 +195,7 @@ MatrixBase::imag() const { return derived(); } */ template template -inline const CwiseUnaryOp::Scalar, NewType>, Derived> +EIGEN_STRONG_INLINE const CwiseUnaryOp::Scalar, NewType>, Derived> MatrixBase::cast() const { return derived(); @@ -203,7 +203,7 @@ MatrixBase::cast() const /** \relates MatrixBase */ template -inline const typename MatrixBase::ScalarMultipleReturnType +EIGEN_STRONG_INLINE const typename MatrixBase::ScalarMultipleReturnType MatrixBase::operator*(const Scalar& scalar) const { return CwiseUnaryOp, Derived> @@ -212,7 +212,7 @@ MatrixBase::operator*(const Scalar& scalar) const /** \relates MatrixBase */ template -inline const CwiseUnaryOp::Scalar>, Derived> +EIGEN_STRONG_INLINE const CwiseUnaryOp::Scalar>, Derived> MatrixBase::operator/(const Scalar& scalar) const { return CwiseUnaryOp, Derived> @@ -220,14 +220,14 @@ MatrixBase::operator/(const Scalar& scalar) const } template -inline Derived& +EIGEN_STRONG_INLINE Derived& MatrixBase::operator*=(const Scalar& other) { return *this = *this * other; } template -inline Derived& +EIGEN_STRONG_INLINE Derived& MatrixBase::operator/=(const Scalar& other) { return *this = *this / other; diff --git a/Eigen/src/Core/Functors.h b/Eigen/src/Core/Functors.h index de45c294d..6784ea946 100644 --- a/Eigen/src/Core/Functors.h +++ b/Eigen/src/Core/Functors.h @@ -33,9 +33,9 @@ * \sa class CwiseBinaryOp, MatrixBase::operator+, class PartialRedux, MatrixBase::sum() */ template struct ei_scalar_sum_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return a + b; } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a + b; } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_padd(a,b); } }; template @@ -52,9 +52,9 @@ struct ei_functor_traits > { * \sa class CwiseBinaryOp, Cwise::operator*(), class PartialRedux, MatrixBase::redux() */ template struct ei_scalar_product_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return a * b; } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a * b; } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_pmul(a,b); } }; template @@ -71,9 +71,9 @@ struct ei_functor_traits > { * \sa class CwiseBinaryOp, MatrixBase::cwiseMin, class PartialRedux, MatrixBase::minCoeff() */ template struct ei_scalar_min_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return std::min(a, b); } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return std::min(a, b); } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_pmin(a,b); } }; template @@ -90,9 +90,9 @@ struct ei_functor_traits > { * \sa class CwiseBinaryOp, MatrixBase::cwiseMax, class PartialRedux, MatrixBase::maxCoeff() */ template struct ei_scalar_max_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return std::max(a, b); } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return std::max(a, b); } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_pmax(a,b); } }; template @@ -112,9 +112,9 @@ struct ei_functor_traits > { * \sa class CwiseBinaryOp, MatrixBase::operator- */ template struct ei_scalar_difference_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return a - b; } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a - b; } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_psub(a,b); } }; template @@ -131,9 +131,9 @@ struct ei_functor_traits > { * \sa class CwiseBinaryOp, Cwise::operator/() */ template struct ei_scalar_quotient_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a, const Scalar& b) const { return a / b; } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a, const Scalar& b) const { return a / b; } template - inline const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a, const PacketScalar& b) const { return ei_pdiv(a,b); } }; template @@ -155,7 +155,7 @@ struct ei_functor_traits > { * \sa class CwiseUnaryOp, MatrixBase::operator- */ template struct ei_scalar_opposite_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a) const { return -a; } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return -a; } }; template struct ei_functor_traits > @@ -168,7 +168,7 @@ struct ei_functor_traits > */ template struct ei_scalar_abs_op EIGEN_EMPTY_STRUCT { typedef typename NumTraits::Real result_type; - inline const result_type operator() (const Scalar& a) const { return ei_abs(a); } + EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return ei_abs(a); } }; template struct ei_functor_traits > @@ -186,9 +186,9 @@ struct ei_functor_traits > */ template struct ei_scalar_abs2_op EIGEN_EMPTY_STRUCT { typedef typename NumTraits::Real result_type; - inline const result_type operator() (const Scalar& a) const { return ei_abs2(a); } + EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return ei_abs2(a); } template - inline const PacketScalar packetOp(const PacketScalar& a) const + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a) const { return ei_pmul(a,a); } }; template @@ -201,9 +201,9 @@ struct ei_functor_traits > * \sa class CwiseUnaryOp, MatrixBase::conjugate() */ template struct ei_scalar_conjugate_op EIGEN_EMPTY_STRUCT { - inline const Scalar operator() (const Scalar& a) const { return ei_conj(a); } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return ei_conj(a); } template - inline const PacketScalar packetOp(const PacketScalar& a) const { return a; } + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a) const { return a; } }; template struct ei_functor_traits > @@ -222,7 +222,7 @@ struct ei_functor_traits > template struct ei_scalar_cast_op EIGEN_EMPTY_STRUCT { typedef NewType result_type; - inline const NewType operator() (const Scalar& a) const { return static_cast(a); } + EIGEN_STRONG_INLINE const NewType operator() (const Scalar& a) const { return static_cast(a); } }; template struct ei_functor_traits > @@ -236,7 +236,7 @@ struct ei_functor_traits > template struct ei_scalar_real_op EIGEN_EMPTY_STRUCT { typedef typename NumTraits::Real result_type; - inline result_type operator() (const Scalar& a) const { return ei_real(a); } + EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return ei_real(a); } }; template struct ei_functor_traits > @@ -250,7 +250,7 @@ struct ei_functor_traits > template struct ei_scalar_imag_op EIGEN_EMPTY_STRUCT { typedef typename NumTraits::Real result_type; - inline result_type operator() (const Scalar& a) const { return ei_imag(a); } + EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return ei_imag(a); } }; template struct ei_functor_traits > @@ -273,10 +273,10 @@ template struct ei_scalar_multiple_op { typedef typename ei_packet_traits::type PacketScalar; // FIXME default copy constructors seems bugged with std::complex<> - inline ei_scalar_multiple_op(const ei_scalar_multiple_op& other) : m_other(other.m_other) { } - inline ei_scalar_multiple_op(const Scalar& other) : m_other(other) { } - inline Scalar operator() (const Scalar& a) const { return a * m_other; } - inline const PacketScalar packetOp(const PacketScalar& a) const + EIGEN_STRONG_INLINE ei_scalar_multiple_op(const ei_scalar_multiple_op& other) : m_other(other.m_other) { } + EIGEN_STRONG_INLINE ei_scalar_multiple_op(const Scalar& other) : m_other(other) { } + EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a * m_other; } + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a) const { return ei_pmul(a, ei_pset1(m_other)); } const Scalar m_other; }; @@ -288,10 +288,10 @@ template struct ei_scalar_quotient1_impl { typedef typename ei_packet_traits::type PacketScalar; // FIXME default copy constructors seems bugged with std::complex<> - inline ei_scalar_quotient1_impl(const ei_scalar_quotient1_impl& other) : m_other(other.m_other) { } - inline ei_scalar_quotient1_impl(const Scalar& other) : m_other(static_cast(1) / other) {} - inline Scalar operator() (const Scalar& a) const { return a * m_other; } - inline const PacketScalar packetOp(const PacketScalar& a) const + EIGEN_STRONG_INLINE ei_scalar_quotient1_impl(const ei_scalar_quotient1_impl& other) : m_other(other.m_other) { } + EIGEN_STRONG_INLINE ei_scalar_quotient1_impl(const Scalar& other) : m_other(static_cast(1) / other) {} + EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a * m_other; } + EIGEN_STRONG_INLINE const PacketScalar packetOp(const PacketScalar& a) const { return ei_pmul(a, ei_pset1(m_other)); } const Scalar m_other; }; @@ -302,9 +302,9 @@ struct ei_functor_traits > template struct ei_scalar_quotient1_impl { // FIXME default copy constructors seems bugged with std::complex<> - inline ei_scalar_quotient1_impl(const ei_scalar_quotient1_impl& other) : m_other(other.m_other) { } - inline ei_scalar_quotient1_impl(const Scalar& other) : m_other(other) {} - inline Scalar operator() (const Scalar& a) const { return a / m_other; } + EIGEN_STRONG_INLINE ei_scalar_quotient1_impl(const ei_scalar_quotient1_impl& other) : m_other(other.m_other) { } + EIGEN_STRONG_INLINE ei_scalar_quotient1_impl(const Scalar& other) : m_other(other) {} + EIGEN_STRONG_INLINE Scalar operator() (const Scalar& a) const { return a / m_other; } const Scalar m_other; }; template @@ -321,7 +321,7 @@ struct ei_functor_traits > */ template struct ei_scalar_quotient1_op : ei_scalar_quotient1_impl::HasFloatingPoint > { - inline ei_scalar_quotient1_op(const Scalar& other) + EIGEN_STRONG_INLINE ei_scalar_quotient1_op(const Scalar& other) : ei_scalar_quotient1_impl::HasFloatingPoint >(other) {} }; @@ -330,10 +330,10 @@ struct ei_scalar_quotient1_op : ei_scalar_quotient1_impl struct ei_scalar_constant_op { typedef typename ei_packet_traits::type PacketScalar; - inline ei_scalar_constant_op(const ei_scalar_constant_op& other) : m_other(other.m_other) { } - inline ei_scalar_constant_op(const Scalar& other) : m_other(other) { } - inline const Scalar operator() (int, int = 0) const { return m_other; } - inline const PacketScalar packetOp() const { return ei_pset1(m_other); } + EIGEN_STRONG_INLINE ei_scalar_constant_op(const ei_scalar_constant_op& other) : m_other(other.m_other) { } + EIGEN_STRONG_INLINE ei_scalar_constant_op(const Scalar& other) : m_other(other) { } + EIGEN_STRONG_INLINE const Scalar operator() (int, int = 0) const { return m_other; } + EIGEN_STRONG_INLINE const PacketScalar packetOp() const { return ei_pset1(m_other); } const Scalar m_other; }; template @@ -341,8 +341,8 @@ struct ei_functor_traits > { enum { Cost = 1, PacketAccess = ei_packet_traits::size>1, IsRepeatable = true }; }; template struct ei_scalar_identity_op EIGEN_EMPTY_STRUCT { - inline ei_scalar_identity_op(void) {} - inline const Scalar operator() (int row, int col) const { return row==col ? Scalar(1) : Scalar(0); } + EIGEN_STRONG_INLINE ei_scalar_identity_op(void) {} + EIGEN_STRONG_INLINE const Scalar operator() (int row, int col) const { return row==col ? Scalar(1) : Scalar(0); } }; template struct ei_functor_traits > diff --git a/Eigen/src/Core/Matrix.h b/Eigen/src/Core/Matrix.h index 3cde1e28b..39b3da6fb 100644 --- a/Eigen/src/Core/Matrix.h +++ b/Eigen/src/Core/Matrix.h @@ -138,10 +138,10 @@ class Matrix public: - inline int rows() const { return m_storage.rows(); } - inline int cols() const { return m_storage.cols(); } + EIGEN_STRONG_INLINE int rows() const { return m_storage.rows(); } + EIGEN_STRONG_INLINE int cols() const { return m_storage.cols(); } - inline int stride(void) const + EIGEN_STRONG_INLINE int stride(void) const { if(Flags & RowMajorBit) return m_storage.cols(); @@ -149,7 +149,7 @@ class Matrix return m_storage.rows(); } - inline const Scalar& coeff(int row, int col) const + EIGEN_STRONG_INLINE const Scalar& coeff(int row, int col) const { if(Flags & RowMajorBit) return m_storage.data()[col + row * m_storage.cols()]; @@ -157,12 +157,12 @@ class Matrix return m_storage.data()[row + col * m_storage.rows()]; } - inline const Scalar& coeff(int index) const + EIGEN_STRONG_INLINE const Scalar& coeff(int index) const { return m_storage.data()[index]; } - inline Scalar& coeffRef(int row, int col) + EIGEN_STRONG_INLINE Scalar& coeffRef(int row, int col) { if(Flags & RowMajorBit) return m_storage.data()[col + row * m_storage.cols()]; @@ -170,13 +170,13 @@ class Matrix return m_storage.data()[row + col * m_storage.rows()]; } - inline Scalar& coeffRef(int index) + EIGEN_STRONG_INLINE Scalar& coeffRef(int index) { return m_storage.data()[index]; } template - inline PacketScalar packet(int row, int col) const + EIGEN_STRONG_INLINE PacketScalar packet(int row, int col) const { return ei_ploadt (m_storage.data() + (Flags & RowMajorBit @@ -185,13 +185,13 @@ class Matrix } template - inline PacketScalar packet(int index) const + EIGEN_STRONG_INLINE PacketScalar packet(int index) const { return ei_ploadt(m_storage.data() + index); } template - inline void writePacket(int row, int col, const PacketScalar& x) + EIGEN_STRONG_INLINE void writePacket(int row, int col, const PacketScalar& x) { ei_pstoret (m_storage.data() + (Flags & RowMajorBit @@ -200,17 +200,17 @@ class Matrix } template - inline void writePacket(int index, const PacketScalar& x) + EIGEN_STRONG_INLINE void writePacket(int index, const PacketScalar& x) { ei_pstoret(m_storage.data() + index, x); } /** \returns a const pointer to the data array of this matrix */ - inline const Scalar *data() const + EIGEN_STRONG_INLINE const Scalar *data() const { return m_storage.data(); } /** \returns a pointer to the data array of this matrix */ - inline Scalar *data() + EIGEN_STRONG_INLINE Scalar *data() { return m_storage.data(); } /** Resizes \c *this to a \a rows x \a cols matrix. @@ -260,7 +260,7 @@ class Matrix * \sa set() */ template - inline Matrix& operator=(const MatrixBase& other) + EIGEN_STRONG_INLINE Matrix& operator=(const MatrixBase& other) { ei_assert(m_storage.data()!=0 && "you cannot use operator= with a non initialized matrix (instead use set()"); return Base::operator=(other.derived()); @@ -297,7 +297,7 @@ class Matrix /** This is a special case of the templated operator=. Its purpose is to * prevent a default operator= from hiding the templated operator=. */ - inline Matrix& operator=(const Matrix& other) + EIGEN_STRONG_INLINE Matrix& operator=(const Matrix& other) { return operator=(other); } @@ -332,7 +332,7 @@ class Matrix * * \sa resize(int,int), set() */ - inline explicit Matrix() : m_storage() + EIGEN_STRONG_INLINE explicit Matrix() : m_storage() { ei_assert(RowsAtCompileTime > 0 && ColsAtCompileTime > 0); } @@ -343,7 +343,7 @@ class Matrix * it is redundant to pass the dimension here, so it makes more sense to use the default * constructor Matrix() instead. */ - inline explicit Matrix(int dim) + EIGEN_STRONG_INLINE explicit Matrix(int dim) : m_storage(dim, RowsAtCompileTime == 1 ? 1 : dim, ColsAtCompileTime == 1 ? 1 : dim) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Matrix) @@ -361,7 +361,7 @@ class Matrix * it is redundant to pass these parameters, so one should use the default constructor * Matrix() instead. */ - inline Matrix(int x, int y) : m_storage(x*y, x, y) + EIGEN_STRONG_INLINE Matrix(int x, int y) : m_storage(x*y, x, y) { if((RowsAtCompileTime == 1 && ColsAtCompileTime == 2) || (RowsAtCompileTime == 2 && ColsAtCompileTime == 1)) @@ -376,21 +376,21 @@ class Matrix } } /** constructs an initialized 2D vector with given coefficients */ - inline Matrix(const float& x, const float& y) + EIGEN_STRONG_INLINE Matrix(const float& x, const float& y) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 2) m_storage.data()[0] = x; m_storage.data()[1] = y; } /** constructs an initialized 2D vector with given coefficients */ - inline Matrix(const double& x, const double& y) + EIGEN_STRONG_INLINE Matrix(const double& x, const double& y) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 2) m_storage.data()[0] = x; m_storage.data()[1] = y; } /** constructs an initialized 3D vector with given coefficients */ - inline Matrix(const Scalar& x, const Scalar& y, const Scalar& z) + EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 3) m_storage.data()[0] = x; @@ -398,7 +398,7 @@ class Matrix m_storage.data()[2] = z; } /** constructs an initialized 4D vector with given coefficients */ - inline Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) + EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 4) m_storage.data()[0] = x; @@ -411,14 +411,14 @@ class Matrix /** Constructor copying the value of the expression \a other */ template - inline Matrix(const MatrixBase& other) + EIGEN_STRONG_INLINE Matrix(const MatrixBase& other) : m_storage(other.rows() * other.cols(), other.rows(), other.cols()) { ei_assign_selector::run(*this, other.derived()); //Base::operator=(other.derived()); } /** Copy constructor */ - inline Matrix(const Matrix& other) + EIGEN_STRONG_INLINE Matrix(const Matrix& other) : Base(), m_storage(other.rows() * other.cols(), other.rows(), other.cols()) { Base::lazyAssign(other); diff --git a/Eigen/src/Core/MatrixBase.h b/Eigen/src/Core/MatrixBase.h index 0c3a04ff4..a9a08dc28 100644 --- a/Eigen/src/Core/MatrixBase.h +++ b/Eigen/src/Core/MatrixBase.h @@ -495,15 +495,8 @@ template class MatrixBase * Notice that in the case of a plain matrix or vector (not an expression) this function just returns * a const reference, in order to avoid a useless copy. */ - #ifdef _MSC_VER - inline // MSVC 2008 can't force-inline this method and emits a warning, so do just 'inline' - #else - EIGEN_ALWAYS_INLINE - #endif - const typename ei_eval::type eval() const - { - return typename ei_eval::type(derived()); - } + EIGEN_STRONG_INLINE const typename ei_eval::type eval() const + { return typename ei_eval::type(derived()); } template void swap(const MatrixBase& other); diff --git a/Eigen/src/Core/Product.h b/Eigen/src/Core/Product.h index 385ed7505..7df7ac67b 100644 --- a/Eigen/src/Core/Product.h +++ b/Eigen/src/Core/Product.h @@ -151,7 +151,8 @@ struct ei_traits > Flags = ((unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & RemovedBits) | EvalBeforeAssigningBit | EvalBeforeNestingBit - | (CanVectorizeLhs || CanVectorizeRhs ? PacketAccessBit : 0), + | (CanVectorizeLhs || CanVectorizeRhs ? PacketAccessBit : 0) + | (LhsFlags & RhsFlags & AlignedBit), CoeffReadCost = InnerSize == Dynamic ? Dynamic : InnerSize * (NumTraits::MulCost + LhsCoeffReadCost + RhsCoeffReadCost) @@ -214,17 +215,17 @@ template class Product /** \internal * \returns whether it is worth it to use the cache friendly product. */ - inline bool _useCacheFriendlyProduct() const + EIGEN_STRONG_INLINE bool _useCacheFriendlyProduct() const { return m_lhs.cols()>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD && ( rows()>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD || cols()>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD); } - inline int rows() const { return m_lhs.rows(); } - inline int cols() const { return m_rhs.cols(); } + EIGEN_STRONG_INLINE int rows() const { return m_lhs.rows(); } + EIGEN_STRONG_INLINE int cols() const { return m_rhs.cols(); } - const Scalar coeff(int row, int col) const + EIGEN_STRONG_INLINE const Scalar coeff(int row, int col) const { Scalar res; ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res); @@ -234,7 +235,7 @@ template class Product /* Allow index-based non-packet access. It is impossible though to allow index-based packed access, * which is why we don't set the LinearAccessBit. */ - const Scalar coeff(int index) const + EIGEN_STRONG_INLINE const Scalar coeff(int index) const { Scalar res; const int row = RowsAtCompileTime == 1 ? 0 : index; @@ -244,7 +245,7 @@ template class Product } template - const PacketScalar packet(int row, int col) const + EIGEN_STRONG_INLINE const PacketScalar packet(int row, int col) const { PacketScalar res; ei_product_packet_impl class Product return res; } - inline const _LhsNested& lhs() const { return m_lhs; } - inline const _RhsNested& rhs() const { return m_rhs; } + EIGEN_STRONG_INLINE const _LhsNested& lhs() const { return m_lhs; } + EIGEN_STRONG_INLINE const _RhsNested& rhs() const { return m_rhs; } protected: const LhsNested m_lhs; @@ -314,7 +315,7 @@ MatrixBase::operator*=(const MatrixBase &other) template struct ei_product_coeff_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { ei_product_coeff_impl::run(row, col, lhs, rhs, res); res += lhs.coeff(row, Index) * rhs.coeff(Index, col); @@ -324,7 +325,7 @@ struct ei_product_coeff_impl template struct ei_product_coeff_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { res = lhs.coeff(row, 0) * rhs.coeff(0, col); } @@ -333,7 +334,7 @@ struct ei_product_coeff_impl template struct ei_product_coeff_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar& res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar& res) { ei_assert(lhs.cols()>0 && "you are using a non initialized matrix"); res = lhs.coeff(row, 0) * rhs.coeff(0, col); @@ -346,7 +347,7 @@ struct ei_product_coeff_impl template struct ei_product_coeff_impl { - inline static void run(int, int, const Lhs&, const Rhs&, typename Lhs::Scalar&) {} + EIGEN_STRONG_INLINE static void run(int, int, const Lhs&, const Rhs&, typename Lhs::Scalar&) {} }; /******************************************* @@ -357,7 +358,7 @@ template struct ei_product_coeff_vectorized_unroller { enum { PacketSize = ei_packet_traits::size }; - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres) { ei_product_coeff_vectorized_unroller::run(row, col, lhs, rhs, pres); pres = ei_padd(pres, ei_pmul( lhs.template packet(row, Index) , rhs.template packet(Index, col) )); @@ -367,7 +368,7 @@ struct ei_product_coeff_vectorized_unroller template struct ei_product_coeff_vectorized_unroller<0, Lhs, Rhs, PacketScalar> { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::PacketScalar &pres) { pres = ei_pmul(lhs.template packet(row, 0) , rhs.template packet(0, col)); } @@ -378,7 +379,7 @@ struct ei_product_coeff_impl { typedef typename Lhs::PacketScalar PacketScalar; enum { PacketSize = ei_packet_traits::size }; - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { PacketScalar pres; ei_product_coeff_vectorized_unroller::run(row, col, lhs, rhs, pres); @@ -390,7 +391,7 @@ struct ei_product_coeff_impl template struct ei_product_coeff_vectorized_dyn_selector { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { res = ei_dot_impl< Block::ColsAtCompileTime>, @@ -404,7 +405,7 @@ struct ei_product_coeff_vectorized_dyn_selector template struct ei_product_coeff_vectorized_dyn_selector { - inline static void run(int /*row*/, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int /*row*/, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { res = ei_dot_impl< Lhs, @@ -416,7 +417,7 @@ struct ei_product_coeff_vectorized_dyn_selector template struct ei_product_coeff_vectorized_dyn_selector { - inline static void run(int row, int /*col*/, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int /*col*/, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { res = ei_dot_impl< Block::ColsAtCompileTime>, @@ -428,7 +429,7 @@ struct ei_product_coeff_vectorized_dyn_selector template struct ei_product_coeff_vectorized_dyn_selector { - inline static void run(int /*row*/, int /*col*/, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int /*row*/, int /*col*/, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { res = ei_dot_impl< Lhs, @@ -440,7 +441,7 @@ struct ei_product_coeff_vectorized_dyn_selector template struct ei_product_coeff_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, typename Lhs::Scalar &res) { ei_product_coeff_vectorized_dyn_selector::run(row, col, lhs, rhs, res); } @@ -453,7 +454,7 @@ struct ei_product_coeff_impl template struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) { ei_product_packet_impl::run(row, col, lhs, rhs, res); res = ei_pmadd(ei_pset1(lhs.coeff(row, Index)), rhs.template packet(Index, col), res); @@ -463,7 +464,7 @@ struct ei_product_packet_impl template struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) { ei_product_packet_impl::run(row, col, lhs, rhs, res); res = ei_pmadd(lhs.template packet(row, Index), ei_pset1(rhs.coeff(Index, col)), res); @@ -473,7 +474,7 @@ struct ei_product_packet_impl template struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) { res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet(0, col)); } @@ -482,7 +483,7 @@ struct ei_product_packet_impl template struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar &res) { res = ei_pmul(lhs.template packet(row, 0), ei_pset1(rhs.coeff(0, col))); } @@ -491,7 +492,7 @@ struct ei_product_packet_impl template struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res) { ei_assert(lhs.cols()>0 && "you are using a non initialized matrix"); res = ei_pmul(ei_pset1(lhs.coeff(row, 0)),rhs.template packet(0, col)); @@ -503,7 +504,7 @@ struct ei_product_packet_impl struct ei_product_packet_impl { - inline static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res) + EIGEN_STRONG_INLINE static void run(int row, int col, const Lhs& lhs, const Rhs& rhs, PacketScalar& res) { ei_assert(lhs.cols()>0 && "you are using a non initialized matrix"); res = ei_pmul(lhs.template packet(row, 0), ei_pset1(rhs.coeff(0, col))); diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index c6740f414..8a49dcaae 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h @@ -37,17 +37,17 @@ template<> struct ei_unpacket_traits<__m128> { typedef float type; enum {size= template<> struct ei_unpacket_traits<__m128d> { typedef double type; enum {size=2}; }; template<> struct ei_unpacket_traits<__m128i> { typedef int type; enum {size=4}; }; -template<> inline __m128 ei_padd<__m128>(const __m128& a, const __m128& b) { return _mm_add_ps(a,b); } -template<> inline __m128d ei_padd<__m128d>(const __m128d& a, const __m128d& b) { return _mm_add_pd(a,b); } -template<> inline __m128i ei_padd<__m128i>(const __m128i& a, const __m128i& b) { return _mm_add_epi32(a,b); } +template<> EIGEN_STRONG_INLINE __m128 ei_padd<__m128>(const __m128& a, const __m128& b) { return _mm_add_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_padd<__m128d>(const __m128d& a, const __m128d& b) { return _mm_add_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128i ei_padd<__m128i>(const __m128i& a, const __m128i& b) { return _mm_add_epi32(a,b); } -template<> inline __m128 ei_psub<__m128>(const __m128& a, const __m128& b) { return _mm_sub_ps(a,b); } -template<> inline __m128d ei_psub<__m128d>(const __m128d& a, const __m128d& b) { return _mm_sub_pd(a,b); } -template<> inline __m128i ei_psub<__m128i>(const __m128i& a, const __m128i& b) { return _mm_sub_epi32(a,b); } +template<> EIGEN_STRONG_INLINE __m128 ei_psub<__m128>(const __m128& a, const __m128& b) { return _mm_sub_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_psub<__m128d>(const __m128d& a, const __m128d& b) { return _mm_sub_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128i ei_psub<__m128i>(const __m128i& a, const __m128i& b) { return _mm_sub_epi32(a,b); } -template<> inline __m128 ei_pmul<__m128>(const __m128& a, const __m128& b) { return _mm_mul_ps(a,b); } -template<> inline __m128d ei_pmul<__m128d>(const __m128d& a, const __m128d& b) { return _mm_mul_pd(a,b); } -template<> inline __m128i ei_pmul<__m128i>(const __m128i& a, const __m128i& b) +template<> EIGEN_STRONG_INLINE __m128 ei_pmul<__m128>(const __m128& a, const __m128& b) { return _mm_mul_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_pmul<__m128d>(const __m128d& a, const __m128d& b) { return _mm_mul_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128i ei_pmul<__m128i>(const __m128i& a, const __m128i& b) { return _mm_or_si128( _mm_and_si128( @@ -59,108 +59,108 @@ template<> inline __m128i ei_pmul<__m128i>(const __m128i& a, const __m128i& b) _mm_setr_epi32(0xffffffff,0,0xffffffff,0)), 4)); } -template<> inline __m128 ei_pdiv<__m128>(const __m128& a, const __m128& b) { return _mm_div_ps(a,b); } -template<> inline __m128d ei_pdiv<__m128d>(const __m128d& a, const __m128d& b) { return _mm_div_pd(a,b); } -template<> inline __m128i ei_pdiv<__m128i>(const __m128i& /*a*/, const __m128i& /*b*/) +template<> EIGEN_STRONG_INLINE __m128 ei_pdiv<__m128>(const __m128& a, const __m128& b) { return _mm_div_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_pdiv<__m128d>(const __m128d& a, const __m128d& b) { return _mm_div_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128i ei_pdiv<__m128i>(const __m128i& /*a*/, const __m128i& /*b*/) { ei_assert(false && "packet integer division are not supported by SSE"); __m128i dummy; return dummy; } // for some weird raisons, it has to be overloaded for packet integer -template<> inline __m128i ei_pmadd(const __m128i& a, const __m128i& b, const __m128i& c) { return ei_padd(ei_pmul(a,b), c); } +template<> EIGEN_STRONG_INLINE __m128i ei_pmadd(const __m128i& a, const __m128i& b, const __m128i& c) { return ei_padd(ei_pmul(a,b), c); } -template<> inline __m128 ei_pmin<__m128>(const __m128& a, const __m128& b) { return _mm_min_ps(a,b); } -template<> inline __m128d ei_pmin<__m128d>(const __m128d& a, const __m128d& b) { return _mm_min_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128 ei_pmin<__m128>(const __m128& a, const __m128& b) { return _mm_min_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_pmin<__m128d>(const __m128d& a, const __m128d& b) { return _mm_min_pd(a,b); } // FIXME this vectorized min operator is likely to be slower than the standard one -template<> inline __m128i ei_pmin<__m128i>(const __m128i& a, const __m128i& b) +template<> EIGEN_STRONG_INLINE __m128i ei_pmin<__m128i>(const __m128i& a, const __m128i& b) { __m128i mask = _mm_cmplt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); } -template<> inline __m128 ei_pmax<__m128>(const __m128& a, const __m128& b) { return _mm_max_ps(a,b); } -template<> inline __m128d ei_pmax<__m128d>(const __m128d& a, const __m128d& b) { return _mm_max_pd(a,b); } +template<> EIGEN_STRONG_INLINE __m128 ei_pmax<__m128>(const __m128& a, const __m128& b) { return _mm_max_ps(a,b); } +template<> EIGEN_STRONG_INLINE __m128d ei_pmax<__m128d>(const __m128d& a, const __m128d& b) { return _mm_max_pd(a,b); } // FIXME this vectorized max operator is likely to be slower than the standard one -template<> inline __m128i ei_pmax<__m128i>(const __m128i& a, const __m128i& b) +template<> EIGEN_STRONG_INLINE __m128i ei_pmax<__m128i>(const __m128i& a, const __m128i& b) { __m128i mask = _mm_cmpgt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); } -template<> inline __m128 ei_pload(const float* from) { return _mm_load_ps(from); } -template<> inline __m128d ei_pload(const double* from) { return _mm_load_pd(from); } -template<> inline __m128i ei_pload(const int* from) { return _mm_load_si128(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE __m128 ei_pload(const float* from) { return _mm_load_ps(from); } +template<> EIGEN_STRONG_INLINE __m128d ei_pload(const double* from) { return _mm_load_pd(from); } +template<> EIGEN_STRONG_INLINE __m128i ei_pload(const int* from) { return _mm_load_si128(reinterpret_cast(from)); } -template<> inline __m128 ei_ploadu(const float* from) { return _mm_loadu_ps(from); } -// template<> inline __m128 ei_ploadu(const float* from) { +template<> EIGEN_STRONG_INLINE __m128 ei_ploadu(const float* from) { return _mm_loadu_ps(from); } +// template<> EIGEN_STRONG_INLINE __m128 ei_ploadu(const float* from) { // if (size_t(from)&0xF) // return _mm_loadu_ps(from); // else // return _mm_loadu_ps(from); // } -template<> inline __m128d ei_ploadu(const double* from) { return _mm_loadu_pd(from); } -template<> inline __m128i ei_ploadu(const int* from) { return _mm_loadu_si128(reinterpret_cast(from)); } +template<> EIGEN_STRONG_INLINE __m128d ei_ploadu(const double* from) { return _mm_loadu_pd(from); } +template<> EIGEN_STRONG_INLINE __m128i ei_ploadu(const int* from) { return _mm_loadu_si128(reinterpret_cast(from)); } -template<> inline __m128 ei_pset1(const float& from) { return _mm_set1_ps(from); } -template<> inline __m128d ei_pset1(const double& from) { return _mm_set1_pd(from); } -template<> inline __m128i ei_pset1(const int& from) { return _mm_set1_epi32(from); } +template<> EIGEN_STRONG_INLINE __m128 ei_pset1(const float& from) { return _mm_set1_ps(from); } +template<> EIGEN_STRONG_INLINE __m128d ei_pset1(const double& from) { return _mm_set1_pd(from); } +template<> EIGEN_STRONG_INLINE __m128i ei_pset1(const int& from) { return _mm_set1_epi32(from); } -template<> inline void ei_pstore(float* to, const __m128& from) { _mm_store_ps(to, from); } -template<> inline void ei_pstore(double* to, const __m128d& from) { _mm_store_pd(to, from); } -template<> inline void ei_pstore(int* to, const __m128i& from) { _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } +template<> EIGEN_STRONG_INLINE void ei_pstore(float* to, const __m128& from) { _mm_store_ps(to, from); } +template<> EIGEN_STRONG_INLINE void ei_pstore(double* to, const __m128d& from) { _mm_store_pd(to, from); } +template<> EIGEN_STRONG_INLINE void ei_pstore(int* to, const __m128i& from) { _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } -template<> inline void ei_pstoreu(float* to, const __m128& from) { _mm_storeu_ps(to, from); } -template<> inline void ei_pstoreu(double* to, const __m128d& from) { _mm_storeu_pd(to, from); } -template<> inline void ei_pstoreu(int* to, const __m128i& from) { _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } +template<> EIGEN_STRONG_INLINE void ei_pstoreu(float* to, const __m128& from) { _mm_storeu_ps(to, from); } +template<> EIGEN_STRONG_INLINE void ei_pstoreu(double* to, const __m128d& from) { _mm_storeu_pd(to, from); } +template<> EIGEN_STRONG_INLINE void ei_pstoreu(int* to, const __m128i& from) { _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } -template<> inline float ei_pfirst<__m128>(const __m128& a) { return _mm_cvtss_f32(a); } -template<> inline double ei_pfirst<__m128d>(const __m128d& a) { return _mm_cvtsd_f64(a); } -template<> inline int ei_pfirst<__m128i>(const __m128i& a) { return _mm_cvtsi128_si32(a); } +template<> EIGEN_STRONG_INLINE float ei_pfirst<__m128>(const __m128& a) { return _mm_cvtss_f32(a); } +template<> EIGEN_STRONG_INLINE double ei_pfirst<__m128d>(const __m128d& a) { return _mm_cvtsd_f64(a); } +template<> EIGEN_STRONG_INLINE int ei_pfirst<__m128i>(const __m128i& a) { return _mm_cvtsi128_si32(a); } #ifdef __SSE3__ // TODO implement SSE2 versions as well as integer versions -template<> inline __m128 ei_preduxp<__m128>(const __m128* vecs) +template<> EIGEN_STRONG_INLINE __m128 ei_preduxp<__m128>(const __m128* vecs) { return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3])); } -template<> inline __m128d ei_preduxp<__m128d>(const __m128d* vecs) +template<> EIGEN_STRONG_INLINE __m128d ei_preduxp<__m128d>(const __m128d* vecs) { return _mm_hadd_pd(vecs[0], vecs[1]); } // SSSE3 version: -// inline __m128i ei_preduxp(const __m128i* vecs) +// EIGEN_STRONG_INLINE __m128i ei_preduxp(const __m128i* vecs) // { // return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3])); // } -template<> inline float ei_predux<__m128>(const __m128& a) +template<> EIGEN_STRONG_INLINE float ei_predux<__m128>(const __m128& a) { __m128 tmp0 = _mm_hadd_ps(a,a); return ei_pfirst(_mm_hadd_ps(tmp0, tmp0)); } -template<> inline double ei_predux<__m128d>(const __m128d& a) { return ei_pfirst(_mm_hadd_pd(a, a)); } +template<> EIGEN_STRONG_INLINE double ei_predux<__m128d>(const __m128d& a) { return ei_pfirst(_mm_hadd_pd(a, a)); } // SSSE3 version: -// inline float ei_predux(const __m128i& a) +// EIGEN_STRONG_INLINE float ei_predux(const __m128i& a) // { // __m128i tmp0 = _mm_hadd_epi32(a,a); // return ei_pfirst(_mm_hadd_epi32(tmp0, tmp0)); // } #else // SSE2 versions -template<> inline float ei_predux<__m128>(const __m128& a) +template<> EIGEN_STRONG_INLINE float ei_predux<__m128>(const __m128& a) { __m128 tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); return ei_pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); } -template<> inline double ei_predux<__m128d>(const __m128d& a) +template<> EIGEN_STRONG_INLINE double ei_predux<__m128d>(const __m128d& a) { return ei_pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); } -template<> inline __m128 ei_preduxp<__m128>(const __m128* vecs) +template<> EIGEN_STRONG_INLINE __m128 ei_preduxp<__m128>(const __m128* vecs) { __m128 tmp0, tmp1, tmp2; tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]); @@ -174,19 +174,19 @@ template<> inline __m128 ei_preduxp<__m128>(const __m128* vecs) return _mm_add_ps(tmp0, tmp2); } -template<> inline __m128d ei_preduxp<__m128d>(const __m128d* vecs) +template<> EIGEN_STRONG_INLINE __m128d ei_preduxp<__m128d>(const __m128d* vecs) { return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1])); } #endif // SSE3 -template<> inline int ei_predux<__m128i>(const __m128i& a) +template<> EIGEN_STRONG_INLINE int ei_predux<__m128i>(const __m128i& a) { __m128i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a)); return ei_pfirst(tmp) + ei_pfirst(_mm_shuffle_epi32(tmp, 1)); } -template<> inline __m128i ei_preduxp<__m128i>(const __m128i* vecs) +template<> EIGEN_STRONG_INLINE __m128i ei_preduxp<__m128i>(const __m128i* vecs) { __m128i tmp0, tmp1, tmp2; tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]); @@ -201,13 +201,13 @@ template<> inline __m128i ei_preduxp<__m128i>(const __m128i* vecs) } #if (defined __GNUC__) -// template <> inline __m128 ei_pmadd(const __m128& a, const __m128& b, const __m128& c) +// template <> EIGEN_STRONG_INLINE __m128 ei_pmadd(const __m128& a, const __m128& b, const __m128& c) // { // __m128 res = b; // asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c)); // return res; // } -// inline __m128i _mm_alignr_epi8(const __m128i& a, const __m128i& b, const int i) +// EIGEN_STRONG_INLINE __m128i _mm_alignr_epi8(const __m128i& a, const __m128i& b, const int i) // { // __m128i res = a; // asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i)); @@ -220,7 +220,7 @@ template<> inline __m128i ei_preduxp<__m128i>(const __m128i* vecs) template struct ei_palign_impl { - inline static void run(__m128& first, const __m128& second) + EIGEN_STRONG_INLINE static void run(__m128& first, const __m128& second) { if (Offset!=0) first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4)); @@ -230,7 +230,7 @@ struct ei_palign_impl template struct ei_palign_impl { - inline static void run(__m128i& first, const __m128i& second) + EIGEN_STRONG_INLINE static void run(__m128i& first, const __m128i& second) { if (Offset!=0) first = _mm_alignr_epi8(second,first, Offset*4); @@ -240,7 +240,7 @@ struct ei_palign_impl template struct ei_palign_impl { - inline static void run(__m128d& first, const __m128d& second) + EIGEN_STRONG_INLINE static void run(__m128d& first, const __m128d& second) { if (Offset==1) first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8)); @@ -251,7 +251,7 @@ struct ei_palign_impl template struct ei_palign_impl { - inline static void run(__m128& first, const __m128& second) + EIGEN_STRONG_INLINE static void run(__m128& first, const __m128& second) { if (Offset==1) { @@ -274,7 +274,7 @@ struct ei_palign_impl template struct ei_palign_impl { - inline static void run(__m128i& first, const __m128i& second) + EIGEN_STRONG_INLINE static void run(__m128i& first, const __m128i& second) { if (Offset==1) { @@ -297,7 +297,7 @@ struct ei_palign_impl template struct ei_palign_impl { - inline static void run(__m128d& first, const __m128d& second) + EIGEN_STRONG_INLINE static void run(__m128d& first, const __m128d& second) { if (Offset==1) { diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index dc18a425c..422bc7bd1 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h @@ -84,16 +84,23 @@ using Eigen::ei_cos; #define EIGEN_ONLY_USED_FOR_DEBUG(x) #endif +// EIGEN_ALWAYS_INLINE_ATTRIB should be use in the declaration of function +// which should be inlined even in debug mode. // FIXME with the always_inline attribute, // gcc 3.4.x reports the following compilation error: // Eval.h:91: sorry, unimplemented: inlining failed in call to 'const Eigen::Eval Eigen::MatrixBase::eval() const' // : function body not available #if EIGEN_GNUC_AT_LEAST(4,0) -#define EIGEN_ALWAYS_INLINE __attribute__((always_inline)) inline -#elif (defined _MSC_VER) -#define EIGEN_ALWAYS_INLINE __forceinline +#define EIGEN_ALWAYS_INLINE_ATTRIB __attribute__((always_inline)) #else -#define EIGEN_ALWAYS_INLINE inline +#define EIGEN_ALWAYS_INLINE_ATTRIB +#endif + +// EIGEN_FORCE_INLINE means "inline as much as possible" +#if (defined _MSC_VER) +#define EIGEN_STRONG_INLINE __forceinline +#else +#define EIGEN_STRONG_INLINE inline #endif #if (defined __GNUC__) @@ -142,18 +149,18 @@ using Eigen::ei_cos; #define EIGEN_INHERIT_ASSIGNMENT_OPERATOR(Derived, Op) \ template \ -Derived& operator Op(const Eigen::MatrixBase& other) \ +EIGEN_STRONG_INLINE Derived& operator Op(const Eigen::MatrixBase& other) \ { \ return Eigen::MatrixBase::operator Op(other.derived()); \ } \ -Derived& operator Op(const Derived& other) \ +EIGEN_STRONG_INLINE Derived& operator Op(const Derived& other) \ { \ return Eigen::MatrixBase::operator Op(other); \ } #define EIGEN_INHERIT_SCALAR_ASSIGNMENT_OPERATOR(Derived, Op) \ template \ -Derived& operator Op(const Other& scalar) \ +EIGEN_STRONG_INLINE Derived& operator Op(const Other& scalar) \ { \ return Eigen::MatrixBase::operator Op(scalar); \ } diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index b09e11bd4..d7c2a0092 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h @@ -74,18 +74,22 @@ inline T* ei_aligned_malloc(size_t size) #ifdef EIGEN_VECTORIZE if(ei_packet_traits::size>1 || ei_force_aligned_malloc::ret) { - #ifdef _MSC_VER - result = static_cast(_aligned_malloc(size*sizeof(T), 16)); - #ifdef EIGEN_EXCEPTIONS - const int failed = (result == 0); - #endif - #else // not MSVC + #ifdef __linux #ifdef EIGEN_EXCEPTIONS const int failed = #endif posix_memalign(reinterpret_cast(&result), 16, size*sizeof(T)); + #else + #ifdef _MSC_VER + result = static_cast(_aligned_malloc(size*sizeof(T), 16)); + #else + result = static_cast(_mm_malloc(size*sizeof(T),16)); + #endif + + #ifdef EIGEN_EXCEPTIONS + const int failed = (result == 0); + #endif #endif - #ifdef EIGEN_EXCEPTIONS if(failed) throw std::bad_alloc(); @@ -107,13 +111,15 @@ template inline void ei_aligned_free(T* ptr) { #ifdef EIGEN_VECTORIZE - if (ei_packet_traits::size>1 || ei_force_aligned_malloc::ret) - #ifdef _MSC_VER - _aligned_free(ptr); - #else - free(ptr); - #endif - else + if (ei_packet_traits::size>1 || ei_force_aligned_malloc::ret) + #if defined(__linux) + free(ptr); + #elif defined(_MSC_VER) + _aligned_free(ptr); + #else + _mm_free(ptr); + #endif + else #endif delete[] ptr; }