diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 3d78fd8be..879b0dbd5 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h @@ -217,7 +217,7 @@ struct evaluator> : evaluator_base { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return const_cast(m_d.data)[index]; } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { if (IsRowMajor) return ploadt(m_d.data + row * m_d.outerStride() + col); else @@ -225,12 +225,12 @@ struct evaluator> : evaluator_base { } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return ploadt(m_d.data + index); } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { if (IsRowMajor) return pstoret(const_cast(m_d.data) + row * m_d.outerStride() + col, x); else @@ -238,7 +238,7 @@ struct evaluator> : evaluator_base { } template - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { return pstoret(const_cast(m_d.data) + index, x); } @@ -296,22 +296,22 @@ struct unary_evaluator, IndexBased> : evaluator_base - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(col, row); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet(index); } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket(col, row, x); } template - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket(index, x); } @@ -490,12 +490,12 @@ struct evaluator> } template - EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const { return m_wrapper.template packetOp(m_functor, row, col); } template - EIGEN_STRONG_INLINE PacketType packet(IndexType index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(IndexType index) const { return m_wrapper.template packetOp(m_functor, index); } @@ -534,12 +534,12 @@ struct unary_evaluator, IndexBased> : evaluator_b } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.argImpl.template packet(row, col)); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.argImpl.template packet(index)); } @@ -627,7 +627,7 @@ struct unary_evaluator, ArgType>, In } template - EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const { constexpr int PacketSize = unpacket_traits::size; Index actualRow = IsRowMajor ? row : row + (offset * PacketSize); Index actualCol = IsRowMajor ? col + (offset * PacketSize) : col; @@ -635,7 +635,7 @@ struct unary_evaluator, ArgType>, In return m_argImpl.template packet(actualRow, actualCol); } template - EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const { constexpr int PacketSize = unpacket_traits::size; Index actualIndex = index + (offset * PacketSize); eigen_assert(check_array_bounds(actualIndex, PacketSize) && "Array index out of bounds"); @@ -652,7 +652,7 @@ struct unary_evaluator, ArgType>, In // If not, perform full load. Otherwise, revert to a scalar loop to perform a partial load. // In either case, perform a vectorized cast of the source packet. template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int DstPacketSize = unpacket_traits::size; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); @@ -669,7 +669,7 @@ struct unary_evaluator, ArgType>, In } // Use the source packet type with the same size as DstPacketType, if it exists template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int DstPacketSize = unpacket_traits::size; using SizedSrcPacketType = typename find_packet_by_size::type; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); @@ -678,14 +678,14 @@ struct unary_evaluator, ArgType>, In } // unpacket_traits::size == 2 * SrcPacketSize template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast(srcPacket(row, col, 0), srcPacket(row, col, 1)); } // unpacket_traits::size == 4 * SrcPacketSize template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast(srcPacket(row, col, 0), srcPacket(row, col, 1), srcPacket(row, col, 2), @@ -693,7 +693,7 @@ struct unary_evaluator, ArgType>, In } // unpacket_traits::size == 8 * SrcPacketSize template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast( srcPacket(row, col, 0), srcPacket(row, col, 1), srcPacket(row, col, 2), @@ -703,7 +703,7 @@ struct unary_evaluator, ArgType>, In // Analogous routines for linear access. template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int DstPacketSize = unpacket_traits::size; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); @@ -719,7 +719,7 @@ struct unary_evaluator, ArgType>, In return pcast(src); } template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int DstPacketSize = unpacket_traits::size; using SizedSrcPacketType = typename find_packet_by_size::type; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); @@ -727,18 +727,18 @@ struct unary_evaluator, ArgType>, In return pcast(srcPacket(index, 0)); } template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast(srcPacket(index, 0), srcPacket(index, 1)); } template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast(srcPacket(index, 0), srcPacket(index, 1), srcPacket(index, 2), srcPacket(index, 3)); } template = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast(srcPacket(index, 0), srcPacket(index, 1), srcPacket(index, 2), srcPacket(index, 3), @@ -810,14 +810,14 @@ struct ternary_evaluator, IndexBased } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.arg1Impl.template packet(row, col), m_d.arg2Impl.template packet(row, col), m_d.arg3Impl.template packet(row, col)); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.arg1Impl.template packet(index), m_d.arg2Impl.template packet(index), m_d.arg3Impl.template packet(index)); @@ -908,13 +908,13 @@ struct binary_evaluator, IndexBased, IndexBase } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.lhsImpl.template packet(row, col), m_d.rhsImpl.template packet(row, col)); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.lhsImpl.template packet(index), m_d.rhsImpl.template packet(index)); } @@ -1030,24 +1030,24 @@ struct mapbase_evaluator : evaluator_base { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_data[index * m_innerStride.value()]; } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::ploadt(ptr); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return internal::ploadt(m_data + index * m_innerStride.value()); } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::pstoret(ptr, x); } template - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { internal::pstoret(m_data + index * m_innerStride.value(), x); } @@ -1217,12 +1217,12 @@ struct unary_evaluator, IndexBa } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(m_startRow.value() + row, m_startCol.value() + col); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { if (ForwardLinearAccess) return m_argImpl.template packet(m_linear_offset.value() + index); else @@ -1230,12 +1230,12 @@ struct unary_evaluator, IndexBa } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { return m_argImpl.template writePacket(m_startRow.value() + row, m_startCol.value() + col, x); } template - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { if (ForwardLinearAccess) return m_argImpl.template writePacket(m_linear_offset.value() + index, x); else @@ -1378,7 +1378,7 @@ struct unary_evaluator> } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { const Index actual_row = internal::traits::RowsAtCompileTime == 1 ? 0 : RowFactor == 1 ? row : row % m_rows.value(); @@ -1390,7 +1390,7 @@ struct unary_evaluator> } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { const Index actual_index = internal::traits::RowsAtCompileTime == 1 ? (ColFactor == 1 ? index : index % m_cols.value()) : (RowFactor == 1 ? index : index % m_rows.value()); @@ -1435,22 +1435,22 @@ struct evaluator_wrapper_base : evaluator_base { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index); } template - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(row, col); } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet(index); } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket(row, col, x); } template - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket(index, x); } @@ -1532,7 +1532,7 @@ struct unary_evaluator> : evaluator_base - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { enum { PacketSize = unpacket_traits::size, OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, @@ -1544,14 +1544,14 @@ struct unary_evaluator> : evaluator_base - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { enum { PacketSize = unpacket_traits::size }; return preverse( m_argImpl.template packet(m_rows.value() * m_cols.value() - index - PacketSize)); } template - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { // FIXME we could factorize some code with packet(i,j) enum { PacketSize = unpacket_traits::size, @@ -1565,7 +1565,7 @@ struct unary_evaluator> : evaluator_base - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { enum { PacketSize = unpacket_traits::size }; m_argImpl.template writePacket(m_rows.value() * m_cols.value() - index - PacketSize, preverse(x)); } diff --git a/Eigen/src/Core/InnerProduct.h b/Eigen/src/Core/InnerProduct.h index 38689daaf..1e16942c2 100644 --- a/Eigen/src/Core/InnerProduct.h +++ b/Eigen/src/Core/InnerProduct.h @@ -85,13 +85,13 @@ struct inner_product_evaluator { } template - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_func.packet(m_lhs.template packet(index), m_rhs.template packet(index)); } template - EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const { return m_func.packet(value, m_lhs.template packet(index), m_rhs.template packet(index)); }