From 0ae7b590189b7b3b8592f28967f06aa2d3dc153f Mon Sep 17 00:00:00 2001 From: Tyler Veness Date: Fri, 21 Feb 2025 18:16:46 +0000 Subject: [PATCH] Make assignment constexpr --- Eigen/src/Core/AssignEvaluator.h | 94 ++++++++++++-------- Eigen/src/Core/EigenBase.h | 2 +- Eigen/src/Core/Fill.h | 8 +- Eigen/src/Core/functors/AssignmentFunctors.h | 2 +- test/constexpr.cpp | 19 ++++ 5 files changed, 81 insertions(+), 44 deletions(-) diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 5a2a3ac17..0ea1bc3c0 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h @@ -191,7 +191,7 @@ struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling { static constexpr int Outer = Index_ / Kernel::AssignmentTraits::InnerSizeAtCompileTime; static constexpr int Inner = Index_ % Kernel::AssignmentTraits::InnerSizeAtCompileTime; - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { kernel.assignCoeffByOuterInner(Outer, Inner); copy_using_evaluator_DefaultTraversal_CompleteUnrolling::run(kernel); } @@ -204,7 +204,7 @@ struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling struct copy_using_evaluator_DefaultTraversal_InnerUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel, Index outer) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Index outer) { kernel.assignCoeffByOuterInner(outer, Index_); copy_using_evaluator_DefaultTraversal_InnerUnrolling::run(kernel, outer); } @@ -212,7 +212,7 @@ struct copy_using_evaluator_DefaultTraversal_InnerUnrolling { template struct copy_using_evaluator_DefaultTraversal_InnerUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) {} + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&, Index) {} }; /*********************** @@ -221,7 +221,7 @@ struct copy_using_evaluator_DefaultTraversal_InnerUnrolling template struct copy_using_evaluator_LinearTraversal_CompleteUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { kernel.assignCoeff(Index_); copy_using_evaluator_LinearTraversal_CompleteUnrolling::run(kernel); } @@ -229,7 +229,7 @@ struct copy_using_evaluator_LinearTraversal_CompleteUnrolling { template struct copy_using_evaluator_LinearTraversal_CompleteUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) {} + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&) {} }; /************************** @@ -270,7 +270,7 @@ struct copy_using_evaluator_innervec_InnerUnrolling { template struct copy_using_evaluator_innervec_InnerUnrolling { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) {} + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel&, Index) {} }; /*************************************************************************** @@ -281,7 +281,21 @@ struct copy_using_evaluator_innervec_InnerUnrolling -struct dense_assignment_loop; +struct dense_assignment_loop_impl; + +template +struct dense_assignment_loop { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { +#ifdef __cpp_lib_is_constant_evaluated + if (internal::is_constant_evaluated()) + dense_assignment_loop_impl::run(kernel); + else +#endif + dense_assignment_loop_impl::run(kernel); + } +}; /************************ ***** Special Cases ***** @@ -289,7 +303,7 @@ struct dense_assignment_loop; // Zero-sized assignment is a no-op. template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime; EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE EIGEN_CONSTEXPR run(Kernel& /*kernel*/) { @@ -302,8 +316,8 @@ struct dense_assignment_loop { ************************/ template -struct dense_assignment_loop { - EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE run(Kernel& kernel) { +struct dense_assignment_loop_impl { + EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE constexpr run(Kernel& kernel) { for (Index outer = 0; outer < kernel.outerSize(); ++outer) { for (Index inner = 0; inner < kernel.innerSize(); ++inner) { kernel.assignCoeffByOuterInner(outer, inner); @@ -313,19 +327,19 @@ struct dense_assignment_loop { }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime; - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { copy_using_evaluator_DefaultTraversal_CompleteUnrolling::run(kernel); } }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { static constexpr int InnerSizeAtCompileTime = Kernel::AssignmentTraits::InnerSizeAtCompileTime; - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { const Index outerSize = kernel.outerSize(); for (Index outer = 0; outer < outerSize; ++outer) copy_using_evaluator_DefaultTraversal_InnerUnrolling::run(kernel, outer); @@ -382,7 +396,7 @@ struct copy_using_evaluator_linearvec_CompleteUnrolling { }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { using Scalar = typename Kernel::Scalar; using PacketType = typename Kernel::PacketType; static constexpr int PacketSize = unpacket_traits::size; @@ -407,7 +421,7 @@ struct dense_assignment_loop { }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { using PacketType = typename Kernel::PacketType; static constexpr int PacketSize = unpacket_traits::size; static constexpr int Size = Kernel::AssignmentTraits::SizeAtCompileTime; @@ -424,7 +438,7 @@ struct dense_assignment_loop -struct dense_assignment_loop { +struct dense_assignment_loop_impl { using PacketType = typename Kernel::PacketType; static constexpr int PacketSize = unpacket_traits::size; static constexpr int SrcAlignment = Kernel::AssignmentTraits::JointAlignment; @@ -440,7 +454,7 @@ struct dense_assignment_loop { }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { static constexpr int SizeAtCompileTime = Kernel::AssignmentTraits::SizeAtCompileTime; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) { @@ -449,7 +463,7 @@ struct dense_assignment_loop -struct dense_assignment_loop { +struct dense_assignment_loop_impl { static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime; static constexpr int SrcAlignment = Kernel::AssignmentTraits::SrcAlignment; static constexpr int DstAlignment = Kernel::AssignmentTraits::DstAlignment; @@ -467,7 +481,7 @@ struct dense_assignment_loop { ***********************/ template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) { const Index size = kernel.size(); for (Index i = 0; i < size; ++i) kernel.assignCoeff(i); @@ -475,7 +489,7 @@ struct dense_assignment_loop { }; template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) { copy_using_evaluator_LinearTraversal_CompleteUnrolling::run( kernel); @@ -487,7 +501,7 @@ struct dense_assignment_loop { ***************************/ template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { using Scalar = typename Kernel::Scalar; using PacketType = typename Kernel::PacketType; static constexpr int PacketSize = unpacket_traits::size; @@ -528,7 +542,7 @@ struct dense_assignment_loop { #if EIGEN_UNALIGNED_VECTORIZE template -struct dense_assignment_loop { +struct dense_assignment_loop_impl { using PacketType = typename Kernel::PacketType; static constexpr int PacketSize = unpacket_traits::size; static constexpr int InnerSize = Kernel::AssignmentTraits::InnerSizeAtCompileTime; @@ -566,9 +580,10 @@ class generic_dense_assignment_kernel { typedef copy_using_evaluator_traits AssignmentTraits; typedef typename AssignmentTraits::PacketType PacketType; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE generic_dense_assignment_kernel(DstEvaluatorType& dst, - const SrcEvaluatorType& src, - const Functor& func, DstXprType& dstExpr) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr generic_dense_assignment_kernel(DstEvaluatorType& dst, + const SrcEvaluatorType& src, + const Functor& func, + DstXprType& dstExpr) : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) { #ifdef EIGEN_DEBUG_ASSIGN AssignmentTraits::debug(); @@ -586,7 +601,7 @@ class generic_dense_assignment_kernel { EIGEN_DEVICE_FUNC const SrcEvaluatorType& srcEvaluator() const EIGEN_NOEXCEPT { return m_src; } /// Assign src(row,col) to dst(row,col) through the assignment functor. - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index row, Index col) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeff(Index row, Index col) { m_functor.assignCoeff(m_dst.coeffRef(row, col), m_src.coeff(row, col)); } @@ -596,7 +611,7 @@ class generic_dense_assignment_kernel { } /// \sa assignCoeff(Index,Index) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeffByOuterInner(Index outer, Index inner) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeffByOuterInner(Index outer, Index inner) { Index row = rowIndexByOuterInner(outer, inner); Index col = colIndexByOuterInner(outer, inner); assignCoeff(row, col); @@ -620,7 +635,7 @@ class generic_dense_assignment_kernel { assignPacket(row, col); } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr Index rowIndexByOuterInner(Index outer, Index inner) { typedef typename DstEvaluatorType::ExpressionTraits Traits; return int(Traits::RowsAtCompileTime) == 1 ? 0 : int(Traits::ColsAtCompileTime) == 1 ? inner @@ -628,7 +643,7 @@ class generic_dense_assignment_kernel { : inner; } - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr Index colIndexByOuterInner(Index outer, Index inner) { typedef typename DstEvaluatorType::ExpressionTraits Traits; return int(Traits::ColsAtCompileTime) == 1 ? 0 : int(Traits::RowsAtCompileTime) == 1 ? inner @@ -672,16 +687,16 @@ class restricted_packet_dense_assignment_kernel ***************************************************************************/ template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize_if_allowed(DstXprType& dst, const SrcXprType& src, - const Functor& /*func*/) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize_if_allowed(DstXprType& dst, const SrcXprType& src, + const Functor& /*func*/) { EIGEN_ONLY_USED_FOR_DEBUG(dst); EIGEN_ONLY_USED_FOR_DEBUG(src); eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize_if_allowed(DstXprType& dst, const SrcXprType& src, - const internal::assign_op& /*func*/) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize_if_allowed(DstXprType& dst, const SrcXprType& src, + const internal::assign_op& /*func*/) { Index dstRows = src.rows(); Index dstCols = src.cols(); if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) dst.resize(dstRows, dstCols); @@ -750,7 +765,7 @@ struct Assignment; // not has to bother about these annoying details. template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(Dst& dst, const Src& src) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment(Dst& dst, const Src& src) { call_assignment(dst, src, internal::assign_op()); } template @@ -767,7 +782,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment( } template -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment( +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment( Dst& dst, const Src& src, const Func& func, std::enable_if_t::value, void*> = 0) { call_assignment_no_alias(dst, src, func); } @@ -851,9 +866,12 @@ EIGEN_DEVICE_FUNC void check_for_aliasing(const Dst& dst, const Src& src); // both partial specialization+SFINAE without ambiguous specialization template struct Assignment { - EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func) { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(DstXprType& dst, const SrcXprType& src, + const Functor& func) { #ifndef EIGEN_NO_DEBUG - internal::check_for_aliasing(dst, src); + if (!internal::is_constant_evaluated()) { + internal::check_for_aliasing(dst, src); + } #endif call_dense_assignment_loop(dst, src, func); diff --git a/Eigen/src/Core/EigenBase.h b/Eigen/src/Core/EigenBase.h index 6d167006a..894bfc13b 100644 --- a/Eigen/src/Core/EigenBase.h +++ b/Eigen/src/Core/EigenBase.h @@ -50,7 +50,7 @@ struct EigenBase { /** \returns a const reference to the derived object */ EIGEN_DEVICE_FUNC constexpr const Derived& derived() const { return *static_cast(this); } - EIGEN_DEVICE_FUNC inline Derived& const_cast_derived() const { + EIGEN_DEVICE_FUNC inline constexpr Derived& const_cast_derived() const { return *static_cast(const_cast(this)); } EIGEN_DEVICE_FUNC inline const Derived& const_derived() const { return *static_cast(this); } diff --git a/Eigen/src/Core/Fill.h b/Eigen/src/Core/Fill.h index dd57ca196..3b0af91a7 100644 --- a/Eigen/src/Core/Fill.h +++ b/Eigen/src/Core/Fill.h @@ -60,12 +60,12 @@ struct eigen_fill_impl { using Func = scalar_constant_op; using PlainObject = typename Xpr::PlainObject; using Constant = typename PlainObject::ConstantReturnType; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const Scalar& val) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const Scalar& val) { const Constant src(dst.rows(), dst.cols(), val); run(dst, src); } template - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const SrcXpr& src) { call_dense_assignment_loop(dst, src, assign_op()); } }; @@ -100,12 +100,12 @@ struct eigen_zero_impl { using Scalar = typename Xpr::Scalar; using PlainObject = typename Xpr::PlainObject; using Zero = typename PlainObject::ZeroReturnType; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst) { const Zero src(dst.rows(), dst.cols()); run(dst, src); } template - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const SrcXpr& src) { call_dense_assignment_loop(dst, src, assign_op()); } }; diff --git a/Eigen/src/Core/functors/AssignmentFunctors.h b/Eigen/src/Core/functors/AssignmentFunctors.h index 09d1da8ca..3687bb20d 100644 --- a/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Eigen/src/Core/functors/AssignmentFunctors.h @@ -23,7 +23,7 @@ namespace internal { */ template struct assign_op { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { diff --git a/test/constexpr.cpp b/test/constexpr.cpp index 34c728f66..826d4e249 100644 --- a/test/constexpr.cpp +++ b/test/constexpr.cpp @@ -10,6 +10,13 @@ #define EIGEN_TESTING_CONSTEXPR #include "main.h" +template +struct ConstexprTest { + constexpr ConstexprTest(const Matrix& B) { A = B; } + + Matrix A; +}; + EIGEN_DECLARE_TEST(constexpr) { // Clang accepts (some of) this code when using C++14/C++17, but GCC does not like // the fact that `T array[Size]` inside Eigen::internal::plain_array is not initialized @@ -33,6 +40,18 @@ EIGEN_DECLARE_TEST(constexpr) { VERIFY_IS_EQUAL(vec.size(), 3); static_assert(vec.coeff(0, 1) == 2); + // Check assignment. A wrapper struct is used to avoid copy ellision. + constexpr ConstexprTest obj1(Matrix2d({{1, 2}, {3, 4}})); + VERIFY_IS_EQUAL(obj1.A.size(), 4); + static_assert(obj1.A(0, 0) == 1); + static_assert(obj1.A(0) == 1); + static_assert(obj1.A.coeff(0, 1) == 2); + constexpr ConstexprTest obj2(Matrix3d({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}})); + VERIFY_IS_EQUAL(obj2.A.size(), 9); + static_assert(obj2.A(0, 0) == 1); + static_assert(obj2.A(0) == 1); + static_assert(obj2.A.coeff(0, 1) == 2); + // Also check dynamic size arrays/matrices with fixed-size storage (currently // only works if all elements are initialized, since otherwise the compiler // complains about uninitialized trailing elements.