diff --git a/Eigen/src/Core/arch/AVX/MathFunctions.h b/Eigen/src/Core/arch/AVX/MathFunctions.h index 5fe2cff32..c0d362fa7 100644 --- a/Eigen/src/Core/arch/AVX/MathFunctions.h +++ b/Eigen/src/Core/arch/AVX/MathFunctions.h @@ -97,33 +97,36 @@ pexp(const Packet4d& _x) { // For detail see here: http://www.beyond3d.com/content/articles/8/ #if EIGEN_FAST_MATH template <> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f -psqrt(const Packet8f& _x) { - Packet8f half = pmul(_x, pset1(.5f)); - Packet8f denormal_mask = _mm256_and_ps( - _mm256_cmp_ps(_x, pset1((std::numeric_limits::min)()), - _CMP_LT_OQ), - _mm256_cmp_ps(_x, _mm256_setzero_ps(), _CMP_GE_OQ)); +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet8f psqrt(const Packet8f& _x) { + Packet8f minus_half_x = pmul(_x, pset1(-0.5f)); + Packet8f denormal_mask = pandnot( + pcmp_lt(_x, pset1((std::numeric_limits::min)())), + pcmp_lt(_x, pzero(_x))); // Compute approximate reciprocal sqrt. Packet8f x = _mm256_rsqrt_ps(_x); // Do a single step of Newton's iteration. - x = pmul(x, psub(pset1(1.5f), pmul(half, pmul(x,x)))); + x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); // Flush results for denormals to zero. - return _mm256_andnot_ps(denormal_mask, pmul(_x,x)); + return pandnot(pmul(_x,x), denormal_mask); } + #else + template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psqrt(const Packet8f& _x) { return _mm256_sqrt_ps(_x); } + #endif + template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d psqrt(const Packet4d& _x) { return _mm256_sqrt_pd(_x); } -#if EIGEN_FAST_MATH +#if EIGEN_FAST_MATH template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f prsqrt(const Packet8f& _x) { _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(inf, 0x7f800000); diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 5883eca38..14f3dbd0f 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h @@ -3273,26 +3273,26 @@ template<> EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) { // effective latency. This is similar to Quake3's fast inverse square root. // For more details see: http://www.beyond3d.com/content/articles/8/ template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){ - Packet4f half = vmulq_n_f32(_x, 0.5f); + Packet4f minus_half_x = vmulq_n_f32(_x, -0.5f); Packet4ui denormal_mask = vandq_u32(vcgeq_f32(_x, vdupq_n_f32(0.0f)), vcltq_f32(_x, pset1((std::numeric_limits::min)()))); // Compute approximate reciprocal sqrt. Packet4f x = vrsqrteq_f32(_x); // Do a single step of Newton's iteration. //the number 1.5f was set reference to Quake3's fast inverse square root - x = vmulq_f32(x, psub(pset1(1.5f), pmul(half, pmul(x, x)))); + x = pmul(x, pmadd(minus_half_x, pmul(x, x), pset1(1.5f))); // Flush results for denormals to zero. return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(pmul(_x, x)), denormal_mask)); } -template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){ - Packet2f half = vmul_n_f32(_x, 0.5f); +template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x) { + Packet2f minus_half_x = vmul_n_f32(_x, -0.5f); Packet2ui denormal_mask = vand_u32(vcge_f32(_x, vdup_n_f32(0.0f)), vclt_f32(_x, pset1((std::numeric_limits::min)()))); // Compute approximate reciprocal sqrt. Packet2f x = vrsqrte_f32(_x); // Do a single step of Newton's iteration. - x = vmul_f32(x, psub(pset1(1.5f), pmul(half, pmul(x, x)))); + x = pmul(x, pmadd(minus_half_x, pmul(x, x), pset1(1.5f))); // Flush results for denormals to zero. return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(pmul(_x, x)), denormal_mask)); } @@ -3877,35 +3877,7 @@ template<> EIGEN_STRONG_INLINE Packet2d pfrexp(const Packet2d& a, Pack template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { return vreinterpretq_f64_u64(vdupq_n_u64(from)); } -#if EIGEN_FAST_MATH - -// Functions for sqrt support packet2d. -// The EIGEN_FAST_MATH version uses the vrsqrte_f64 approximation and one step -// of Newton's method, at a cost of 1-2 bits of precision as opposed to the -// exact solution. It does not handle +inf, or denormalized numbers correctly. -// The main advantage of this approach is not just speed, but also the fact that -// it can be inlined and pipelined with other computations, further reducing its -// effective latency. This is similar to Quake3's fast inverse square root. -// For more details see: http://www.beyond3d.com/content/articles/8/ -template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ - Packet2d half = vmulq_n_f64(_x, 0.5); - Packet2ul denormal_mask = vandq_u64(vcgeq_f64(_x, vdupq_n_f64(0.0)), - vcltq_f64(_x, pset1((std::numeric_limits::min)()))); - // Compute approximate reciprocal sqrt. - Packet2d x = vrsqrteq_f64(_x); - // Do a single step of Newton's iteration. - //the number 1.5f was set reference to Quake3's fast inverse square root - x = vmulq_f64(x, psub(pset1(1.5), pmul(half, pmul(x, x)))); - // Do two more Newton's iteration to get a result accurate to 1 ulp. - x = vmulq_f64(x, psub(pset1(1.5), pmul(half, pmul(x, x)))); - x = vmulq_f64(x, psub(pset1(1.5), pmul(half, pmul(x, x)))); - // Flush results for denormals to zero. - return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(pmul(_x, x)), denormal_mask)); -} - -#else template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); } -#endif #endif // EIGEN_ARCH_ARM64 diff --git a/Eigen/src/Core/arch/SSE/MathFunctions.h b/Eigen/src/Core/arch/SSE/MathFunctions.h index 2d7df2f7b..9f66d8ab3 100644 --- a/Eigen/src/Core/arch/SSE/MathFunctions.h +++ b/Eigen/src/Core/arch/SSE/MathFunctions.h @@ -86,17 +86,17 @@ Packet4f pcos(const Packet4f& _x) template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt(const Packet4f& _x) { - Packet4f half = pmul(_x, pset1(.5f)); - Packet4f denormal_mask = _mm_and_ps( - _mm_cmpge_ps(_x, _mm_setzero_ps()), - _mm_cmplt_ps(_x, pset1((std::numeric_limits::min)()))); + Packet4f minus_half_x = pmul(_x, pset1(-0.5f)); + Packet4f denormal_mask = pandnot( + pcmp_lt(_x, pset1((std::numeric_limits::min)())), + pcmp_lt(_x, pzero(_x))); // Compute approximate reciprocal sqrt. Packet4f x = _mm_rsqrt_ps(_x); // Do a single step of Newton's iteration. - x = pmul(x, psub(pset1(1.5f), pmul(half, pmul(x,x)))); + x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); // Flush results for denormals to zero. - return _mm_andnot_ps(denormal_mask, pmul(_x,x)); + return pandnot(pmul(_x,x), denormal_mask); } #else