mirror of
https://gitlab.com/libeigen/eigen.git
synced 2025-05-03 01:04:23 +08:00
822 lines
32 KiB
C++
Executable File
822 lines
32 KiB
C++
Executable File
// This file is part of Eigen, a lightweight C++ template library
|
|
// for linear algebra.
|
|
//
|
|
// Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com>
|
|
// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
|
|
// Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk>
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla
|
|
// Public License v. 2.0. If a copy of the MPL was not distributed
|
|
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
#ifndef EIGEN_ASSIGN_EVALUATOR_H
|
|
#define EIGEN_ASSIGN_EVALUATOR_H
|
|
|
|
namespace Eigen {
|
|
|
|
// This implementation is based on Assign.h
|
|
|
|
namespace internal {
|
|
|
|
/***************************************************************************
|
|
* Part 1 : the logic deciding a strategy for traversal and unrolling *
|
|
***************************************************************************/
|
|
|
|
// copy_using_evaluator_traits is based on assign_traits
|
|
|
|
template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc>
|
|
struct copy_using_evaluator_traits
|
|
{
|
|
typedef typename DstEvaluator::XprType Dst;
|
|
typedef typename Dst::Scalar DstScalar;
|
|
// TODO distinguish between linear traversal and inner-traversals
|
|
typedef typename find_best_packet<DstScalar,Dst::SizeAtCompileTime>::type PacketType;
|
|
|
|
enum {
|
|
DstFlags = DstEvaluator::Flags,
|
|
SrcFlags = SrcEvaluator::Flags,
|
|
RequiredAlignment = unpacket_traits<PacketType>::alignment
|
|
};
|
|
|
|
public:
|
|
enum {
|
|
DstAlignment = DstEvaluator::Alignment,
|
|
SrcAlignment = SrcEvaluator::Alignment,
|
|
DstHasDirectAccess = DstFlags & DirectAccessBit,
|
|
JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment)
|
|
};
|
|
|
|
private:
|
|
enum {
|
|
InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
|
|
: int(DstFlags)&RowMajorBit ? int(Dst::ColsAtCompileTime)
|
|
: int(Dst::RowsAtCompileTime),
|
|
InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
|
|
: int(DstFlags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime)
|
|
: int(Dst::MaxRowsAtCompileTime),
|
|
OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
|
|
MaxSizeAtCompileTime = Dst::SizeAtCompileTime,
|
|
PacketSize = unpacket_traits<PacketType>::size
|
|
};
|
|
|
|
enum {
|
|
DstIsRowMajor = DstFlags&RowMajorBit,
|
|
SrcIsRowMajor = SrcFlags&RowMajorBit,
|
|
StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
|
|
MightVectorize = StorageOrdersAgree
|
|
&& (int(DstFlags) & int(SrcFlags) & ActualPacketAccessBit)
|
|
&& (functor_traits<AssignFunc>::PacketAccess),
|
|
MayInnerVectorize = MightVectorize
|
|
&& int(InnerSize)!=Dynamic && int(InnerSize)%int(PacketSize)==0
|
|
&& int(OuterStride)!=Dynamic && int(OuterStride)%int(PacketSize)==0
|
|
&& int(JointAlignment)>=int(RequiredAlignment),
|
|
MayLinearize = StorageOrdersAgree && (int(DstFlags) & int(SrcFlags) & LinearAccessBit),
|
|
MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess
|
|
&& ((int(DstAlignment)>=int(RequiredAlignment)) || MaxSizeAtCompileTime == Dynamic),
|
|
/* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
|
|
so it's only good for large enough sizes. */
|
|
MaySliceVectorize = MightVectorize && DstHasDirectAccess
|
|
&& (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=3*PacketSize)
|
|
/* slice vectorization can be slow, so we only want it if the slices are big, which is
|
|
indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
|
|
in a fixed-size matrix */
|
|
};
|
|
|
|
public:
|
|
enum {
|
|
Traversal = int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
|
|
: int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
|
|
: int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
|
|
: int(MayLinearize) ? int(LinearTraversal)
|
|
: int(DefaultTraversal),
|
|
Vectorized = int(Traversal) == InnerVectorizedTraversal
|
|
|| int(Traversal) == LinearVectorizedTraversal
|
|
|| int(Traversal) == SliceVectorizedTraversal
|
|
};
|
|
|
|
private:
|
|
enum {
|
|
UnrollingLimit = EIGEN_UNROLLING_LIMIT * (Vectorized ? int(PacketSize) : 1),
|
|
MayUnrollCompletely = int(Dst::SizeAtCompileTime) != Dynamic
|
|
&& int(Dst::SizeAtCompileTime) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit),
|
|
MayUnrollInner = int(InnerSize) != Dynamic
|
|
&& int(InnerSize) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit)
|
|
};
|
|
|
|
public:
|
|
enum {
|
|
Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
|
|
? (
|
|
int(MayUnrollCompletely) ? int(CompleteUnrolling)
|
|
: int(MayUnrollInner) ? int(InnerUnrolling)
|
|
: int(NoUnrolling)
|
|
)
|
|
: int(Traversal) == int(LinearVectorizedTraversal)
|
|
? ( bool(MayUnrollCompletely) && (int(DstAlignment)>=int(RequiredAlignment)) ? int(CompleteUnrolling)
|
|
: int(NoUnrolling) )
|
|
: int(Traversal) == int(LinearTraversal)
|
|
? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling)
|
|
: int(NoUnrolling) )
|
|
: int(NoUnrolling)
|
|
};
|
|
|
|
#ifdef EIGEN_DEBUG_ASSIGN
|
|
static void debug()
|
|
{
|
|
std::cerr << "DstXpr: " << typeid(typename DstEvaluator::XprType).name() << std::endl;
|
|
std::cerr << "SrcXpr: " << typeid(typename SrcEvaluator::XprType).name() << std::endl;
|
|
std::cerr.setf(std::ios::hex, std::ios::basefield);
|
|
std::cerr << "DstFlags" << " = " << DstFlags << " (" << demangle_flags(DstFlags) << " )" << std::endl;
|
|
std::cerr << "SrcFlags" << " = " << SrcFlags << " (" << demangle_flags(SrcFlags) << " )" << std::endl;
|
|
std::cerr.unsetf(std::ios::hex);
|
|
EIGEN_DEBUG_VAR(DstAlignment)
|
|
EIGEN_DEBUG_VAR(SrcAlignment)
|
|
EIGEN_DEBUG_VAR(RequiredAlignment)
|
|
EIGEN_DEBUG_VAR(JointAlignment)
|
|
EIGEN_DEBUG_VAR(InnerSize)
|
|
EIGEN_DEBUG_VAR(InnerMaxSize)
|
|
EIGEN_DEBUG_VAR(PacketSize)
|
|
EIGEN_DEBUG_VAR(StorageOrdersAgree)
|
|
EIGEN_DEBUG_VAR(MightVectorize)
|
|
EIGEN_DEBUG_VAR(MayLinearize)
|
|
EIGEN_DEBUG_VAR(MayInnerVectorize)
|
|
EIGEN_DEBUG_VAR(MayLinearVectorize)
|
|
EIGEN_DEBUG_VAR(MaySliceVectorize)
|
|
std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl;
|
|
EIGEN_DEBUG_VAR(UnrollingLimit)
|
|
EIGEN_DEBUG_VAR(MayUnrollCompletely)
|
|
EIGEN_DEBUG_VAR(MayUnrollInner)
|
|
std::cerr << "Unrolling" << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl;
|
|
std::cerr << std::endl;
|
|
}
|
|
#endif
|
|
};
|
|
|
|
/***************************************************************************
|
|
* Part 2 : meta-unrollers
|
|
***************************************************************************/
|
|
|
|
/************************
|
|
*** Default traversal ***
|
|
************************/
|
|
|
|
template<typename Kernel, int Index, int Stop>
|
|
struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling
|
|
{
|
|
// FIXME: this is not very clean, perhaps this information should be provided by the kernel?
|
|
typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
|
|
typedef typename DstEvaluatorType::XprType DstXprType;
|
|
|
|
enum {
|
|
outer = Index / DstXprType::InnerSizeAtCompileTime,
|
|
inner = Index % DstXprType::InnerSizeAtCompileTime
|
|
};
|
|
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
kernel.assignCoeffByOuterInner(outer, inner);
|
|
copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel, int Stop>
|
|
struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
|
|
};
|
|
|
|
template<typename Kernel, int Index_, int Stop>
|
|
struct copy_using_evaluator_DefaultTraversal_InnerUnrolling
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
|
|
{
|
|
kernel.assignCoeffByOuterInner(outer, Index_);
|
|
copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_+1, Stop>::run(kernel, outer);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel, int Stop>
|
|
struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) { }
|
|
};
|
|
|
|
/***********************
|
|
*** Linear traversal ***
|
|
***********************/
|
|
|
|
template<typename Kernel, int Index, int Stop>
|
|
struct copy_using_evaluator_LinearTraversal_CompleteUnrolling
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel)
|
|
{
|
|
kernel.assignCoeff(Index);
|
|
copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel, int Stop>
|
|
struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
|
|
};
|
|
|
|
/**************************
|
|
*** Inner vectorization ***
|
|
**************************/
|
|
|
|
template<typename Kernel, int Index, int Stop>
|
|
struct copy_using_evaluator_innervec_CompleteUnrolling
|
|
{
|
|
// FIXME: this is not very clean, perhaps this information should be provided by the kernel?
|
|
typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
|
|
typedef typename DstEvaluatorType::XprType DstXprType;
|
|
typedef typename Kernel::PacketType PacketType;
|
|
|
|
enum {
|
|
outer = Index / DstXprType::InnerSizeAtCompileTime,
|
|
inner = Index % DstXprType::InnerSizeAtCompileTime,
|
|
JointAlignment = Kernel::AssignmentTraits::JointAlignment
|
|
};
|
|
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
kernel.template assignPacketByOuterInner<Aligned, JointAlignment, PacketType>(outer, inner);
|
|
enum { NextIndex = Index + unpacket_traits<PacketType>::size };
|
|
copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel, int Stop>
|
|
struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
|
|
};
|
|
|
|
template<typename Kernel, int Index_, int Stop>
|
|
struct copy_using_evaluator_innervec_InnerUnrolling
|
|
{
|
|
typedef typename Kernel::PacketType PacketType;
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
|
|
{
|
|
kernel.template assignPacketByOuterInner<Aligned, Aligned, PacketType>(outer, Index_);
|
|
enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
|
|
copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop>::run(kernel, outer);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel, int Stop>
|
|
struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &, Index) { }
|
|
};
|
|
|
|
/***************************************************************************
|
|
* Part 3 : implementation of all cases
|
|
***************************************************************************/
|
|
|
|
// dense_assignment_loop is based on assign_impl
|
|
|
|
template<typename Kernel,
|
|
int Traversal = Kernel::AssignmentTraits::Traversal,
|
|
int Unrolling = Kernel::AssignmentTraits::Unrolling>
|
|
struct dense_assignment_loop;
|
|
|
|
/************************
|
|
*** Default traversal ***
|
|
************************/
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE run(Kernel &kernel)
|
|
{
|
|
for(Index outer = 0; outer < kernel.outerSize(); ++outer) {
|
|
for(Index inner = 0; inner < kernel.innerSize(); ++inner) {
|
|
kernel.assignCoeffByOuterInner(outer, inner);
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
|
|
const Index outerSize = kernel.outerSize();
|
|
for(Index outer = 0; outer < outerSize; ++outer)
|
|
copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
|
|
}
|
|
};
|
|
|
|
/***************************
|
|
*** Linear vectorization ***
|
|
***************************/
|
|
|
|
|
|
// The goal of unaligned_dense_assignment_loop is simply to factorize the handling
|
|
// of the non vectorizable beginning and ending parts
|
|
|
|
template <bool IsAligned = false>
|
|
struct unaligned_dense_assignment_loop
|
|
{
|
|
// if IsAligned = true, then do nothing
|
|
template <typename Kernel>
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index, Index) {}
|
|
};
|
|
|
|
template <>
|
|
struct unaligned_dense_assignment_loop<false>
|
|
{
|
|
// MSVC must not inline this functions. If it does, it fails to optimize the
|
|
// packet access path.
|
|
// FIXME check which version exhibits this issue
|
|
#if EIGEN_COMP_MSVC
|
|
template <typename Kernel>
|
|
static EIGEN_DONT_INLINE void run(Kernel &kernel,
|
|
Index start,
|
|
Index end)
|
|
#else
|
|
template <typename Kernel>
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel,
|
|
Index start,
|
|
Index end)
|
|
#endif
|
|
{
|
|
for (Index index = start; index < end; ++index)
|
|
kernel.assignCoeff(index);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
const Index size = kernel.size();
|
|
typedef typename Kernel::Scalar Scalar;
|
|
typedef typename Kernel::PacketType PacketType;
|
|
enum {
|
|
requestedAlignment = Kernel::AssignmentTraits::RequiredAlignment,
|
|
packetSize = unpacket_traits<PacketType>::size,
|
|
dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
|
|
dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
|
|
: int(Kernel::AssignmentTraits::DstAlignment),
|
|
srcAlignment = Kernel::AssignmentTraits::JointAlignment
|
|
};
|
|
const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(&kernel.dstEvaluator().coeffRef(0), size);
|
|
const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
|
|
|
|
unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart);
|
|
|
|
for(Index index = alignedStart; index < alignedEnd; index += packetSize)
|
|
kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
|
|
|
|
unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
|
|
enum { size = DstXprType::SizeAtCompileTime,
|
|
packetSize = packet_traits<typename Kernel::Scalar>::size,
|
|
alignedSize = (size/packetSize)*packetSize };
|
|
|
|
copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
|
|
copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
|
|
}
|
|
};
|
|
|
|
/**************************
|
|
*** Inner vectorization ***
|
|
**************************/
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling>
|
|
{
|
|
typedef typename Kernel::PacketType PacketType;
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
const Index innerSize = kernel.innerSize();
|
|
const Index outerSize = kernel.outerSize();
|
|
const Index packetSize = unpacket_traits<PacketType>::size;
|
|
for(Index outer = 0; outer < outerSize; ++outer)
|
|
for(Index inner = 0; inner < innerSize; inner+=packetSize)
|
|
kernel.template assignPacketByOuterInner<Aligned, Aligned, PacketType>(outer, inner);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
const Index outerSize = kernel.outerSize();
|
|
for(Index outer = 0; outer < outerSize; ++outer)
|
|
copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
|
|
}
|
|
};
|
|
|
|
/***********************
|
|
*** Linear traversal ***
|
|
***********************/
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
const Index size = kernel.size();
|
|
for(Index i = 0; i < size; ++i)
|
|
kernel.assignCoeff(i);
|
|
}
|
|
};
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
|
|
copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
|
|
}
|
|
};
|
|
|
|
/**************************
|
|
*** Slice vectorization ***
|
|
***************************/
|
|
|
|
template<typename Kernel>
|
|
struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling>
|
|
{
|
|
EIGEN_DEVICE_FUNC static inline void run(Kernel &kernel)
|
|
{
|
|
typedef typename Kernel::Scalar Scalar;
|
|
typedef typename Kernel::PacketType PacketType;
|
|
enum {
|
|
packetSize = unpacket_traits<PacketType>::size,
|
|
requestedAlignment = int(Kernel::AssignmentTraits::RequiredAlignment),
|
|
alignable = packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment)>=sizeof(Scalar),
|
|
dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
|
|
dstAlignment = alignable ? int(requestedAlignment)
|
|
: int(Kernel::AssignmentTraits::DstAlignment)
|
|
};
|
|
const Scalar *dst_ptr = &kernel.dstEvaluator().coeffRef(0,0);
|
|
if((!bool(dstIsAligned)) && (size_t(dst_ptr) % sizeof(Scalar))>0)
|
|
{
|
|
// the pointer is not aligend-on scalar, so alignment is not possible
|
|
return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel);
|
|
}
|
|
const Index packetAlignedMask = packetSize - 1;
|
|
const Index innerSize = kernel.innerSize();
|
|
const Index outerSize = kernel.outerSize();
|
|
const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
|
|
Index alignedStart = ((!alignable) || bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
|
|
|
|
for(Index outer = 0; outer < outerSize; ++outer)
|
|
{
|
|
const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
|
|
// do the non-vectorizable part of the assignment
|
|
for(Index inner = 0; inner<alignedStart ; ++inner)
|
|
kernel.assignCoeffByOuterInner(outer, inner);
|
|
|
|
// do the vectorizable part of the assignment
|
|
for(Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
|
|
kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
|
|
|
|
// do the non-vectorizable part of the assignment
|
|
for(Index inner = alignedEnd; inner<innerSize ; ++inner)
|
|
kernel.assignCoeffByOuterInner(outer, inner);
|
|
|
|
alignedStart = std::min<Index>((alignedStart+alignedStep)%packetSize, innerSize);
|
|
}
|
|
}
|
|
};
|
|
|
|
/***************************************************************************
|
|
* Part 4 : Generic dense assignment kernel
|
|
***************************************************************************/
|
|
|
|
// This class generalize the assignment of a coefficient (or packet) from one dense evaluator
|
|
// to another dense writable evaluator.
|
|
// It is parametrized by the two evaluators, and the actual assignment functor.
|
|
// This abstraction level permits to keep the evaluation loops as simple and as generic as possible.
|
|
// One can customize the assignment using this generic dense_assignment_kernel with different
|
|
// functors, or by completely overloading it, by-passing a functor.
|
|
template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized>
|
|
class generic_dense_assignment_kernel
|
|
{
|
|
protected:
|
|
typedef typename DstEvaluatorTypeT::XprType DstXprType;
|
|
typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
|
|
public:
|
|
|
|
typedef DstEvaluatorTypeT DstEvaluatorType;
|
|
typedef SrcEvaluatorTypeT SrcEvaluatorType;
|
|
typedef typename DstEvaluatorType::Scalar Scalar;
|
|
typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
|
|
typedef typename AssignmentTraits::PacketType PacketType;
|
|
|
|
|
|
EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr)
|
|
: m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
|
|
{
|
|
#ifdef EIGEN_DEBUG_ASSIGN
|
|
AssignmentTraits::debug();
|
|
#endif
|
|
}
|
|
|
|
EIGEN_DEVICE_FUNC Index size() const { return m_dstExpr.size(); }
|
|
EIGEN_DEVICE_FUNC Index innerSize() const { return m_dstExpr.innerSize(); }
|
|
EIGEN_DEVICE_FUNC Index outerSize() const { return m_dstExpr.outerSize(); }
|
|
EIGEN_DEVICE_FUNC Index rows() const { return m_dstExpr.rows(); }
|
|
EIGEN_DEVICE_FUNC Index cols() const { return m_dstExpr.cols(); }
|
|
EIGEN_DEVICE_FUNC Index outerStride() const { return m_dstExpr.outerStride(); }
|
|
|
|
EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() { return m_dst; }
|
|
EIGEN_DEVICE_FUNC const SrcEvaluatorType& srcEvaluator() const { return m_src; }
|
|
|
|
/// Assign src(row,col) to dst(row,col) through the assignment functor.
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index row, Index col)
|
|
{
|
|
m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
|
|
}
|
|
|
|
/// \sa assignCoeff(Index,Index)
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index index)
|
|
{
|
|
m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
|
|
}
|
|
|
|
/// \sa assignCoeff(Index,Index)
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeffByOuterInner(Index outer, Index inner)
|
|
{
|
|
Index row = rowIndexByOuterInner(outer, inner);
|
|
Index col = colIndexByOuterInner(outer, inner);
|
|
assignCoeff(row, col);
|
|
}
|
|
|
|
|
|
template<int StoreMode, int LoadMode, typename PacketType>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col)
|
|
{
|
|
m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(row,col));
|
|
}
|
|
|
|
template<int StoreMode, int LoadMode, typename PacketType>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index)
|
|
{
|
|
m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
|
|
}
|
|
|
|
template<int StoreMode, int LoadMode, typename PacketType>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner)
|
|
{
|
|
Index row = rowIndexByOuterInner(outer, inner);
|
|
Index col = colIndexByOuterInner(outer, inner);
|
|
assignPacket<StoreMode,LoadMode,PacketType>(row, col);
|
|
}
|
|
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner)
|
|
{
|
|
typedef typename DstEvaluatorType::ExpressionTraits Traits;
|
|
return int(Traits::RowsAtCompileTime) == 1 ? 0
|
|
: int(Traits::ColsAtCompileTime) == 1 ? inner
|
|
: int(DstEvaluatorType::Flags)&RowMajorBit ? outer
|
|
: inner;
|
|
}
|
|
|
|
EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner)
|
|
{
|
|
typedef typename DstEvaluatorType::ExpressionTraits Traits;
|
|
return int(Traits::ColsAtCompileTime) == 1 ? 0
|
|
: int(Traits::RowsAtCompileTime) == 1 ? inner
|
|
: int(DstEvaluatorType::Flags)&RowMajorBit ? inner
|
|
: outer;
|
|
}
|
|
|
|
protected:
|
|
DstEvaluatorType& m_dst;
|
|
const SrcEvaluatorType& m_src;
|
|
const Functor &m_functor;
|
|
// TODO find a way to avoid the needs of the original expression
|
|
DstXprType& m_dstExpr;
|
|
};
|
|
|
|
/***************************************************************************
|
|
* Part 5 : Entry point for dense rectangular assignment
|
|
***************************************************************************/
|
|
|
|
template<typename DstXprType, typename SrcXprType, typename Functor>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func)
|
|
{
|
|
eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
|
|
|
|
typedef evaluator<DstXprType> DstEvaluatorType;
|
|
typedef evaluator<SrcXprType> SrcEvaluatorType;
|
|
|
|
DstEvaluatorType dstEvaluator(dst);
|
|
SrcEvaluatorType srcEvaluator(src);
|
|
|
|
typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel;
|
|
Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
|
|
|
|
dense_assignment_loop<Kernel>::run(kernel);
|
|
}
|
|
|
|
template<typename DstXprType, typename SrcXprType>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src)
|
|
{
|
|
call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar>());
|
|
}
|
|
|
|
/***************************************************************************
|
|
* Part 6 : Generic assignment
|
|
***************************************************************************/
|
|
|
|
// Based on the respective shapes of the destination and source,
|
|
// the class AssignmentKind determine the kind of assignment mechanism.
|
|
// AssignmentKind must define a Kind typedef.
|
|
template<typename DstShape, typename SrcShape> struct AssignmentKind;
|
|
|
|
// Assignement kind defined in this file:
|
|
struct Dense2Dense {};
|
|
struct EigenBase2EigenBase {};
|
|
|
|
template<typename,typename> struct AssignmentKind { typedef EigenBase2EigenBase Kind; };
|
|
template<> struct AssignmentKind<DenseShape,DenseShape> { typedef Dense2Dense Kind; };
|
|
|
|
// This is the main assignment class
|
|
template< typename DstXprType, typename SrcXprType, typename Functor,
|
|
typename Kind = typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape , typename evaluator_traits<SrcXprType>::Shape >::Kind,
|
|
typename Scalar = typename DstXprType::Scalar>
|
|
struct Assignment;
|
|
|
|
|
|
// The only purpose of this call_assignment() function is to deal with noalias() / "assume-aliasing" and automatic transposition.
|
|
// Indeed, I (Gael) think that this concept of "assume-aliasing" was a mistake, and it makes thing quite complicated.
|
|
// So this intermediate function removes everything related to "assume-aliasing" such that Assignment
|
|
// does not has to bother about these annoying details.
|
|
|
|
template<typename Dst, typename Src>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment(Dst& dst, const Src& src)
|
|
{
|
|
call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>());
|
|
}
|
|
template<typename Dst, typename Src>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment(const Dst& dst, const Src& src)
|
|
{
|
|
call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>());
|
|
}
|
|
|
|
// Deal with "assume-aliasing"
|
|
template<typename Dst, typename Src, typename Func>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if< evaluator_assume_aliasing<Src>::value, void*>::type = 0)
|
|
{
|
|
typename plain_matrix_type<Src>::type tmp(src);
|
|
call_assignment_no_alias(dst, tmp, func);
|
|
}
|
|
|
|
template<typename Dst, typename Src, typename Func>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<!evaluator_assume_aliasing<Src>::value, void*>::type = 0)
|
|
{
|
|
call_assignment_no_alias(dst, src, func);
|
|
}
|
|
|
|
// by-pass "assume-aliasing"
|
|
// When there is no aliasing, we require that 'dst' has been properly resized
|
|
template<typename Dst, template <typename> class StorageBase, typename Src, typename Func>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment(NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func)
|
|
{
|
|
call_assignment_no_alias(dst.expression(), src, func);
|
|
}
|
|
|
|
|
|
template<typename Dst, typename Src, typename Func>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func)
|
|
{
|
|
enum {
|
|
NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
|
|
|| (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)
|
|
) && int(Dst::SizeAtCompileTime) != 1
|
|
};
|
|
|
|
Index dstRows = NeedToTranspose ? src.cols() : src.rows();
|
|
Index dstCols = NeedToTranspose ? src.rows() : src.cols();
|
|
if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
|
|
dst.resize(dstRows, dstCols);
|
|
|
|
typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned;
|
|
typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType;
|
|
ActualDstType actualDst(dst);
|
|
|
|
// TODO check whether this is the right place to perform these checks:
|
|
EIGEN_STATIC_ASSERT_LVALUE(Dst)
|
|
EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src)
|
|
EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename ActualDstTypeCleaned::Scalar,typename Src::Scalar);
|
|
|
|
Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
|
|
}
|
|
template<typename Dst, typename Src>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment_no_alias(Dst& dst, const Src& src)
|
|
{
|
|
call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar>());
|
|
}
|
|
|
|
template<typename Dst, typename Src, typename Func>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func)
|
|
{
|
|
Index dstRows = src.rows();
|
|
Index dstCols = src.cols();
|
|
if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
|
|
dst.resize(dstRows, dstCols);
|
|
|
|
// TODO check whether this is the right place to perform these checks:
|
|
EIGEN_STATIC_ASSERT_LVALUE(Dst)
|
|
EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src)
|
|
|
|
Assignment<Dst,Src,Func>::run(dst, src, func);
|
|
}
|
|
template<typename Dst, typename Src>
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src)
|
|
{
|
|
call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar>());
|
|
}
|
|
|
|
// forward declaration
|
|
template<typename Dst, typename Src> void check_for_aliasing(const Dst &dst, const Src &src);
|
|
|
|
// Generic Dense to Dense assignment
|
|
template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar>
|
|
struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Scalar>
|
|
{
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
static void run(DstXprType &dst, const SrcXprType &src, const Functor &func)
|
|
{
|
|
eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
|
|
|
|
#ifndef EIGEN_NO_DEBUG
|
|
internal::check_for_aliasing(dst, src);
|
|
#endif
|
|
|
|
call_dense_assignment_loop(dst, src, func);
|
|
}
|
|
};
|
|
|
|
// Generic assignment through evalTo.
|
|
// TODO: not sure we have to keep that one, but it helps porting current code to new evaluator mechanism.
|
|
template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar>
|
|
struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Scalar>
|
|
{
|
|
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
|
static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/)
|
|
{
|
|
eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
|
|
src.evalTo(dst);
|
|
}
|
|
};
|
|
|
|
} // namespace internal
|
|
|
|
} // end namespace Eigen
|
|
|
|
#endif // EIGEN_ASSIGN_EVALUATOR_H
|