Fix RowMajorBit <-> RowMajor mixup.

(cherry picked from commit 40eb34bc5dea506fdbb2508c35d9965bb8c68013)
This commit is contained in:
Tobias Schlüter 2022-03-17 15:28:12 +00:00 committed by Antonio Sanchez
parent c473d69d22
commit 5cb2dfec1d

View File

@ -161,13 +161,13 @@ void upperbidiagonalization_blocked_helper(MatrixType& A,
typedef typename MatrixType::Scalar Scalar;
typedef typename MatrixType::RealScalar RealScalar;
typedef typename NumTraits<RealScalar>::Literal Literal;
enum { StorageOrder = traits<MatrixType>::Flags & RowMajorBit };
typedef InnerStride<int(StorageOrder) == int(ColMajor) ? 1 : Dynamic> ColInnerStride;
typedef InnerStride<int(StorageOrder) == int(ColMajor) ? Dynamic : 1> RowInnerStride;
enum { StorageOrder = (traits<MatrixType>::Flags & RowMajorBit) ? RowMajor : ColMajor };
typedef InnerStride<StorageOrder == ColMajor ? 1 : Dynamic> ColInnerStride;
typedef InnerStride<StorageOrder == ColMajor ? Dynamic : 1> RowInnerStride;
typedef Ref<Matrix<Scalar, Dynamic, 1>, 0, ColInnerStride> SubColumnType;
typedef Ref<Matrix<Scalar, 1, Dynamic>, 0, RowInnerStride> SubRowType;
typedef Ref<Matrix<Scalar, Dynamic, Dynamic, StorageOrder > > SubMatType;
Index brows = A.rows();
Index bcols = A.cols();
@ -293,7 +293,7 @@ void upperbidiagonalization_inplace_blocked(MatrixType& A, BidiagType& bidiagona
Index size = (std::min)(rows, cols);
// X and Y are work space
enum { StorageOrder = traits<MatrixType>::Flags & RowMajorBit };
enum { StorageOrder = (traits<MatrixType>::Flags & RowMajorBit) ? RowMajor : ColMajor };
Matrix<Scalar,
MatrixType::RowsAtCompileTime,
Dynamic,