fix bug #127. our product selection logic was flawed in that it used the Max-sized to determine whether the size is 1.

+ test.

Eigen/src/Core/Product.h

@@ -57,23 +57,39 @@ namespace internal {
 
 template<int Rows, int Cols, int Depth> struct product_type_selector;
 
+template<int Size, int MaxSize> struct product_size_category
+{
+  enum { is_large = MaxSize == Dynamic ||
+                    Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD,
+         value = is_large  ? Large
+               : Size == 1 ? 1
+                           : Small
+  };
+};
+
 template<typename Lhs, typename Rhs> struct product_type
 {
   typedef typename remove_all<Lhs>::type _Lhs;
   typedef typename remove_all<Rhs>::type _Rhs;
   enum {
-    Rows  = _Lhs::MaxRowsAtCompileTime,
+    MaxRows  = _Lhs::MaxRowsAtCompileTime,
-    Cols  = _Rhs::MaxColsAtCompileTime,
+    Rows  = _Lhs::RowsAtCompileTime,
-    Depth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::MaxColsAtCompileTime,_Rhs::MaxRowsAtCompileTime)
+    MaxCols  = _Rhs::MaxColsAtCompileTime,
+    Cols  = _Rhs::ColsAtCompileTime,
+    MaxDepth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::MaxColsAtCompileTime,
+                                           _Rhs::MaxRowsAtCompileTime),
+    Depth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::ColsAtCompileTime,
+                                        _Rhs::RowsAtCompileTime),
+    LargeThreshold = EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
   };
 
   // the splitting into different lines of code here, introducing the _select enums and the typedef below,
   // is to work around an internal compiler error with gcc 4.1 and 4.2.
 private:
   enum {
-    rows_select   = Rows == Dynamic || Rows >=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ? Large : (Rows==1   ? 1 : Small),
+    rows_select = product_size_category<Rows,MaxRows>::value,
-    cols_select   = Cols == Dynamic || Cols >=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ? Large : (Cols==1   ? 1 : Small),
+    cols_select = product_size_category<Cols,MaxCols>::value,
-    depth_select  = Depth == Dynamic || Depth>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ? Large : (Depth==1  ? 1 : Small)
+    depth_select = product_size_category<Depth,MaxDepth>::value
   };
   typedef product_type_selector<rows_select, cols_select, depth_select> selector;
 

test/product_extra.cpp

@@ -116,6 +116,30 @@ template<typename MatrixType> void product_extra(const MatrixType& m)
   VERIFY_IS_APPROX(tmp, m1 * m1.adjoint() * s1);
 }
 
+void zero_sized_objects()
+{
+  // Bug 127
+  //
+  // a product of the form lhs*rhs with
+  //
+  // lhs:
+  // rows = 1, cols = 4
+  // RowsAtCompileTime = 1, ColsAtCompileTime = -1
+  // MaxRowsAtCompileTime = 1, MaxColsAtCompileTime = 5
+  //
+  // rhs:
+  // rows = 4, cols = 0
+  // RowsAtCompileTime = -1, ColsAtCompileTime = -1
+  // MaxRowsAtCompileTime = 5, MaxColsAtCompileTime = 1
+  //
+  // was failing on a runtime assertion, because it had been mis-compiled as a dot product because Product.h was using the
+  // max-sizes to detect size 1 indicating vectors, and that didn't account for 0-sized object with max-size 1.
+
+  Matrix<float,1,Dynamic,RowMajor,1,5> a(1,4);
+  Matrix<float,Dynamic,Dynamic,ColMajor,5,1> b(4,0);
+  a*b;
+}
+
 void test_product_extra()
 {
   for(int i = 0; i < g_repeat; i++) {
@@ -123,5 +147,6 @@ void test_product_extra()
     CALL_SUBTEST_2( product_extra(MatrixXd(internal::random<int>(1,320), internal::random<int>(1,320))) );
     CALL_SUBTEST_3( product_extra(MatrixXcf(internal::random<int>(1,150), internal::random<int>(1,150))) );
     CALL_SUBTEST_4( product_extra(MatrixXcd(internal::random<int>(1,150), internal::random<int>(1,150))) );
+    CALL_SUBTEST_5( zero_sized_objects() );
   }
 }