FIX: normal_matrix calculation lack transpose in overhang

Jira: STUDIO-4244
Change-Id: If706f098c43f15aa407dde68e8a914bd53ab7188
This commit is contained in:
zhou.xu 2023-08-29 15:22:27 +08:00 committed by Lane.Wei
parent f942d5d55d
commit 7df99021c1

View File

@ -290,7 +290,8 @@ void TriangleSelector::select_patch(int facet_start, std::unique_ptr<Cursor> &&c
while (facet_idx < int(facets_to_check.size())) {
int facet = facets_to_check[facet_idx];
const Vec3f& facet_normal = m_face_normals[m_triangles[facet].source_triangle];
float world_normal_z = (trafo_no_translate.inverse() * facet_normal.cast<double>()).normalized().cast<float>().z();
Matrix3f normal_matrix = static_cast<Matrix3f>(trafo_no_translate.matrix().block(0, 0, 3, 3).inverse().transpose().cast<float>());
float world_normal_z = (normal_matrix* facet_normal).normalized().z();
if (!visited[facet] && (highlight_by_angle_deg == 0.f || world_normal_z < highlight_angle_limit)) {
if (select_triangle(facet, new_state, triangle_splitting)) {
// add neighboring facets to list to be processed later
@ -339,7 +340,8 @@ void TriangleSelector::seed_fill_select_triangles(const Vec3f &hit, int facet_st
facet_queue.pop();
const Vec3f &facet_normal = m_face_normals[m_triangles[current_facet].source_triangle];
float world_normal_z = (trafo_no_translate.inverse() * facet_normal.cast<double>()).normalized().cast<float>().z();
Matrix3f normal_matrix = static_cast<Matrix3f>(trafo_no_translate.matrix().block(0, 0, 3, 3).inverse().transpose().cast<float>());
float world_normal_z = (normal_matrix * facet_normal).normalized().z();
if (!visited[current_facet] && (highlight_by_angle_deg == 0.f || world_normal_z < highlight_angle_limit)) {
if (m_triangles[current_facet].is_split()) {
for (int split_triangle_idx = 0; split_triangle_idx <= m_triangles[current_facet].number_of_split_sides(); ++split_triangle_idx) {