Morph target fixes for CMF Viewer

src/viewer/assets/shaders/geometryprepass.comp

@@ -2,6 +2,8 @@
 
 #version 450
 
+#include "packed_tangent.inc"
+
 // mapping to CMF ElementType
 const uint FLOAT32_TYPE = 0;
 const uint FLOAT16_TYPE = 1;
@@ -14,6 +16,18 @@ const uint UINT8_TYPE = 7;
 const uint INT8NORM_TYPE = 8;
 const uint INT8_TYPE = 9;
 
+// mapping to CMF UsageType
+const uint USAGE_POSITION = 0;
+const uint USAGE_NORMAL = 1;
+const uint USAGE_TANGENT = 2;
+const uint USAGE_BINORMAL = 3;
+const uint USAGE_TEXCOORD = 4;
+const uint USAGE_COLOR = 5;
+const uint USAGE_BONEINDICES = 6;
+const uint USAGE_BONEWEIGHTS = 7;
+const uint USAGE_PACKEDTANGENT = 8;
+const uint USAGE_PACKEDTANGENTLEGACY = 9;
+
 // The buffers are all 4 byte aligned
 // meaning we can safely cast things from float to the desired type and back to float
 
@@ -42,6 +56,7 @@ struct Element {
     uint count; // 1-4 , how many components does this element have
     uint offset; // the offset of this element in the vertex
     uint normalized; // 0 = not normalized, 1 = normalized
+	uint usage; // the element usage, see USAGE_*** constants
 };
 
 layout(std430, binding = 1) buffer VertexIn {
@@ -291,6 +306,47 @@ vec4 ApplyWeight( float weight, const Element sourceElement, const Element morph
     return weight * ( morphValueAsFloat - sourceValueAsFloat );
 }
 
+TangentSpace UnpackTangentValue( vec4 value, const Element element )
+{
+    if( element.usage == USAGE_PACKEDTANGENTLEGACY )
+    {
+        return UnpackTangentSpaceLegacy( value );
+    }
+    return UnpackTangentSpace( value );
+}
+
+vec4 PackTangentValue( TangentSpace space, const Element element )
+{
+    space = Orthonormalize( space );
+
+    if( element.usage == USAGE_PACKEDTANGENTLEGACY )
+    {
+        return PackTangentSpaceLegacy( space );
+    }
+    return PackTangentSpace( space );
+}
+
+TangentSpace ApplyPackedTangentWeight( float weight, const Element sourceElement, const Element morphElement, uint vertexIndex, uint morphBufferIndex )
+{
+    uint sourceIndexStart = vertexIndex * params.vertexStride + sourceElement.offset;
+    uint morphIndexStart = morphBufferIndex * params.morphBufferStride + vertexIndex * params.morphVertexStride + morphElement.offset;
+
+	// morph elements have a requirement to not introduce new usages, so if the source element has packed tangents then the morph element has the same
+    TangentSpace sourceSpace = UnpackTangentValue( GetSourceValue( sourceIndexStart, sourceElement ), sourceElement );
+    TangentSpace morphSpace = UnpackTangentValue( GetMorphValue( morphIndexStart, morphElement ), morphElement );
+
+    TangentSpace change;
+    change.tangent = weight * ( morphSpace.tangent - sourceSpace.tangent );
+    change.binormal = weight * ( morphSpace.binormal - sourceSpace.binormal );
+    change.normal = weight * ( morphSpace.normal - sourceSpace.normal );
+    return change;
+}
+
+bool IsPackedTangentElement( const Element element )
+{
+    return element.usage == USAGE_PACKEDTANGENT || element.usage == USAGE_PACKEDTANGENTLEGACY;
+}
+
 void WriteOut( vec4 modifiedValue, const Element element, uint vertexAttributeIndex )
 {
     switch( element.type )
@@ -346,6 +402,12 @@ void main()
         Element sourceElement = elements[ morphJob.sourceElementIndex ];
         Element morphElement = elements[ morphJob.morphElementIndex ];
 
+		bool isPackedTangentElement = IsPackedTangentElement( sourceElement );
+        TangentSpace tangentChange;
+        tangentChange.tangent = vec3( 0.0f );
+        tangentChange.binormal = vec3( 0.0f );
+        tangentChange.normal = vec3( 0.0f );
+
         // step 1. morphing
         for( uint morphIndex = 0; morphIndex < params.morphBufferCount; ++morphIndex ) 
         {
@@ -355,16 +417,38 @@ void main()
                 continue;
             }
 
-            change += ApplyWeight( weight, sourceElement, morphElement, vertexIndex, morphIndex );
+            if( isPackedTangentElement )
+            {
+                TangentSpace weightedChange = ApplyPackedTangentWeight( weight, sourceElement, morphElement, vertexIndex, morphIndex );
+                tangentChange.tangent += weightedChange.tangent;
+                tangentChange.binormal += weightedChange.binormal;
+                tangentChange.normal += weightedChange.normal;
+            }
+            else
+            {
+                change += ApplyWeight( weight, sourceElement, morphElement, vertexIndex, morphIndex );
+            }            
         }
 
         uint vertexAttributeIndex = vertexIndex * params.vertexStride + sourceElement.offset;
 
         // step 2. apply the morphing to the source value
-        vec4 modifiedValue = GetSourceValue( vertexAttributeIndex, sourceElement ) + change;
-
+        vec4 modifiedValue;
+        if( isPackedTangentElement )
+        {
+            TangentSpace sourceSpace = UnpackTangentValue( GetSourceValue( vertexAttributeIndex, sourceElement ), sourceElement );
+            sourceSpace.tangent += tangentChange.tangent;
+            sourceSpace.binormal += tangentChange.binormal;
+            sourceSpace.normal += tangentChange.normal;
+            modifiedValue = PackTangentValue( sourceSpace, sourceElement );
+        }
+        else
+        {
+            modifiedValue = GetSourceValue( vertexAttributeIndex, sourceElement ) + change;
+        }
+
         // step 3. normalization if needed
-        if( sourceElement.normalized == 1 ) 
+        if( sourceElement.normalized == 1 && !isPackedTangentElement ) 
         {
             modifiedValue = normalize( modifiedValue );
         }

src/viewer/assets/shaders/packed_tangent.inc

@@ -11,6 +11,32 @@ vec2 SinCosFloat( float angle )
     return vec2( sin( angle ), cos( angle ) );
 }
 
+float RemapTangentAngle( float angle )
+{
+	return clamp( ( angle + 3.14159265359 ) / 6.28318530718, 0.0f, 1.0f );
+}
+
+vec4 PackTangentSpaceLegacy( TangentSpace space )
+{
+	vec4 angles;
+	angles.x = atan( space.tangent.y, space.tangent.x );
+	angles.y = acos( clamp( space.tangent.z, -1.0f, 1.0f ) );
+	angles.z = atan( space.binormal.y, space.binormal.x );
+	angles.w = acos( clamp( space.binormal.z, -1.0f, 1.0f ) );
+
+	if( dot( space.normal, cross( space.tangent, space.binormal ) ) < 0.0f )
+	{
+		angles.y = -angles.y;
+		angles.w = -angles.w;
+	}
+
+	return vec4(
+		RemapTangentAngle( angles.x ),
+		RemapTangentAngle( angles.y ),
+		RemapTangentAngle( angles.z ),
+		RemapTangentAngle( angles.w ) );
+}
+
 TangentSpace UnpackTangentSpaceLegacy( vec4 tangents )
 {
 	vec4 angles = tangents * 6.28318530718 - 3.14159265359;
@@ -30,6 +56,65 @@ TangentSpace UnpackTangentSpaceLegacy( vec4 tangents )
     return TangentSpace( tangent, binormal, normal );
 }
 
+vec4 PackTangentSpace( TangentSpace space )
+{
+	float normalSign = dot( space.normal, cross( space.tangent, space.binormal ) ) < 0.0f ? -1.0f : 1.0f;
+	vec3 normal = space.normal * normalSign;
+
+	float m00 = space.tangent.x;
+	float m01 = space.tangent.y;
+	float m02 = space.tangent.z;
+	float m10 = space.binormal.x;
+	float m11 = space.binormal.y;
+	float m12 = space.binormal.z;
+	float m20 = normal.x;
+	float m21 = normal.y;
+	float m22 = normal.z;
+
+	vec4 q;
+	float trace = m00 + m11 + m22;
+	if( trace > 0.0f )
+	{
+		float s = sqrt( trace + 1.0f ) * 2.0f;
+		q.x = ( m12 - m21 ) / s;
+		q.y = ( m20 - m02 ) / s;
+		q.z = ( m01 - m10 ) / s;
+		q.w = 0.25f * s;
+	}
+	else if( m00 > m11 && m00 > m22 )
+	{
+		float s = sqrt( 1.0f + m00 - m11 - m22 ) * 2.0f;
+		q.x = 0.25f * s;
+		q.y = ( m01 + m10 ) / s;
+		q.z = ( m20 + m02 ) / s;
+		q.w = ( m12 - m21 ) / s;
+	}
+	else if( m11 > m22 )
+	{
+		float s = sqrt( 1.0f + m11 - m00 - m22 ) * 2.0f;
+		q.x = ( m01 + m10 ) / s;
+		q.y = 0.25f * s;
+		q.z = ( m12 + m21 ) / s;
+		q.w = ( m20 - m02 ) / s;
+	}
+	else
+	{
+		float s = sqrt( 1.0f + m22 - m00 - m11 ) * 2.0f;
+		q.x = ( m20 + m02 ) / s;
+		q.y = ( m12 + m21 ) / s;
+		q.z = 0.25f * s;
+		q.w = ( m01 - m10 ) / s;
+	}
+
+	q = normalize( q );
+	if( q.w < 0.0f )
+	{
+		q = -q;
+	}
+
+	return vec4( q.xyz, normalSign );
+}
+
 TangentSpace UnpackTangentSpace( vec4 t ) {
 	// Heavily optimized shader code that constructs the TBN matrix.
 
@@ -63,4 +148,15 @@ TangentSpace UnpackTangentSpace( vec4 t ) {
 	space.normal = vec3( + yw + xz, + yz - xw, fma( -2.0f, x2, fma( -2.0f, y2, 1.0f ) ) ) * t.w; // packed normal sign multiplication
 
 	return space;
+}
+
+TangentSpace Orthonormalize( TangentSpace space )
+{
+    float handedness = dot( space.normal, cross( space.tangent, space.binormal ) ) < 0.0f ? -1.0f : 1.0f;
+
+	space.normal = normalize( space.normal );
+	space.tangent = normalize( space.tangent - space.normal * dot( space.normal, space.tangent ) );
+	space.binormal = cross( space.normal, space.tangent ) * handedness;
+
+    return space;
 }

src/viewer/rendering/renderable/geometryprepass.cpp

@@ -221,7 +221,7 @@ void GeometryPrePass::SetupForDynamicMesh( const cmf::MeshLod& lod )
 			vertexDecl.usage == cmf::Usage::Binormal ||
 			vertexDecl.usage == cmf::Usage::PackedTangent ||
 			vertexDecl.usage == cmf::Usage::PackedTangentLegacy;
-		Element element = { (uint32_t)vertexDecl.type, (uint32_t)vertexDecl.elementCount, vertexDecl.offset / 4, normalized };
+		Element element = { (uint32_t)vertexDecl.type, (uint32_t)vertexDecl.elementCount, vertexDecl.offset / 4, normalized, (uint32_t)vertexDecl.usage };
 		elements.push_back( element );
 
 		if( vertexDecl.usage == cmf::Usage::BoneIndices )
@@ -242,7 +242,7 @@ void GeometryPrePass::SetupForDynamicMesh( const cmf::MeshLod& lod )
 	for( const auto morphTargetDecl : m_cmfMesh.morphTargets.decl )
 	{
 		// no need to normalize the morph target data
-		elements.push_back( { (uint32_t)morphTargetDecl.type, (uint32_t)morphTargetDecl.elementCount, morphTargetDecl.offset / 4, false } );
+		elements.push_back( { (uint32_t)morphTargetDecl.type, (uint32_t)morphTargetDecl.elementCount, morphTargetDecl.offset / 4, false, (uint32_t)morphTargetDecl.usage } );
 	}
 
 	// gather all the morph jobs for this morph target, one job per vertex element

src/viewer/rendering/renderable/geometryprepass.h

@@ -69,6 +69,7 @@ class GeometryPrePass
 		uint32_t elementCount;
 		uint32_t offset;
 		uint32_t normalized;
+		uint32_t usage; // vertex attribute usage, for example position or normal
 	};
 
 	const cmf::Mesh m_cmfMesh;

src/viewer/rendering/renderable/mesh.cpp

@@ -157,6 +157,30 @@ void MeshRenderable::Initialize( AppState& appState )
 	m_initialized = true;
 }
 
+void MeshRenderable::SetAnimation( const cmf::Animation* animation )
+{
+	m_morphCurveToTargetMapping.clear();
+	if( !animation )
+	{
+		return;
+	}
+
+	for( const auto& channel : animation->channels )
+	{
+		if( channel.targetType == cmf::AnimationChannelTargetType::MorphTarget )
+		{
+			auto* morphTarget = std::find_if( m_cmfMesh.morphTargets.targets.begin(), m_cmfMesh.morphTargets.targets.end(), [&channel]( const cmf::MorphTarget& morphTarget ) {
+				return morphTarget.name == channel.target;
+			} );
+			if( morphTarget != m_cmfMesh.morphTargets.targets.end() )
+			{
+				const auto morphIndex = static_cast<uint32_t>( std::distance( m_cmfMesh.morphTargets.targets.begin(), morphTarget ) );
+				m_morphCurveToTargetMapping.emplace_back( channel.curveIndex, morphIndex );
+			}
+		}
+	}
+}
+
 void MeshRenderable::UpdateMeshCurves( float animationTime, const cmf::Animation* animation, AppState& appState )
 {
 	if( animation )

src/viewer/rendering/renderable/mesh.h

@@ -24,6 +24,7 @@ class MeshRenderable
 	void PrepareMesh( ComputeCommandBuffer& computeCommandBuffer );
 	VkResult SetRenderingMode( std::string shaderName, GraphicsEffectTypes::ShaderInputDeclaration shaderInputDeclaration, VkPolygonMode polygonMode );
 
+	void SetAnimation( const cmf::Animation* animation );
 	void UpdateMeshCurves( float animationTime, const cmf::Animation* animation, AppState& appState );
 	void SetSkeletonPose( const std::array<Matrix, 0xFF>& boneTransforms );
 	uint8_t GetSkeletonIndex() const;

src/viewer/rendering/renderable/model.cpp

@@ -42,6 +42,10 @@ VkResult ModelRenderable::Initialize( AppState& appState )
 
 	// when the active animation owner changes, we need to update the animation state and meshes
 	appState.modelState.activeAnimationOwner.RegisterCallback( [&]( std::shared_ptr<CmfContent> activeAnimationOwner, AppState& appState ) {
+		for( auto& mesh : m_meshes )
+		{
+			mesh.SetAnimation( nullptr );
+		}
 		for( auto& animationState : m_animationStates )
 		{
 			animationState.SetAnimationOwner( activeAnimationOwner );
@@ -69,6 +73,10 @@ VkResult ModelRenderable::Initialize( AppState& appState )
 			{
 				Log::Error( "Animation %s not found in active animation owner.", animationName.c_str() );
 			}
+			for( auto& mesh : m_meshes )
+			{
+				mesh.SetAnimation( nullptr );
+			}
 			for( auto& animationState : m_animationStates )
 			{
 				animationState.SetAnimation( nullptr );
@@ -78,6 +86,10 @@ VkResult ModelRenderable::Initialize( AppState& appState )
 		{
 			const auto& animation = *animationIt;
 
+			for( auto& mesh : m_meshes )
+			{
+				mesh.SetAnimation( &animation );
+			}
 			for( auto& animationState : m_animationStates )
 			{
 				animationState.SetAnimation( &animation );