glTF morph animation proof of concept

Some things to look at:
- Normal recalculation for vertex morphing
- Bounding boxes (esp. after rebasing this upon the other PR)

irr/include/SSkinMeshBuffer.h

@@ -14,6 +14,7 @@
 #include <cassert>
 #include <cstddef>
 #include <cstring>
+#include <optional>
 #include <vector>
 
 namespace irr
@@ -373,10 +374,15 @@ public:
 	//! Call this after changing the positions of any vertex.
 	void boundingBoxNeedsRecalculated() { BoundingBoxNeedsRecalculated = true; }
 
-	void morph(const std::vector<f32> &weights)
+	void setMorph(const std::optional<std::vector<f32>> &weights)
+	{
+		resetMorph();
+		addMorph(weights.value_or(DefaultWeights));
+	}
+
+	void addMorph(const std::vector<f32> &weights)
 	{
 		assert(weights.size() == MorphTargets.size());
-		resetMorph();
 		for (size_t i = 0; i < weights.size(); ++i) {
 			MorphTargets[i].add(getVertexBuffer(), weights[i]);
 			if (!MorphTargets[i].positions.empty())
@@ -401,6 +407,7 @@ public:
 	// TODO consolidate with Static(Pos|Normal) in weights
 	MorphTargetDelta MorphStaticPose;
 	std::vector<MorphTargetDelta> MorphTargets;
+	std::vector<f32> DefaultWeights;
 
 	video::SMaterial Material;
 	video::E_VERTEX_TYPE VertexType;

irr/include/SkinnedMesh.h

@@ -11,6 +11,7 @@
 #include "quaternion.h"
 #include "vector3d.h"
 
+#include <cstddef>
 #include <optional>
 #include <string>
 
@@ -166,8 +167,6 @@ public:
 		//! Internal members used by SkinnedMesh
 		friend class SkinnedMesh;
 		char *Moved;
-		core::vector3df StaticPos;
-		core::vector3df StaticNormal;
 	};
 
 	template <class T>
@@ -252,26 +251,98 @@ public:
 		}
 	};
 
+	struct WeightChannel
+	{
+		std::vector<f32> timestamps;
+		std::vector<f32> weights;
+		bool interpolate = true;
+		size_t n_weights;
+
+		bool empty() const {
+			return timestamps.empty();
+		}
+
+		f32 getEndFrame() const {
+			return timestamps.empty() ? 0 : timestamps.back();
+		}
+
+		void reserve(size_t n) {
+			timestamps.reserve(n);
+			weights.reserve(n);
+		}
+
+		void cleanup() {
+			timestamps.shrink_to_fit();
+			weights.shrink_to_fit();
+		}
+
+		static core::vector3df interpolateValue(core::vector3df from, core::vector3df to, f32 time) {
+			// Note: `from` and `to` are swapped here compared to quaternion slerp
+			return to.getInterpolated(from, time);
+		}
+
+		std::optional<std::vector<f32>> get(f32 time) const {
+			if (empty())
+				return std::nullopt;
+
+			const size_t i = std::distance(timestamps.begin(),
+					std::lower_bound(timestamps.begin(), timestamps.end(), time));
+			if (i == 0)
+				return getWeightsVec(0);
+			if (i == timestamps.size())
+				return getWeightsVec(i - 1);
+
+			auto res = getWeightsVec(i - 1);
+			if (!interpolate)
+				return res;
+
+			f32 prev_time = timestamps[i - 1];
+			f32 next_time = timestamps[i];
+			f32 progress = (time - prev_time) / (next_time - prev_time);
+			for (size_t j = 0; j < n_weights; ++j) {
+				res[j] = progress * *(getWeights(i) + j) + (1.0f - progress) * res[j];
+			}
+			return res;
+		}
+
+private:
+		std::vector<f32>::const_iterator getWeights(size_t i) const
+		{
+			return weights.begin() + i * n_weights;
+		}
+
+		std::vector<f32> getWeightsVec(size_t i) const
+		{
+			std::vector<f32> res;
+			res.insert(res.begin(), getWeights(i), getWeights(i + 1));
+			return res;
+		}
+	};
+
 	struct Keys {
 		Channel<core::vector3df> position;
 		Channel<core::quaternion> rotation;
 		Channel<core::vector3df> scale;
+		WeightChannel weights;
 
 		bool empty() const {
-			return position.empty() && rotation.empty() && scale.empty();
+			return position.empty() && rotation.empty() && scale.empty() && weights.empty();
 		}
 
 		void append(const Keys &other) {
 			position.append(other.position);
 			rotation.append(other.rotation);
 			scale.append(other.scale);
+			// This is only used by .x, which has no morph animation
+			assert(other.weights.empty());
 		}
 
 		f32 getEndFrame() const {
 			return std::max({
 				position.getEndFrame(),
 				rotation.getEndFrame(),
-				scale.getEndFrame()
+				scale.getEndFrame(),
+				weights.getEndFrame(),
 			});
 		}
 
@@ -290,6 +361,7 @@ public:
 			position.cleanup();
 			rotation.cleanup();
 			scale.cleanup();
+			weights.cleanup();
 		}
 	};
 
@@ -307,6 +379,9 @@ public:
 		//! List of child joints
 		std::vector<SJoint *> Children;
 
+		//! List of meshes which may be affected by morph animations targeting this joint
+		std::vector<size_t> MorphedMeshes;
+
 		//! List of attached meshes
 		std::vector<u32> AttachedMeshes;
 
@@ -360,6 +435,9 @@ protected:
 	std::vector<SSkinMeshBuffer *> *SkinningBuffers; // Meshbuffer to skin, default is to skin localBuffers
 
 	std::vector<SSkinMeshBuffer *> LocalBuffers;
+	//! Buffers involved in skinning.
+	//! These have a MorphStaticPose which can be used to reset positions & normals.
+	std::vector<u32> SkinnedBuffers;
 	//! Mapping from meshbuffer number to bindable texture slot
 	std::vector<u32> TextureSlots;
 

irr/src/CGLTFMeshFileLoader.cpp

@@ -440,6 +440,7 @@ void SelfType::MeshExtractor::addPrimitive(
 	const auto meshbufNr = m_irr_model->getMeshBufferCount() - 1;
 
 	if (auto morphTargets = primitive.targets) {
+		parent->MorphedMeshes.push_back(meshbufNr);
 		MorphTargetDelta::Flags used;
 		std::vector<MorphTargetDelta> deltas;
 		deltas.reserve(morphTargets->size());
@@ -492,8 +493,8 @@ void SelfType::MeshExtractor::addPrimitive(
 			weights.reserve(morphWeights->size());
 			for (f64 weight : *morphWeights)
 				weights.push_back(static_cast<f32>(weight));
-			// TODO does some unnecessary work - resets what we just read
+			meshbuf->DefaultWeights = weights;
-			meshbuf->morph(weights);
+			meshbuf->addMorph(weights);
 		}
 	}
 
@@ -522,6 +523,11 @@ void SelfType::MeshExtractor::addPrimitive(
 		return;
 	}
 
+	if (!primitive.targets) {
+		// We use morphing to reset to static pose.
+		parent->MorphedMeshes.push_back(meshbufNr);
+	}
+
 	// Otherwise: "Only the joint transforms are applied to the skinned mesh;
 	// the transform of the skinned mesh node MUST be ignored."
 
@@ -708,6 +714,18 @@ void SelfType::MeshExtractor::loadSkins()
 	}
 }
 
+static size_t countMorphTargets(const tiniergltf::GlTF &gltfModel, size_t nodeIdx)
+{
+	const auto &node = gltfModel.nodes->at(nodeIdx);
+	const auto &mesh = gltfModel.meshes->at(node.mesh.value());
+	size_t morph_targets = 0;
+	for (const auto &primitive : mesh.primitives) {
+		if (primitive.targets)
+			morph_targets = std::max(morph_targets, primitive.targets->size());
+	}
+	return morph_targets;
+}
+
 void SelfType::MeshExtractor::loadAnimation(const std::size_t animIdx)
 {
 	const auto &anim = m_gltf_model.animations->at(animIdx);
@@ -731,8 +749,9 @@ void SelfType::MeshExtractor::loadAnimation(const std::size_t animIdx)
 
 		if (!channel.target.node.has_value())
 			throw std::runtime_error("no animated node");
+		size_t targetNodeIdx = *channel.target.node;
 
-		auto *joint = m_loaded_nodes.at(*channel.target.node);
+		auto *joint = m_loaded_nodes.at(targetNodeIdx);
 		switch (channel.target.path) {
 		case tiniergltf::AnimationChannelTarget::Path::TRANSLATION: {
 			const auto outputAccessor = Accessor<core::vector3df>::make(m_gltf_model, sampler.output);
@@ -767,8 +786,24 @@ void SelfType::MeshExtractor::loadAnimation(const std::size_t animIdx)
 			}
 			break;
 		}
-		case tiniergltf::AnimationChannelTarget::Path::WEIGHTS:
+		case tiniergltf::AnimationChannelTarget::Path::WEIGHTS: {
-			throw std::runtime_error("no support for morph animations");
+			// FIXME support normalized bytes and stuff
+			const auto outputAccessor = Accessor<f32>::make(m_gltf_model, sampler.output);
+			const size_t count = outputAccessor.getCount();
+			auto &channel = joint->keys.weights;
+			channel.n_weights = countMorphTargets(m_gltf_model, targetNodeIdx);
+			if (channel.n_weights == 0)
+				throw std::runtime_error("missing morph targets for morph animation");
+			if (count % channel.n_weights != 0)
+				throw std::runtime_error("wrong number of values in morph weight accessor");
+			for (size_t i = 0; i < count / channel.n_weights; ++i) {
+				channel.timestamps.push_back(inputAccessor.get(i));
+				for (size_t j = 0; j < channel.n_weights; ++j) {
+					channel.weights.push_back(outputAccessor.get(i * channel.n_weights + j));
+				}
+			}
+			break;
+		}
 		}
 	}
 }

irr/src/SkinnedMesh.cpp

@@ -8,6 +8,8 @@
 #include "IAnimatedMeshSceneNode.h"
 #include "SSkinMeshBuffer.h"
 #include "os.h"
+#include <bitset>
+#include <unordered_set>
 #include <vector>
 #include <cassert>
 
@@ -73,6 +75,9 @@ void SkinnedMesh::animateMesh(f32 frame)
 	LastAnimatedFrame = frame;
 	SkinnedLastFrame = false;
 
+	// TODO do something if there are too many mesh buffers
+	std::bitset<256> morphed_buffers;
+
 	for (auto *joint : AllJoints) {
 		// The joints can be animated here with no input from their
 		// parents, but for setAnimationMode extra checks are needed
@@ -81,6 +86,17 @@ void SkinnedMesh::animateMesh(f32 frame)
 				joint->Animatedposition,
 				joint->Animatedrotation,
 				joint->Animatedscale);
+		auto weights = joint->keys.weights.get(frame);
+		for (size_t meshbufIdx : joint->MorphedMeshes) {
+			LocalBuffers[meshbufIdx]->setMorph(weights);
+			morphed_buffers.set(meshbufIdx);
+		}
+	}
+
+	for (size_t meshbufIdx : SkinnedBuffers) {
+		if (!morphed_buffers.test(meshbufIdx)) {
+			LocalBuffers[meshbufIdx]->setMorph(std::nullopt);
+		}
 	}
 
 	// Note:
@@ -226,11 +242,13 @@ void SkinnedMesh::skinJoint(SJoint *joint, SJoint *parentJoint)
 
 		// Skin Vertices Positions and Normals...
 		for (const auto &weight : joint->Weights) {
+			auto *buf = buffersUsed[weight.buffer_id];
 			// Pull this vertex...
-			jointVertexPull.transformVect(thisVertexMove, weight.StaticPos);
+			auto *vertex = buf->getVertex(weight.vertex_id);
+			jointVertexPull.transformVect(thisVertexMove, vertex->Pos);
 
 			if (AnimateNormals) {
-				thisNormalMove = jointVertexPull.rotateAndScaleVect(weight.StaticNormal);
+				thisNormalMove = jointVertexPull.rotateAndScaleVect(vertex->Normal);
 				thisNormalMove.normalize(); // must renormalize after potentially scaling
 			}
 
@@ -345,12 +363,8 @@ bool SkinnedMesh::setHardwareSkinning(bool on)
 
 			// set mesh to static pose...
 			for (auto *joint : AllJoints) {
-				for (const auto &weight : joint->Weights) {
+				for (auto buf_id : joint->MorphedMeshes) {
-					const u16 buffer_id = weight.buffer_id;
+					LocalBuffers[buf_id]->setMorph(std::nullopt);
-					const u32 vertex_id = weight.vertex_id;
-					LocalBuffers[buffer_id]->getVertex(vertex_id)->Pos = weight.StaticPos;
-					LocalBuffers[buffer_id]->getVertex(vertex_id)->Normal = weight.StaticNormal;
-					LocalBuffers[buffer_id]->boundingBoxNeedsRecalculated();
 				}
 			}
 		}
@@ -360,15 +374,14 @@ bool SkinnedMesh::setHardwareSkinning(bool on)
 	return HardwareSkinning;
 }
 
+// TODO the only usage of this only needs to refresh normals
 void SkinnedMesh::refreshJointCache()
 {
 	// copy cache from the mesh...
 	for (auto *joint : AllJoints) {
-		for (auto &weight : joint->Weights) {
+		for (auto buf_id : joint->MorphedMeshes) {
-			const u16 buffer_id = weight.buffer_id;
+			auto *buf = LocalBuffers[buf_id];
-			const u32 vertex_id = weight.vertex_id;
+			buf->MorphStaticPose.get(buf->getVertexBuffer(), {true, true, false});
-			weight.StaticPos = LocalBuffers[buffer_id]->getVertex(vertex_id)->Pos;
-			weight.StaticNormal = LocalBuffers[buffer_id]->getVertex(vertex_id)->Normal;
 		}
 	}
 }
@@ -377,11 +390,8 @@ void SkinnedMesh::resetAnimation()
 {
 	// copy from the cache to the mesh...
 	for (auto *joint : AllJoints) {
-		for (const auto &weight : joint->Weights) {
+		for (auto buf_id : joint->MorphedMeshes) {
-			const u16 buffer_id = weight.buffer_id;
+			LocalBuffers[buf_id]->setMorph(std::nullopt);
-			const u32 vertex_id = weight.vertex_id;
-			LocalBuffers[buffer_id]->getVertex(vertex_id)->Pos = weight.StaticPos;
-			LocalBuffers[buffer_id]->getVertex(vertex_id)->Normal = weight.StaticNormal;
 		}
 	}
 	SkinnedLastFrame = false;
@@ -449,6 +459,8 @@ void SkinnedMesh::checkForAnimation()
 	if (HasAnimation && !PreparedForSkinning) {
 		PreparedForSkinning = true;
 
+		std::unordered_set<u16> bufs;
+
 		// check for bugs:
 		for (auto *joint : AllJoints) {
 			for (auto &weight : joint->Weights) {
@@ -463,6 +475,8 @@ void SkinnedMesh::checkForAnimation()
 					os::Printer::log("Skinned Mesh: Weight vertex id too large", ELL_WARNING);
 					weight.buffer_id = weight.vertex_id = 0;
 				}
+
+				bufs.insert(weight.buffer_id);
 			}
 		}
 
@@ -472,18 +486,20 @@ void SkinnedMesh::checkForAnimation()
 			for (u32 j = 0; j < Vertices_Moved[i].size(); ++j)
 				Vertices_Moved[i][j] = false;
 
-		// For skinning: cache weight values for speed
+		for (auto id : bufs) {
+			SkinnedBuffers.emplace_back(id);
+			auto &msp = LocalBuffers[id]->MorphStaticPose;
+			// Make sure we have static pose data for positions & normals
+			msp.get(LocalBuffers[id]->getVertexBuffer(), {
+				msp.positions.empty(),
+				msp.normals.empty(),
+				false,
+			});
+		}
 
 		for (auto *joint : AllJoints) {
 			for (auto &weight : joint->Weights) {
-				const u16 buffer_id = weight.buffer_id;
+				weight.Moved = &Vertices_Moved[weight.buffer_id][weight.vertex_id];
-				const u32 vertex_id = weight.vertex_id;
-
-				weight.Moved = &Vertices_Moved[buffer_id][vertex_id];
-				weight.StaticPos = LocalBuffers[buffer_id]->getVertex(vertex_id)->Pos;
-				weight.StaticNormal = LocalBuffers[buffer_id]->getVertex(vertex_id)->Normal;
-
-				// weight._Pos=&Buffers[buffer_id]->getVertex(vertex_id)->Pos;
 			}
 		}