Merge pull request #88 from you-win/feature/real-time-lip-sync-gap

Feature/real time lip sync

Former-commit-id: c235cbc

addons/real-time-lip-sync-gd/lip_sync.gdns

@@ -0,0 +1,8 @@
+[gd_resource type="NativeScript" load_steps=2 format=2]
+
+[ext_resource path="res://addons/real-time-lip-sync-gd/real_time_lip_sync.gdnlib" type="GDNativeLibrary" id=1]
+
+[resource]
+resource_name = "LipSync"
+class_name = "LipSync"
+library = ExtResource( 1 )

addons/real-time-lip-sync-gd/real_time_lip_sync.gdnlib

@@ -0,0 +1,14 @@
+[general]
+
+singleton=false
+load_once=true
+symbol_prefix="godot_"
+reloadable=true
+
+[entry]
+
+Windows.64="res://addons/real-time-lip-sync-gd/real_time_lip_sync_gd.dll"
+
+[dependencies]
+
+Windows.64=[  ]

entities/vrm/VRMModel.gd

@@ -57,6 +57,11 @@ var u: ExpressionData
 # TODO stopgap
 var last_expression: ExpressionData
 
+var current_mouth_shape: ExpressionData
+const VOWEL_HISTORY: int = 5 # TODO move to config
+const MIN_VOWEL_CHANGE: int = 3 # TODO move to config
+var last_vowels: Array = []
+
 var all_expressions: Dictionary = {} # String: ExpressionData
 
 ###############################################################################
@@ -79,6 +84,8 @@ func _ready() -> void:
 	# TODO stopgap
 	AppManager.sb.connect("blend_shapes", self, "_on_blend_shapes")
 
+	AppManager.sb.connect("lip_sync_updated", self, "_on_lip_sync_updated")
+
 	# Map expressions
 	var anim_player: AnimationPlayer = find_node("anim")
 
@@ -113,6 +120,8 @@ func _ready() -> void:
 
 	for key in all_expressions.keys():
 		set(key, all_expressions[key])
+
+	current_mouth_shape = a
 
 	_map_eye_expressions(all_expressions)
 
@@ -154,6 +163,53 @@ func _on_blend_shapes(value: String) -> void:
 
 	last_expression = ed
 
+func _on_lip_sync_updated(data: Dictionary) -> void:
+	for x in current_mouth_shape.morphs:
+		_modify_blend_shape(x.mesh, x.morph, 1)
+
+	last_vowels.push_back(data["vowel"])
+	if last_vowels.size() > VOWEL_HISTORY:
+		last_vowels.pop_front()
+
+	var vowel_count: Dictionary = {
+		"a": 0,
+		"i": 0,
+		"u": 0,
+		"e": 0,
+		"o": 0
+	}
+	for x in last_vowels:
+		match x:
+			0: # A
+				vowel_count.a += 1
+			1: # I
+				vowel_count.i += 1
+			2: # U
+				vowel_count.u += 1
+			3: # E
+				vowel_count.e += 1
+			4: # O
+				vowel_count.o += 1
+
+	var last_shape = current_mouth_shape
+
+	if vowel_count.a >= MIN_VOWEL_CHANGE:
+		current_mouth_shape = a
+	elif vowel_count.i >= MIN_VOWEL_CHANGE:
+		current_mouth_shape = i
+	elif vowel_count.u >= MIN_VOWEL_CHANGE:
+		current_mouth_shape = u
+	elif vowel_count.e >= MIN_VOWEL_CHANGE:
+		current_mouth_shape = e
+	elif vowel_count.o >= MIN_VOWEL_CHANGE:
+		current_mouth_shape = o
+
+	if current_mouth_shape != last_shape:
+		for x in current_mouth_shape.morphs:
+			_modify_blend_shape(x.mesh, x.morph, 1)
+		for x in last_shape.morphs:
+			_modify_blend_shape(x.mesh, x.morph, 0)
+
 ###############################################################################
 # Private functions                                                           #
 ###############################################################################
@@ -276,25 +332,25 @@ func custom_update(data, interpolation_data) -> void:
 		if (last_expression != joy and last_expression != sorrow):
 			# Left eye blinking
 			if data.left_eye_open >= blink_threshold:
-				for i in blink_r.morphs:
-					_modify_blend_shape(i.mesh, i.morph, i.values[1] - interpolation_data.interpolate(InterpolationData.InterpolationDataType.LEFT_EYE_BLINK, 1.0))
+				for x in blink_r.morphs:
+					_modify_blend_shape(x.mesh, x.morph, x.values[1] - interpolation_data.interpolate(InterpolationData.InterpolationDataType.LEFT_EYE_BLINK, 1.0))
 			else:
-				for i in blink_r.morphs:
-					_modify_blend_shape(i.mesh, i.morph, i.values[1])
+				for x in blink_r.morphs:
+					_modify_blend_shape(x.mesh, x.morph, x.values[1])
 
 			# Right eye blinking
 			if data.right_eye_open >= blink_threshold:
-				for i in blink_l.morphs:
-					_modify_blend_shape(i.mesh, i.morph, i.values[1] - interpolation_data.interpolate(InterpolationData.InterpolationDataType.RIGHT_EYE_BLINK, 1.0))
+				for x in blink_l.morphs:
+					_modify_blend_shape(x.mesh, x.morph, x.values[1] - interpolation_data.interpolate(InterpolationData.InterpolationDataType.RIGHT_EYE_BLINK, 1.0))
 			else:
-				for i in blink_l.morphs:
-					_modify_blend_shape(i.mesh, i.morph, i.values[1])
+				for x in blink_l.morphs:
+					_modify_blend_shape(x.mesh, x.morph, x.values[1])
 		else:
 			# Unblink if the facial expression doesn't allow blinking
-			for i in blink_r.morphs:
-				_modify_blend_shape(i.mesh, i.morph, i.values[0])
-			for i in blink_l.morphs:
-				_modify_blend_shape(i.mesh, i.morph, i.values[0])
+			for x in blink_r.morphs:
+				_modify_blend_shape(x.mesh, x.morph, x.values[0])
+			for x in blink_l.morphs:
+				_modify_blend_shape(x.mesh, x.morph, x.values[0])
 
 		# TODO eyes show weird behaviour when blinking
 		# TODO make sure angle between eyes' x values are at least parallel
@@ -340,10 +396,10 @@ func custom_update(data, interpolation_data) -> void:
 		skeleton.set_bone_pose(left_eye_id, right_eye_transform)
 
 		# Mouth tracking
-		for i in a.morphs:
-			_modify_blend_shape(i.mesh, i.morph,
-					min(max(i.values[0], interpolation_data.interpolate(InterpolationData.InterpolationDataType.MOUTH_MOVEMENT, 2.0)),
-					i.values[1]))
+		for x in current_mouth_shape.morphs:
+			_modify_blend_shape(x.mesh, x.morph,
+					min(max(x.values[0], interpolation_data.interpolate(InterpolationData.InterpolationDataType.MOUTH_MOVEMENT, 2.0)),
+					x.values[1]))
 	else:
 		# TODO implement eco mode, should be more efficient than standard mode
 		# Eco-mode blinking

project.godot

@@ -69,6 +69,11 @@ _global_script_classes=[ {
 "language": "GDScript",
 "path": "res://utils/JSONUtil.gd"
 }, {
+"base": "Node",
+"class": "LipSyncManager",
+"language": "GDScript",
+"path": "res://utils/LipSyncManager.gd"
+}, {
 "base": "Reference",
 "class": "Logger",
 "language": "GDScript",
@@ -122,6 +127,7 @@ _global_script_class_icons={
 "GOTHGui": "",
 "InterpolationData": "",
 "JSONUtil": "",
+"LipSyncManager": "",
 "Logger": "",
 "MainScreen": "",
 "ModelDisplayScreen": "",
@@ -138,6 +144,10 @@ config/name="OpenSeeFace GD"
 run/main_scene="res://screens/MainScreen.tscn"
 config/icon="res://assets/osfgd_icon.png"
 
+[audio]
+
+enable_audio_input=true
+
 [autoload]
 
 AppManager="*res://utils/AppManager.gd"

resources/gui/tracking.xml

@@ -37,6 +37,8 @@
         <input name="Tracker Address" data="tracker_address" event="tracker_address" type="string"/>
         <input name="Tracker Port" data="tracker_port" event="tracker_port" type="integer"/>
         <button name="Start Tracker" event="toggle_tracker" label_updatable="true"/>
+        <label name="Lip Sync"/>
+        <toggle name="Use lip sync" data="use_lip_sync" event="use_lip_sync"/>
         <!-- TODO this is bad, should refactor into a list -->
         <label name="Blend Shapes"/>
         <drop_down name="Blend Shapes" event="blend_shapes" setup="setup_blend_shapes"/>

screens/MainScreen.gd

@@ -39,7 +39,7 @@ func _ready() -> void:
 
 	AppManager.cm.metadata_config.apply_rendering_changes(get_viewport())
 
-	AppManager.logger.notify("Welcome to openseeface-gd!")
+	AppManager.logger.notify("Press TAB to hide the UI")
 
 func _unhandled_input(event: InputEvent) -> void:
 	if(event.is_action_pressed("ui_cancel") and OS.is_debug_build()):

screens/ModelDisplayScreen.gd

@@ -78,6 +78,8 @@ func _ready() -> void:
 		AppManager.sb.connect(i, self, "_on_%s" % i)
 		set(i, AppManager.cm.current_model_config.get(i))
 
+	AppManager.sb.connect("lip_sync_updated", self, "_on_lip_sync_updated")
+
 	if model_resource_path:
 		_try_load_model(model_resource_path)
 
@@ -218,6 +220,10 @@ func _on_apply_rotation(value: bool) -> void:
 func _on_should_track_eye(value: bool) -> void:
 	should_track_eye = value
 
+func _on_lip_sync_updated(_data: Dictionary) -> void:
+	interpolation_data.target_mouth_movement = 1
+	interpolation_data.interpolate(InterpolationData.InterpolationDataType.MOUTH_MOVEMENT, 0.8)
+
 ###############################################################################
 # Private functions                                                           #
 ###############################################################################
@@ -226,6 +232,7 @@ func _try_load_model(file_path):
 	var dir := Directory.new()
 	if not dir.file_exists(file_path):
 		AppManager.logger.error("File path not found: %s" % file_path)
+		AppManager.logger.notify("File path not found: %s" % file_path)
 		return
 
 	match file_path.get_extension():
@@ -268,7 +275,7 @@ func _try_load_model(file_path):
 			rotation_adjustment = Vector3(-1, -1, 1)
 			AppManager.logger.info("TSCN file loaded successfully.")
 		_:
-			AppManager.logger.info("File extension not recognized. %s" % file_path)
+			AppManager.logger.notify("File extension not recognized. %s" % file_path)
 			printerr("File extension not recognized. %s" % file_path)
 
 # TODO probably incorrect?

screens/gui/Toast.gd

@@ -37,6 +37,10 @@ func _process(delta: float) -> void:
 		_on_close()
 		is_closing = true
 
+func _unhandled_input(event: InputEvent) -> void:
+	if event.is_action_pressed("toggle_gui"):
+		toast.visible = not toast.visible
+
 ###############################################################################
 # Connections                                                                 #
 ###############################################################################

utils/AppManager.gd

@@ -7,6 +7,13 @@ const DYNAMIC_PHYSICS_BONES: bool = false
 onready var sb: SignalBroadcaster = load("res://utils/SignalBroadcaster.gd").new()
 onready var cm: ConfigManager = load("res://utils/ConfigManager.gd").new()
 var nm: NotificationManager = load("res://utils/NotificationManager.gd").new()
+onready var lsm: LipSyncManager = load("res://utils/LipSyncManager.gd").new()
+# TODO clean this up with a stripped down implementation
+#onready var estimate_vowel = load("res://addons/godot-audio-processing/EstimateVowel.gd").new()
+#onready var rtls = load("res://addons/real-time-lip-sync-gd/lip_sync.gdns").new()
+#var effect
+#var buffer = 5
+
 onready var logger: Logger = load("res://utils/Logger.gd").new()
 
 # Debounce
@@ -36,8 +43,10 @@ func _ready() -> void:
 
 	cm.setup()
 	add_child(nm)
+	add_child(lsm)
 
 func _process(delta: float) -> void:
+#	rtls.poll()
 	if should_save:
 		debounce_counter += delta
 		if debounce_counter > DEBOUNCE_TIME:

utils/ConfigManager.gd

@@ -30,6 +30,9 @@ class Metadata:
 	# will always come as a String
 	var camera_index: String = "0"
 
+	# Lip sync
+	var use_lip_sync: bool = false
+
 	func load_from_json(json_string: String) -> bool:
 		var json_data = parse_json(json_string)
 
@@ -316,6 +319,8 @@ func _init() -> void:
 	AppManager.sb.connect("tracker_address", self, "_on_tracker_address")
 	AppManager.sb.connect("tracker_port", self, "_on_tracker_port")
 
+	AppManager.sb.connect("use_lip_sync", self, "_on_use_lip_sync")
+
 	# Features
 
 	AppManager.sb.connect("main_light", self, "_on_main_light")
@@ -411,6 +416,9 @@ func _on_tracker_address(value: String) -> void:
 func _on_tracker_port(value: int) -> void:
 	current_model_config.tracker_port = value
 
+func _on_use_lip_sync(value: bool) -> void:
+	metadata_config.use_lip_sync = value
+
 func _on_camera_select(camera_index: String) -> void:
 	metadata_config.camera_index = camera_index