Skip to content

Commit

Permalink
Fix Lip sync. Breaking change
Browse files Browse the repository at this point in the history 
voice volume expressions are now optional arg {name: value, ....}
  • Loading branch information
@RaSan147
RaSan147 committed Oct 12, 2023
1 parent f53e25a commit 7deba12
Show file tree
Hide file tree
Showing 6 changed files with 120 additions and 41 deletions.
28 changes: 20 additions & 8 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,13 @@ import { Live2DModel } from 'pixi-live2d-display/cubism4';


<!-- if support for both Cubism 2.1 and 4 -->
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]1/dist/index.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]2/dist/index.min.js"></script>

<!-- if only Cubism 2.1 -->
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]1/dist/cubism2.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]2/dist/cubism2.min.js"></script>

<!-- if only Cubism 4 -->
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]1/dist/cubism4.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/RaSan147/[email protected]2/dist/cubism4.min.js"></script>
```

In this way, all the exported members are available under `PIXI.live2d` namespace, such as `PIXI.live2d.Live2DModel`.
Expand Down Expand Up @@ -175,13 +175,20 @@ var model_proxy; //make a global scale handler to use later
```js
var category_name = "Idle" // name of the morion category
var animation_index = 0 // index of animation under that motion category
var priority_number = 3 // if you want to keep the current animation going or move to new animation by force
var priority_number = 3 // if you want to keep the current animation going or move to new animation by force [0: no priority, 1: idle, 2: normal, 3: forced]
var audio_link = "https://cdn.jsdelivr.net/gh/RaSan147/[email protected]/playground/test.mp3" //[Optional arg, can be null or empty] [relative or full url path] [mp3 or wav file]
var volume = 1; //[Optional arg, can be null or empty] [0.0 - 1.0]
var expression = 4; //[Optional arg, can be null or empty] [index|name of expression]
var resetExpression = true; //[Optional arg, can be null or empty] [true|false] [default: true] [if true, expression will be reset to default after animation is over]

model_proxy.motion(category_name, animation_index, priority_number, audio_link, volume, expression)
model_proxy.motion(category_name, animation_index, priority_number, {voice: audio_link, volume: volume, expression:expression, resetExpression:resetExpression})
// Note: during this animation with sound, other animation will be ignored, even its forced. Once over, it'll be back to normal

// if you dont want voice, just ignore the option
model_proxy.motion(category_name, animation_index, priority_number)
model_proxy.motion(category_name, animation_index, priority_number, {expression:expression, resetExpression:resetExpression})
model_proxy.motion(category_name, animation_index, priority_number, {expression:expression, resetExpression:false})

```

## Lipsync Only
Expand All @@ -190,12 +197,17 @@ model_proxy.motion(category_name, animation_index, priority_number, audio_link,
* Demo code
```js
var audio_link = "https://cdn.jsdelivr.net/gh/RaSan147/[email protected]/playground/test.mp3" // [relative or full url path] [mp3 or wav file]

var volume = 1; // [Optional arg, can be null or empty] [0.0 - 1.0]

var expression = 4; // [Optional arg, can be null or empty] [index|name of expression]
var resetExpression = true; // [Optional arg, can be null or empty] [true|false] [default: true] [if true, expression will be reset to default after animation is over]

model_proxy.speak(audio_link, {volume: volume, expression:expression, resetExpression:resetExpression})

// Or if you want to keep some things default
model_proxy.speak(audio_link)
model_proxy.speak(audio_link, {volume: volume})
model_proxy.speak(audio_link, {expression:expression, resetExpression:resetExpression})

model_proxy.speak(audio_link, volume, expression)
```

## Suddenly stop audio and lipsync
Expand Down
34 changes: 26 additions & 8 deletions src/Live2DModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,16 +199,28 @@ export class Live2DModel<IM extends InternalModel = InternalModel> extends Conta
* Shorthand to start a motion.
* @param group - The motion group.
* @param index - Index in the motion group.
* @param priority - The priority to be applied.
* @param priority - The priority to be applied. (0: No priority, 1: IDLE, 2:NORMAL, 3:FORCE) (default: 2)
* ### OPTIONAL: `{name: value, ...}`
* @param sound - The audio url to file or base64 content
* @param volume - Volume of the sound (0-1) /*new in 1.0.4*
* @param volume - Volume of the sound (0-1) (default: 1)
* @param expression - In case you want to mix up a expression while playing sound (bind with Model.expression())
* @param resetExpression - Reset the expression to default after the motion is finished (default: true)
* @return Promise that resolves with true if the motion is successfully started, with false otherwise.
*/
motion(group: string, index?: number, priority?: MotionPriority, sound?: string, volume?:number, expression?: number | string): Promise<boolean> {
motion(group: string, index: number, {priority=2, sound, volume=1, expression, resetExpression=true}:{priority?: MotionPriority, sound?: string, volume?:number, expression?: number | string, resetExpression?:boolean}={}): Promise<boolean> {
return index === undefined
? this.internalModel.motionManager.startRandomMotion(group, priority)
: this.internalModel.motionManager.startMotion(group, index, priority, sound, volume, expression);
? this.internalModel.motionManager.startRandomMotion(group, priority, {
sound: sound,
volume: volume,
expression: expression,
resetExpression: resetExpression
})
: this.internalModel.motionManager.startMotion(group, index, priority, {
sound: sound,
volume: volume,
expression: expression,
resetExpression: resetExpression
});
}


Expand All @@ -223,12 +235,18 @@ export class Live2DModel<IM extends InternalModel = InternalModel> extends Conta
/**
* Shorthand to start speaking a sound with an expression. /*new in 1.0.3*
* @param sound - The audio url to file or base64 content
* ### OPTIONAL: {name: value, ...}
* @param volume - Volume of the sound (0-1) /*new in 1.0.4*
* @param expression - In case you want to mix up a expression while playing sound (bind with Model.expression())
* @param resetExpression - Reset the expression to default after the motion is finished (default: true)
* @returns Promise that resolves with true if the sound is playing, false if it's not
*/
speak(sound: string, volume?: number, expression?: number | string): Promise<boolean> {
return this.internalModel.motionManager.speakUp(sound, volume, expression);
speak(sound: string, {volume=1, expression, resetExpression=true}:{volume?:number, expression?: number | string, resetExpression?:boolean}={}): Promise<boolean> {
return this.internalModel.motionManager.speakUp(sound, {
volume: volume,
expression: expression,
resetExpression: resetExpression
});
}


Expand Down Expand Up @@ -460,4 +478,4 @@ export class Live2DModel<IM extends InternalModel = InternalModel> extends Conta
}
}

applyMixins(Live2DModel, [InteractionMixin]);
applyMixins(Live2DModel, [InteractionMixin]);
39 changes: 25 additions & 14 deletions src/cubism-common/MotionManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,12 +212,13 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even

/**
* Only play sound with lip sync /*new in 1.0.3*
* @param sound - The audio url to file or base64 content
* @param sound - The audio url to file or base64 content
* ### OPTIONAL: {name: value, ...}
* @param volume - Volume of the sound (0-1) /*new in 1.0.4*
* @param expression - In case you want to mix up a expression while playing sound (bind with Model.expression())
* @returns Promise that resolves with true if the sound is playing, false if it's not
*/
async speakUp(sound: string, volume?: number, expression?: number | string) {
async speakUp(sound: string, {volume=1, expression, resetExpression=true}:{volume?:number, expression?: number | string, resetExpression?:boolean}={}) {
if (!config.sound) {
return false;
}
Expand Down Expand Up @@ -258,9 +259,9 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
// start to load the audio
audio = SoundManager.add(
file,
() => {expression && that.expressionManager && that.expressionManager.resetExpression();
() => {resetExpression && expression && that.expressionManager && that.expressionManager.resetExpression();
that.currentAudio = undefined}, // reset expression when audio is done
() => {expression && that.expressionManager && that.expressionManager.resetExpression();
() => {resetExpression && expression && that.expressionManager && that.expressionManager.resetExpression();
that.currentAudio = undefined} // on error
);
this.currentAudio = audio!;
Expand Down Expand Up @@ -311,13 +312,15 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
* Starts a motion as given priority.
* @param group - The motion group.
* @param index - Index in the motion group.
* @param priority - The priority to be applied.
* @param priority - The priority to be applied. default: 2 (NORMAL)
* ### OPTIONAL: {name: value, ...}
* @param sound - The audio url to file or base64 content
* @param volume - Volume of the sound (0-1)
* @param expression - In case you want to mix up a expression while playing sound (bind with Model.expression())
* @param resetExpression - Reset expression before and after playing sound (default: true)
* @return Promise that resolves with true if the motion is successfully started, with false otherwise.
*/
async startMotion(group: string, index: number, priority = MotionPriority.NORMAL, sound?: string, volume?: number, expression?: number | string): Promise<boolean> {
async startMotion(group: string, index: number, priority = MotionPriority.NORMAL, {sound=undefined, volume=1, expression=undefined, resetExpression=true}:{sound?: string, volume?:number, expression?: number | string, resetExpression?:boolean}={}): Promise<boolean> {
// Does not start a new motion if audio is still playing
if(this.currentAudio){
if (!this.currentAudio.ended){
Expand Down Expand Up @@ -367,9 +370,9 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
// start to load the audio
audio = SoundManager.add(
file,
() => {expression && that.expressionManager && that.expressionManager.resetExpression();
() => {resetExpression && expression && that.expressionManager && that.expressionManager.resetExpression();
that.currentAudio = undefined}, // reset expression when audio is done
() => {expression && that.expressionManager && that.expressionManager.resetExpression();
() => {resetExpression && expression && that.expressionManager && that.expressionManager.resetExpression();
that.currentAudio = undefined} // on error
);
this.currentAudio = audio!;
Expand Down Expand Up @@ -416,6 +419,7 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even

return false;
}



if (this.state.shouldOverrideExpression()) {
Expand All @@ -440,12 +444,15 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
/**
* Starts a random Motion as given priority.
* @param group - The motion group.
* @param priority - The priority to be applied.
* @param sound - The wav url file or base64 content
* @param volume - Volume of the sound (0-1)
* @param priority - The priority to be applied. (default: 1 `IDLE`)
* ### OPTIONAL: {name: value, ...}
* @param sound - The wav url file or base64 content+
* @param volume - Volume of the sound (0-1) (default: 1)
* @param expression - In case you want to mix up a expression while playing sound (name/index)
* @param resetExpression - Reset expression before and after playing sound (default: true)
* @return Promise that resolves with true if the motion is successfully started, with false otherwise.
*/
async startRandomMotion(group: string, priority?: MotionPriority, sound?: string, volume?: number): Promise<boolean> {
async startRandomMotion(group: string, priority?:MotionPriority, {sound=undefined, volume=1, expression=undefined, resetExpression=true}:{sound?: string, volume?:number, expression?: number | string, resetExpression?:boolean}={}): Promise<boolean> {
const groupDefs = this.definitions[group];

if (groupDefs?.length) {
Expand All @@ -460,7 +467,11 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
if (availableIndices.length) {
const index = availableIndices[Math.floor(Math.random() * availableIndices.length)]!;

return this.startMotion(group, index, priority, sound, volume);
return this.startMotion(group, index, priority, {
sound: sound,
volume: volume,
expression: expression,
resetExpression: resetExpression});
}
}

Expand Down Expand Up @@ -593,4 +604,4 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends Even
* @return True if the parameters have been actually updated.
*/
protected abstract updateParameters(model: object, now: DOMHighResTimeStamp): boolean;
}
}
22 changes: 21 additions & 1 deletion src/cubism2/Cubism2InternalModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Cubism2MotionManager } from './Cubism2MotionManager';
import { Live2DEyeBlink } from './Live2DEyeBlink';
import { Live2DPhysics } from './Live2DPhysics';
import { Live2DPose } from './Live2DPose';
import { clamp } from '@/utils';

// prettier-ignore
const tempMatrixArray = new Float32Array([
Expand Down Expand Up @@ -33,9 +34,12 @@ export class Cubism2InternalModel extends InternalModel {
angleZParamIndex: number;
bodyAngleXParamIndex: number;
breathParamIndex: number;
mouthFormIndex: number;

textureFlipY = true;

lipSync = true;

/**
* Number of the drawables in this model.
*/
Expand All @@ -62,6 +66,7 @@ export class Cubism2InternalModel extends InternalModel {
this.angleZParamIndex = coreModel.getParamIndex('PARAM_ANGLE_Z');
this.bodyAngleXParamIndex = coreModel.getParamIndex('PARAM_BODY_ANGLE_X');
this.breathParamIndex = coreModel.getParamIndex('PARAM_BREATH');
this.mouthFormIndex = coreModel.getParamIndex("PARAM_MOUTH_FORM");

this.init();
}
Expand Down Expand Up @@ -217,6 +222,21 @@ export class Cubism2InternalModel extends InternalModel {

this.updateFocus();
this.updateNaturalMovements(dt, now);

if (this.lipSync && this.motionManager.currentAudio) {
let value = this.motionManager.mouthSync();
let min_ = 0;
let max_ = 1;
let weight = 1.2;
if (value > 0) {
min_ = 0.4;
}
value = clamp(value * weight, min_, max_);

for (let i = 0; i < this.motionManager.lipSyncIds.length; ++i) {
this.coreModel.setParamFloat(this.coreModel.getParamIndex(this.motionManager.lipSyncIds[i]!), value);
}
}

this.physics?.update(now);
this.pose?.update(dt);
Expand Down Expand Up @@ -277,4 +297,4 @@ export class Cubism2InternalModel extends InternalModel {
// cubism2 core has a super dumb memory management so there's basically nothing much to do to release the model
(this as Partial<this>).coreModel = undefined;
}
}
}
6 changes: 5 additions & 1 deletion src/cubism2/Cubism2MotionManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ export class Cubism2MotionManager extends MotionManager<Live2DMotion, Cubism2Spe

readonly queueManager = new MotionQueueManager();

readonly lipSyncIds: string[];

declare readonly settings: Cubism2ModelSettings;

expressionManager?: Cubism2ExpressionManager;
Expand All @@ -25,6 +27,8 @@ export class Cubism2MotionManager extends MotionManager<Live2DMotion, Cubism2Spe
this.definitions = this.settings.motions;

this.init(options);

this.lipSyncIds = ["PARAM_MOUTH_OPEN_Y"];
}

protected init(options?: MotionManagerOptions) {
Expand Down Expand Up @@ -86,4 +90,4 @@ export class Cubism2MotionManager extends MotionManager<Live2DMotion, Cubism2Spe

(this as Partial<Mutable<this>>).queueManager = undefined;
}
}
}
32 changes: 23 additions & 9 deletions src/cubism4/Cubism4InternalModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
ParamBreath,
ParamEyeBallX,
ParamEyeBallY,
ParamMouthForm,
} from '@cubism/cubismdefaultparameterid';
import { BreathParameterData, CubismBreath } from '@cubism/effect/cubismbreath';
import { CubismEyeBlink } from '@cubism/effect/cubismeyeblink';
Expand All @@ -21,6 +22,7 @@ import { CubismPhysics } from '@cubism/physics/cubismphysics';
import { CubismRenderer_WebGL, CubismShader_WebGL } from '@cubism/rendering/cubismrenderer_webgl';
import { Matrix } from '@pixi/math';
import { Mutable } from '../types/helpers';
import { clamp } from '@/utils';

const tempMatrix = new CubismMatrix44();

Expand Down Expand Up @@ -49,6 +51,7 @@ export class Cubism4InternalModel extends InternalModel {
idParamEyeBallY = ParamEyeBallY;
idParamBodyAngleX = ParamBodyAngleX;
idParamBreath = ParamBreath;
idParamMouthForm = ParamMouthForm;

/**
* The model's internal scale, defined in the moc3 file.
Expand Down Expand Up @@ -209,14 +212,20 @@ export class Cubism4InternalModel extends InternalModel {
// revert the timestamps to be milliseconds
this.updateNaturalMovements(dt * 1000, now * 1000);

// TODO: Add lip sync API
// if (this.lipSync) {
// const value = 0; // 0 ~ 1
//
// for (let i = 0; i < this.lipSyncIds.length; ++i) {
// model.addParameterValueById(this.lipSyncIds[i], value, 0.8);
// }
// }
if (this.lipSync && this.motionManager.currentAudio) {
let value = this.motionManager.mouthSync();
let min_ = 0;
let max_ = 1;
let weight = 1.2;
if (value > 0) {
min_ = 0.4;
}
value = clamp(value * weight, min_, max_);

for (let i = 0; i < this.motionManager.lipSyncIds.length; ++i) {
model.addParameterValueById(this.motionManager.lipSyncIds[i], value, 0.8);
}
}

this.physics?.evaluate(model, dt);
this.pose?.updateParameters(model, dt);
Expand All @@ -236,6 +245,11 @@ export class Cubism4InternalModel extends InternalModel {
this.coreModel.addParameterValueById(this.idParamBodyAngleX, this.focusController.x * 10); // -10 ~ 10
}


updateFacialEmotion(mouthForm: number) {
this.coreModel.addParameterValueById(this.idParamMouthForm, mouthForm); // -1 ~ 1
}

updateNaturalMovements(dt: DOMHighResTimeStamp, now: DOMHighResTimeStamp) {
this.breath?.updateParameters(this.coreModel, dt / 1000);
}
Expand Down Expand Up @@ -266,4 +280,4 @@ export class Cubism4InternalModel extends InternalModel {
(this as Partial<this>).renderer = undefined;
(this as Partial<this>).coreModel = undefined;
}
}
}

0 comments on commit 7deba12

Please sign in to comment.