diff --git a/.eslintrc.json b/.eslintrc.json index 1ad75e1a..16651129 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -18,9 +18,29 @@ "simple-import-sort" ], "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "argsIgnorePattern": "^_", + "varsIgnorePattern": "^_", + "caughtErrorsIgnorePattern": "^_" + } + ], "simple-import-sort/imports": "error", "simple-import-sort/exports": "error", + "no-console": ["error", { "allow": ["warn", "error"] }], "no-throw-literal": "off" }, + "overrides": [ + { + "files": [ + "examples/**/*.ts", + "examples/**/*.tsx" + ], + "rules": { + "no-console": "off" + } + } + ], "ignorePatterns": ["proto", "build"] } diff --git a/.npmignore b/.npmignore index 51891ab7..e057301c 100644 --- a/.npmignore +++ b/.npmignore @@ -1,3 +1,5 @@ +.github +.husky coverage examples __tests__ @@ -6,7 +8,7 @@ __tests__ */**/.eslintcache .eslintrc.json .prettierrc.json -jest.config.js +jest.config.json /tsconfig.json /tsconfig.paths.json build/jest-result.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 732af1ed..eb2f0235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2022-04-20 v1.2.0 + +* Add phonemes event support +* Add silence event support +* Improve manual reconnect latency + ## 2022-04-06 v1.1.8 * Remove deprecated emotions attributes: joy, fear, trust and surprise diff --git a/__tests__/clients/inworld.client.spec.ts b/__tests__/clients/inworld.client.spec.ts index 427ca181..135f9b98 100644 --- a/__tests__/clients/inworld.client.spec.ts +++ b/__tests__/clients/inworld.client.spec.ts @@ -1,12 +1,14 @@ import { InworldClient } from '../../src/clients/inworld.client'; +import { GetterSetter } from '../../src/common/interfaces'; +import { Session } from '../../src/entities/session.entity'; import { TokenClientGrpcService } from '../../src/services/gprc/token_client_grpc.service'; import { capabilitiesProps, KEY, SCENE, SECRET, - session, sessionProto, + sessionToken, user, } from '../helpers'; @@ -16,6 +18,10 @@ describe('should finish with success', () => { const onMessage = jest.fn(); const onDisconnect = jest.fn(); const generateSessionTokenFn = jest.fn(); + const sessionGetterSetter = { + get: jest.fn(), + set: jest.fn(), + } as GetterSetter; beforeEach(() => { jest.clearAllMocks(); @@ -28,7 +34,8 @@ describe('should finish with success', () => { .setOnDisconnect(onDisconnect) .setOnMessage(onMessage) .setOnError(onError) - .setGenerateSessionToken(generateSessionTokenFn); + .setGenerateSessionToken(generateSessionTokenFn) + .setOnSession(sessionGetterSetter); }); test('should generate session token', async () => { @@ -39,7 +46,7 @@ describe('should finish with success', () => { const result = await client.generateSessionToken(); expect(generateSessionToken).toHaveBeenCalledTimes(1); - expect(result).toEqual(session); + expect(result).toEqual(sessionToken); }); test('should build', async () => { diff --git a/__tests__/entities/character.entity.spec.ts b/__tests__/entities/character.entity.spec.ts index 87ce8034..abbb0ea8 100644 --- a/__tests__/entities/character.entity.spec.ts +++ b/__tests__/entities/character.entity.spec.ts @@ -15,6 +15,25 @@ test('should get character fields', () => { }; const character = new Character({ id, assets, resourceName, displayName }); + expect(character.id).toEqual(id); + expect(character.resourceName).toEqual(resourceName); + expect(character.displayName).toEqual(displayName); + expect(character.assets).toEqual(assets); +}); + +test('should get character fields in the deprecated way', () => { + const id = v4(); + const resourceName = v4(); + const displayName = v4(); + const assets: Assets = { + avatarImg: v4(), + avatarImgOriginal: v4(), + rpmModelUri: v4(), + rpmImageUriPortrait: v4(), + rpmImageUriPosture: v4(), + }; + const character = new Character({ id, assets, resourceName, displayName }); + expect(character.getId()).toEqual(id); expect(character.getResourceName()).toEqual(resourceName); expect(character.getDisplayName()).toEqual(displayName); diff --git a/__tests__/entities/inworld_packet.entity.spec.ts b/__tests__/entities/inworld_packet.entity.spec.ts index c2b75073..1545caf8 100644 --- a/__tests__/entities/inworld_packet.entity.spec.ts +++ b/__tests__/entities/inworld_packet.entity.spec.ts @@ -98,6 +98,20 @@ test('should get emotion packet fields', () => { expect(packet.packetId).toEqual(packetId); }); +test('should get silence packet fields', () => { + const packet = new InworldPacket({ + packetId, + routing, + date, + type: InworldPacketType.SILENCE, + }); + + expect(packet.isSilence()).toEqual(true); + expect(packet.routing).toEqual(routing); + expect(packet.date).toEqual(date); + expect(packet.packetId).toEqual(packetId); +}); + describe('control', () => { test('should get interaction end packet fields', () => { const packet = new InworldPacket({ diff --git a/__tests__/entities/scene.entity.spec.ts b/__tests__/entities/scene.entity.spec.ts new file mode 100644 index 00000000..31972bb6 --- /dev/null +++ b/__tests__/entities/scene.entity.spec.ts @@ -0,0 +1,50 @@ +import { v4 } from 'uuid'; + +import { LoadSceneResponse } from '../../proto/world-engine_pb'; +import { Character } from '../../src/entities/character.entity'; +import { Scene } from '../../src/entities/scene.entity'; +import { createAgent, createCharacter } from '../helpers'; + +let key: string; +let characters: Array = []; +let scene: Scene; +let json: string; + +beforeEach(() => { + jest.clearAllMocks(); + + key = v4(); + characters = [createCharacter(), createCharacter()]; + scene = new Scene({ + characters, + key, + }); + json = JSON.stringify(scene); +}); + +test('should return scene fields', () => { + expect(scene.characters).toEqual(characters); + expect(scene.key).toEqual(key); +}); + +test('should serialize', () => { + expect(Scene.serialize(scene)).toEqual(json); +}); + +test('should deserialize', () => { + const result = Scene.deserialize(json); + + expect(result.key).toEqual(scene.key); + expect(result.characters).toEqual(scene.characters); +}); + +test('should convert proto to scene', () => { + const agents = [createAgent(), createAgent(false)]; + const proto = new LoadSceneResponse().setAgentsList(agents).setKey(key); + const scene = Scene.fromProto(proto); + + expect(scene.key).toEqual(key); + expect(scene.characters[0].id).toEqual(agents[0].getAgentId()); + expect(scene.characters[1].id).toEqual(agents[1].getAgentId()); + expect(scene.characters[1].assets.avatarImg).toEqual(undefined); +}); diff --git a/__tests__/entities/session.entity.spec.ts b/__tests__/entities/session.entity.spec.ts new file mode 100644 index 00000000..a460f2d1 --- /dev/null +++ b/__tests__/entities/session.entity.spec.ts @@ -0,0 +1,38 @@ +import { v4 } from 'uuid'; + +import { Scene } from '../../src/entities/scene.entity'; +import { Session } from '../../src/entities/session.entity'; +import { createCharacter, sessionToken } from '../helpers'; + +let scene: Scene = { + key: v4(), + characters: [createCharacter(), createCharacter()], +}; +let session: Session; +let json: string; + +beforeEach(() => { + jest.clearAllMocks(); + + session = new Session({ + scene, + sessionToken, + }); + json = JSON.stringify(session); +}); + +test('should return session fields', () => { + expect(session.scene).toEqual(scene); + expect(session.sessionToken).toEqual(sessionToken); +}); + +test('should serialize', () => { + expect(Session.serialize(session)).toEqual(json); +}); + +test('should deserialize', () => { + const result = Session.deserialize(json); + + expect(result.scene).toEqual(session.scene); + expect(result.sessionToken).toEqual(session.sessionToken); +}); diff --git a/__tests__/entities/session_token.entity.spec.ts b/__tests__/entities/session_token.entity.spec.ts index 43e0c429..3a0c27af 100644 --- a/__tests__/entities/session_token.entity.spec.ts +++ b/__tests__/entities/session_token.entity.spec.ts @@ -1,6 +1,11 @@ +import { SessionAccessToken } from '@proto/ai/inworld/studio/v1alpha/tokens_pb'; import { v4 } from 'uuid'; +import { protoTimestamp } from '../../src/common/helpers'; import { SessionToken } from '../../src/entities/session_token.entity'; +import { sessionToken } from '../helpers'; + +const json = JSON.stringify(sessionToken); test('should get session fields', () => { const token = v4(); @@ -15,8 +20,81 @@ test('should get session fields', () => { expirationTime, }); + expect(session.token).toEqual(token); + expect(session.type).toEqual(type); + expect(session.sessionId).toEqual(sessionId); + expect(session.expirationTime).toEqual(expirationTime); +}); + +test('should get session fields in the deprecated way', () => { + const token = v4(); + const type = v4(); + const sessionId = v4(); + const expirationTime = new Date(); + + const session = new SessionToken({ + token, + type, + sessionId, + expirationTime, + }); + expect(session.getToken()).toEqual(token); expect(session.getType()).toEqual(type); expect(session.getSessionId()).toEqual(sessionId); expect(session.getExpirationTime()).toEqual(expirationTime); }); + +test('should serialize', () => { + expect(SessionToken.serialize(sessionToken)).toEqual(json); +}); + +test('should deserialize', () => { + const result = SessionToken.deserialize(json); + + expect(result.token).toEqual(sessionToken.token); + expect(result.type).toEqual(sessionToken.type); + expect(result.sessionId).toEqual(sessionToken.sessionId); + expect(result.expirationTime).toEqual(sessionToken.expirationTime); +}); + +test('should convert proto to token', () => { + const token = v4(); + const type = v4(); + const sessionId = v4(); + const expirationTime = new Date(); + + const proto = new SessionAccessToken() + .setToken(token) + .setType(type) + .setSessionId(sessionId) + .setExpirationTime(protoTimestamp(expirationTime)); + const scene = SessionToken.fromProto(proto); + + expect(scene.token).toEqual(proto.getToken()); + expect(scene.type).toEqual(proto.getType()); + expect(scene.sessionId).toEqual(proto.getSessionId()); + expect(scene.expirationTime).toEqual(proto.getExpirationTime().toDate()); +}); + +describe('isExpired', () => { + test('should detect expired session', () => { + const token = v4(); + const type = v4(); + const sessionId = v4(); + const expirationTime = new Date(); + + const session = new SessionToken({ + token, + type, + sessionId, + expirationTime, + }); + + expect(SessionToken.isExpired(session)).toEqual(true); + }); + + test('should not detect expired session', () => { + expect(SessionToken.isExpired(sessionToken)).toEqual(false); + }); +}); diff --git a/__tests__/factories/event.spec.ts b/__tests__/factories/event.spec.ts index 194e3d44..0ea50d1f 100644 --- a/__tests__/factories/event.spec.ts +++ b/__tests__/factories/event.spec.ts @@ -1,5 +1,6 @@ import { Actor, + AdditionalPhonemeInfo, ControlEvent, DataChunk, EmotionEvent, @@ -7,6 +8,7 @@ import { PacketId, Routing, } from '@proto/packets_pb'; +import { Duration } from 'google-protobuf/google/protobuf/duration_pb'; import { v4 } from 'uuid'; import { protoTimestamp } from '../../src/common/helpers'; @@ -28,7 +30,7 @@ test('should set and get character', () => { const found = factory.getCurrentCharacter(); expect(found).toEqual(character); - expect(found.getId()).toEqual(character.getId()); + expect(found.id).toEqual(character.id); }); describe('event types', () => { @@ -49,7 +51,7 @@ describe('event types', () => { expect(event.getDataChunk().getType()).toEqual(DataChunk.DataType.AUDIO); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -62,7 +64,7 @@ describe('event types', () => { ); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -75,7 +77,7 @@ describe('event types', () => { ); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -87,7 +89,7 @@ describe('event types', () => { expect(event.getText().getText()).toEqual(text); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -99,7 +101,7 @@ describe('event types', () => { expect(event.getCustom().getName()).toEqual(name); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -109,7 +111,7 @@ describe('event types', () => { expect(event.hasCancelresponses()).toEqual(true); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -122,7 +124,7 @@ describe('event types', () => { expect(event.hasCancelresponses()).toEqual(true); expect(event.hasPacketId()).toEqual(true); expect(event.hasRouting()).toEqual(true); - expect(event.getRouting().getTarget().getName()).toEqual(character.getId()); + expect(event.getRouting().getTarget().getName()).toEqual(character.id); expect(event.hasTimestamp()).toEqual(true); }); @@ -140,23 +142,37 @@ describe('event types', () => { describe('convert packet to external one', () => { test('audio', () => { - const result = factory.convertToInworldPacket( - factory.dataChunk(v4(), DataChunk.DataType.AUDIO), - ); + const rounting = new Routing() + .setSource(new Actor()) + .setTarget(new Actor()); + const dataChunk = new DataChunk() + .setType(DataChunk.DataType.AUDIO) + .setChunk(v4()) + .setAdditionalPhonemeInfoList([ + new AdditionalPhonemeInfo() + .setPhoneme(v4()) + .setStartOffset(new Duration().setSeconds(100).setNanos(10)), + ]); + const packet = new ProtoPacket() + .setPacketId(new PacketId().setPacketId(v4())) + .setRouting(rounting) + .setTimestamp(protoTimestamp()) + .setDataChunk(dataChunk); + const result = EventFactory.fromProto(packet); expect(result).toBeInstanceOf(InworldPacket); expect(result.isAudio()).toEqual(true); }); test('text', () => { - const result = factory.convertToInworldPacket(factory.text(v4())); + const result = EventFactory.fromProto(factory.text(v4())); expect(result).toBeInstanceOf(InworldPacket); expect(result.isText()).toEqual(true); }); test('trigger', () => { - const result = factory.convertToInworldPacket(factory.trigger(v4())); + const result = EventFactory.fromProto(factory.trigger(v4())); expect(result).toBeInstanceOf(InworldPacket); expect(result.isTrigger()).toEqual(true); @@ -173,12 +189,30 @@ describe('convert packet to external one', () => { .setTimestamp(protoTimestamp()) .setEmotion(new EmotionEvent()); - const result = factory.convertToInworldPacket(packet); + const result = EventFactory.fromProto(packet); expect(result).toBeInstanceOf(InworldPacket); expect(result.isEmotion()).toEqual(true); }); + test('silence', () => { + const rounting = new Routing() + .setSource(new Actor()) + .setTarget(new Actor()); + const dataChunk = new DataChunk() + .setType(DataChunk.DataType.SILENCE) + .setDurationMs(100); + const packet = new ProtoPacket() + .setPacketId(new PacketId().setPacketId(v4())) + .setRouting(rounting) + .setTimestamp(protoTimestamp()) + .setDataChunk(dataChunk); + const result = EventFactory.fromProto(packet); + + expect(result).toBeInstanceOf(InworldPacket); + expect(result.isSilence()).toEqual(true); + }); + test('unknown', () => { const rounting = new Routing() .setSource(new Actor()) @@ -189,7 +223,7 @@ describe('convert packet to external one', () => { .setRouting(rounting) .setTimestamp(protoTimestamp()); - const result = factory.convertToInworldPacket(packet); + const result = EventFactory.fromProto(packet); expect(result).toBeInstanceOf(InworldPacket); expect(result.isEmotion()).toEqual(false); @@ -211,7 +245,7 @@ describe('convert packet to external one', () => { .setRouting(new Routing().setSource(new Actor()).setTarget(new Actor())) .setTimestamp(protoTimestamp(today)); - const result = factory.convertToInworldPacket(packet); + const result = EventFactory.fromProto(packet); expect(result).toBeInstanceOf(InworldPacket); expect(result.isControl()).toEqual(true); @@ -228,7 +262,7 @@ describe('convert packet to external one', () => { .setRouting(new Routing().setSource(new Actor()).setTarget(new Actor())) .setTimestamp(protoTimestamp(today)); - const result = factory.convertToInworldPacket(packet); + const result = EventFactory.fromProto(packet); expect(result).toBeInstanceOf(InworldPacket); expect(result.isControl()).toEqual(true); diff --git a/__tests__/helpers.ts b/__tests__/helpers.ts index 394325cf..a8789f25 100644 --- a/__tests__/helpers.ts +++ b/__tests__/helpers.ts @@ -38,14 +38,16 @@ export const createCharacter = () => displayName: v4(), }); -export const createAgent = () => { +export const createAgent = (useAssets: boolean = true) => { const agent = new LoadSceneResponse.Agent(); - const assets = new LoadSceneResponse.Agent.CharacterAssets() - .setAvatarImg(v4()) - .setAvatarImgOriginal(v4()) - .setRpmModelUri(v4()) - .setRpmImageUriPortrait(v4()) - .setRpmImageUriPosture(v4()); + const assets = useAssets + ? new LoadSceneResponse.Agent.CharacterAssets() + .setAvatarImg(v4()) + .setAvatarImgOriginal(v4()) + .setRpmModelUri(v4()) + .setRpmImageUriPortrait(v4()) + .setRpmImageUriPosture(v4()) + : undefined; return agent.setAgentId(v4()).setBrainName(v4()).setCharacterAssets(assets); }; @@ -89,7 +91,7 @@ export const sessionProto = new SessionAccessToken() .setType('Bearer') .setExpirationTime(protoTimestamp(today)); -export const session = new SessionToken({ +export const sessionToken = new SessionToken({ sessionId: sessionProto.getSessionId(), token: sessionProto.getToken(), type: sessionProto.getType(), @@ -100,12 +102,16 @@ export const capabilitiesProps: Capabilities = { emotions: true, audio: true, interruptions: true, + phonemes: true, + silence: true, }; export const capabilities = new CapabilitiesRequest() .setAudio(true) .setEmotions(true) .setInterruptions(true) + .setPhonemeInfo(true) + .setSilenceEvents(true) .setText(true) .setTriggers(true); diff --git a/__tests__/services/connection.service.spec.ts b/__tests__/services/connection.service.spec.ts index 335fe4f7..71cd5ec3 100644 --- a/__tests__/services/connection.service.spec.ts +++ b/__tests__/services/connection.service.spec.ts @@ -10,6 +10,8 @@ import { LoadSceneResponse } from '@proto/world-engine_pb'; import { v4 } from 'uuid'; import { protoTimestamp } from '../../src/common/helpers'; +import { Scene } from '../../src/entities/scene.entity'; +import { Session } from '../../src/entities/session.entity'; import { SessionToken } from '../../src/entities/session_token.entity'; import { EventFactory } from '../../src/factories/event'; import { ConnectionService } from '../../src/services/connection.service'; @@ -22,7 +24,7 @@ import { KEY, SCENE, SECRET, - session, + sessionToken, user, } from '../helpers'; @@ -68,12 +70,12 @@ test('should generate session token', async () => { }); const generateSessionToken = jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); const result = await connection.generateSessionToken(); expect(generateSessionToken).toHaveBeenCalledTimes(1); - expect(result).toEqual(session); + expect(result).toEqual(sessionToken); }); describe('open', () => { @@ -96,7 +98,7 @@ describe('open', () => { test('should execute without errors', async () => { const generateSessionToken = jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); const openSession = jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') .mockImplementationOnce(() => stream); @@ -113,18 +115,18 @@ describe('open', () => { expect(loadScene).toHaveBeenCalledWith({ name: SCENE, capabilities, - session, + sessionToken, user, }); expect(openSession).toHaveBeenCalledTimes(1); - expect(loaded[0].getId()).toBe(characters[0].getId()); - expect(loaded[1].getId()).toBe(characters[1].getId()); + expect(loaded[0].id).toBe(characters[0].id); + expect(loaded[1].id).toBe(characters[1].id); }); test('should call external generate session token', async () => { const generateSessionTokenFn = jest .fn() - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); const connection = new ConnectionService({ apiKey: { key: KEY, secret: SECRET }, @@ -139,7 +141,7 @@ describe('open', () => { const generateSessionToken = jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') @@ -156,6 +158,44 @@ describe('open', () => { expect(generateSessionTokenFn).toHaveBeenCalledTimes(1); }); + test('should call getter and setter for session', async () => { + const sessionGetterSetter = { + get: jest.fn().mockImplementationOnce(() => + Promise.resolve({ + scene: Scene.fromProto(scene), + sessionToken, + } as Session), + ), + set: jest.fn(), + }; + const connection = new ConnectionService({ + apiKey: { key: KEY, secret: SECRET }, + name: SCENE, + config: { capabilities }, + user, + onError, + onMessage, + onDisconnect, + sessionGetterSetter, + }); + + jest + .spyOn(connection, 'generateSessionToken') + .mockImplementationOnce(() => Promise.resolve(sessionToken)); + + jest + .spyOn(WorldEngineClientGrpcService.prototype, 'session') + .mockImplementationOnce(() => stream); + jest + .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') + .mockImplementationOnce(() => Promise.resolve(scene)); + + await connection.open(); + + expect(sessionGetterSetter.get).toHaveBeenCalledTimes(1); + expect(sessionGetterSetter.set).toHaveBeenCalledTimes(1); + }); + test('should catch error on load scene and pass it to handler', async () => { const err = new Error(); const generateSessionToken = jest @@ -194,7 +234,7 @@ describe('open', () => { test('should catch error on connection establishing and pass it to handler', async () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)); @@ -215,7 +255,7 @@ describe('open', () => { test('should inactivate connection on disconnect stream event', async () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)); @@ -244,7 +284,7 @@ describe('open', () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)); @@ -264,7 +304,7 @@ describe('open', () => { test('should not generate actual token twice', async () => { const generateSessionToken = jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') .mockImplementationOnce(() => stream); @@ -280,9 +320,7 @@ describe('open', () => { test('should regenerate expired token', async () => { const expiredSession = new SessionToken({ - sessionId: session.getSessionId(), - token: session.getToken(), - type: session.getType(), + ...sessionToken, expirationTime: new Date(), }); const generateSessionToken = jest.spyOn(connection, 'generateSessionToken'); @@ -326,7 +364,7 @@ describe('open', () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') .mockImplementationOnce(() => stream); @@ -409,7 +447,7 @@ describe('open manually', () => { .mockImplementationOnce(jest.fn()); jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)); @@ -454,7 +492,7 @@ describe('close', () => { .mockImplementationOnce(jest.fn()); jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') .mockImplementationOnce(() => stream); @@ -506,7 +544,7 @@ describe('send', () => { const open = jest.spyOn(ConnectionService.prototype, 'open'); jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'session') .mockImplementationOnce(() => stream); @@ -595,7 +633,7 @@ describe('send', () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementation(() => Promise.resolve(session)); + .mockImplementation(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClient.prototype, 'session') .mockImplementation(() => stream); @@ -637,7 +675,7 @@ describe('message', () => { jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)); jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)); @@ -650,9 +688,7 @@ describe('message', () => { stream.emit('data', packet); expect(onMessage).toHaveBeenCalledTimes(1); - expect(onMessage).toHaveBeenCalledWith( - connection.getEventFactory().convertToInworldPacket(packet), - ); + expect(onMessage).toHaveBeenCalledWith(EventFactory.fromProto(packet)); }); }); @@ -684,8 +720,8 @@ describe('getCharactersList', () => { ); const generateSessionToken = jest .spyOn(connection, 'generateSessionToken') - .mockImplementationOnce(() => Promise.resolve(session)) - .mockImplementationOnce(() => Promise.resolve(session)); + .mockImplementationOnce(() => Promise.resolve(sessionToken)) + .mockImplementationOnce(() => Promise.resolve(sessionToken)); const loadScene = jest .spyOn(WorldEngineClientGrpcService.prototype, 'loadScene') .mockImplementationOnce(() => Promise.resolve(scene)) diff --git a/__tests__/services/grpc/world_engine_client_grpc.service.spec.ts b/__tests__/services/grpc/world_engine_client_grpc.service.spec.ts index f755d0ca..68891bf6 100644 --- a/__tests__/services/grpc/world_engine_client_grpc.service.spec.ts +++ b/__tests__/services/grpc/world_engine_client_grpc.service.spec.ts @@ -14,7 +14,7 @@ import { v4 } from 'uuid'; import { Config } from '../../../src/common/config'; import { CLIENT_ID } from '../../../src/common/constants'; import { WorldEngineClientGrpcService } from '../../../src/services/gprc/world_engine_client_grpc.service'; -import { createAgent, session, user } from '../../helpers'; +import { createAgent, sessionToken, user } from '../../helpers'; const SCENE = v4(); const agents = [createAgent(), createAgent()]; @@ -80,7 +80,7 @@ describe('load scene', () => { const result = await client.loadScene({ name: SCENE, capabilities, - session, + sessionToken, user, }); @@ -113,7 +113,7 @@ describe('load scene', () => { name: SCENE, client: sceneClient, capabilities, - session, + sessionToken, user, }); @@ -140,7 +140,7 @@ describe('session', () => { const onDisconnect = jest.fn(); const connection = client.session({ - session, + sessionToken, onError, onMessage, onDisconnect, @@ -163,7 +163,7 @@ describe('session', () => { .mockImplementation(() => stream); client.session({ - session, + sessionToken, onError, }); @@ -181,7 +181,7 @@ describe('session', () => { .mockImplementation(() => stream); client.session({ - session, + sessionToken, onMessage, }); @@ -198,7 +198,7 @@ describe('session', () => { .mockImplementation(() => stream); client.session({ - session, + sessionToken, onDisconnect, }); diff --git a/examples/cli/README.md b/examples/cli/README.md deleted file mode 100644 index 50e15302..00000000 --- a/examples/cli/README.md +++ /dev/null @@ -1,243 +0,0 @@ -# Using the Command-Line Interface - -This folder contains two examples based on **automatic** and **manual** reconnects. - -## Requirements -- Node 16 is recommended -- Latest version of [sox](http://sox.sourceforge.net) installed and available in PATH - -## Installation - -### Setup variables in .env file - -|Name|Description|Details| -|--:|--:|--:| -|INWORLD_KEY|Inworld application key|Get key from [integrations page](https://studio.inworld.ai)| -|INWORLD_SECRET|Inworld application secret|Get secret from [integrations page](https://studio.inworld.ai)| -|INWORLD_SCENE|Full scene name|It should have one of the following format: workspaces/{WORKSPACE_NAME}/characters/{CHARACTER_NAME} workspaces/{WORKSPACE_NAME}/scenes/{SCENE_NAME}| - -### Install dependencies - -```sh -yarn install -``` - -## Example with auto reconnect - -### Start application - -```sh -yarn start:auto-reconnect -``` - -You will need to use the following console commands to start the client with auto reconnect: - |- /start - starts audio capturing. - |- /end - ends audio capturing. - |- /trigger - send custom trigger. - |- /info - shows current character. - |- /list-all - shows available characters (created within the scene). - |- /character %character-id% - id of the target character (Get full list using /list-all command). - |- c - cancel current response. - |- - sends text event to server. - -### Setup connection - -```typescript -const client = new InworldClient() - .setApiKey({ - key: process.env.INWORLD_KEY!, - secret: process.env.INWORLD_SECRET!, - }) - .setScene(process.env.INWORLD_SCENE!) - .setOnError((err: ServiceError) => console.error(`Error: ${err.message}`)) - .setOnDisconnect(() => console.log('Disconnected')) - .setOnMessage(() => console.log('Message was received')) - .build(); -} -``` - -### Get list of characters and select one of them - -If the scene contains more than one character, then the fist one will be used by default. You can change this manually by running, -```typescript -const characters = await connection.getCharacters(); -const character = characters.find((c) => c.getId() === id); - -connection.setCurrentCharacter(character); -``` - -This should produce the following console output, - -```sh -/info -Character: 4bafd052-fe3e-483b-9eaf-ab20cadf84b2 (workspaces/test-workspace/characters/character1:First character) ------------------------------- -/list-all -0: 4bafd052-fe3e-483b-9eaf-ab20cadf84b2 (workspaces/test-workspace/characters/character1:First character) -1: 82825e9c-7d80-42a4-b6b2-f313c1ac29ed (workspaces/test-workspace/characters/character2:Second character) ------------------------------- -/character 82825e9c-7d80-42a4-b6b2-f313c1ac29ed -Character: 82825e9c-7d80-42a4-b6b2-f313c1ac29ed (workspaces/test-workspace/characters/character2:Second character) ------------------------------- -``` - -### Sending a text message, trigger, or cancelling the response to a character -The connection will be opened automatically on send if it is closed, or an existing open connection will be used. Our server will close the connection automatically in one minute if there is inactivity. - -```typescript -await connection.sendText('Hello'); -await connection.sendTrigger('Some action'); -await connection.sendCancelResponse(); -``` - -### Send audio chunk to character - -Initialize [recorder](https://www.npmjs.com/package/node-record-lpcm16) and send audio on each data chunk that it receives. Before beginning capture, send an `audio session start` event to the connection. Do not forget to send an `audio session end` event at the end of the capture. - -```typescript -await connection.sendAudioSessionStart(); -// Start audio capturing here -await connection.sendAudio(chunk); -... -// Stop audio capturing here -await connection.sendAudioSessionEnd(); -``` -This should produce the following console output, - -```sh -/start -Recognized: How are you?, final=true -Character(aa1c8634-df62-4d8b-94eb-4188cf8fbea7) to Player (i=599d9d2e-9994-4208-bb52-b8d5d8997819, u=620d9711-42d4-433f-b5e9-455ac0b7f6aa): I\'m good, thanks! -Character(aa1c8634-df62-4d8b-94eb-4188cf8fbea7) to Player (i=211644a2-2657-40be-b6e6-b88194604717, u=606c9cf7-334b-43e6-af4e-269b78083a87): Thank you for asking. -/end -``` - -### Close connection - -You can close the connection manually if it is not currently required. Be sure to do this on your application stop, e.g., ‘SIGINT’ event. It will be open automatically on next packet send. - -```typescript -connection.close(); -``` - -## Example with manual reconnect - -### Start application - -```sh -yarn start:manual-reconnect -``` - -You will need to use the following console commands to start the client with auto reconnect: - |- /open - open connection. - |- /close - close connection. - |- /start - starts audio capturing. - |- /end - ends audio capturing. - |- /trigger - send trigger event. - |- /info - shows current character. - |- /list-all - shows available characters (created within the scene). - |- /character %character-id% - id of the target character (Get full list using /list-all command). - |- c - cancel current response. - |- - sends text event to server. - -### Setup connection -If you would like to open and close the connection manually, then you should use the `autoReconnect = false` option to configuration. Also it's possible to specify disconnectTimeout (1 minute is used by default). - -```typescript -const client = new InworldClient() - .setApiKey({ - key: process.env.INWORLD_KEY!, - secret: process.env.INWORLD_SECRET!, - }) - .setScene(process.env.INWORLD_SCENE!) - .setConfiguration({ - connection: { - autoReconnect: false, - disconnectTimeout: 30 * 1000, // 30 seconds - }, - }) - .setOnError((err: ServiceError) => console.error(`Error: ${err.message}`)) - .setOnDisconnect(() => console.log('Disconnected')) - .setOnMessage(() => console.log('Message was received')) - .build(); -} -``` - -### Open connection -You should open the connection and manage it manually as follows, - -```typescript - if (!connection.isActive()) { - await connection.open(); - } - // Do something here -``` - -### Send message to character - -If the connection is open, a message will be sent to the character. Otherwise, you will receive an error message. - -```typescript -await connection.sendText('Hello'); -// Error: Unable to send data due to an inactive connection -``` - -### Close connection - -You can close the connection manually if it is not currently required. Be sure to do this on your application stop, e.g., ‘SIGINT’ event. It can be opened later. - -```typescript -connection.close(); -``` - -## Handlers - -### onDisconnect - -Each time the server closes connection or you call `connection.close`, a disconnect event will be triggered. - -```typescript -connection.setOnDisconnect(() => console.log('Disconnected')); -``` - -### onError - -```typescript -connection.setOnError((err: ServiceError) => console.error(`Error: ${err.message}`)); -``` - -### onMessage - -To detect the type of message and print or play, you can use, - -```typescript -connection.setOnMessage(async (packet: InworldPacket) => { - const { packetId } = packet; - const i = packetId.packetId; - const u = packetId.utteranceId; - - // TEXT - if (packet.isText()) { - const textEvent = packet.text; - - if (packet.routing.source.isPlayer) { - if (textEvent.final) { - console.log( - `Recognized: ${textEvent.text}, final=${textEvent.final}`, - ); - } - } else { - console.log( - `${this.renderEventRouting(packet)} (i=${i}, u=${u}): ${ - textEvent.text - }`, - ); - } - } - - // AUDIO - if (packet.isAudio()) { - player.send(packet.audio.chunk); - } -}); -``` diff --git a/examples/upload_local_audio_file/README.md b/examples/upload_local_audio_file/README.md deleted file mode 100644 index dc49757c..00000000 --- a/examples/upload_local_audio_file/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Install dependencies - -```sh -yarn install -``` - -# Setup variables in .env file or using export command - -|Name|Description|Details| -|--:|--:|--:| -|INWORLD_KEY|Inworld application key|Get key from [integrations page](https://studio.inworld.ai)| -|INWORLD_SECRET|Inworld application secret|Get secret from [integrations page](https://studio.inworld.ai)| -|INWORLD_SCENE|Full scene name|It should have one of the following format: workspaces/{WORKSPACE_NAME}/characters/{CHARACTER_NAME} workspaces/{WORKSPACE_NAME}/scenes/{SCENE_NAME}| - -# Start application - -```sh -yarn start -``` - -# How it works - -* Ensure you file has sample **rate = 16000 Hz** and is in **LINEAR16** format. Otherwise convert it manually - -```sh -ffmpeg -i test.mp3 -ar 16000 -sample_fmt s16 test.wav -``` - -* Split file to chunks. I.e. specify `highWaterMark` for `createReadStream` call. Chunk's size should not be to large. - -* Send `sendAudioSessionStart` event. - -* Send chunks one by one using timeout. - -* Send `sendAudioSessionEnd` event. diff --git a/package.json b/package.json index b1a7c70b..1878fb9b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@inworld/nodejs-sdk", - "version": "1.1.8", + "version": "1.2.0", "license": "SEE LICENSE IN LICENSE.md", "main": "build/src/index.js", "types": "build/src/index.d.ts", @@ -30,7 +30,7 @@ "build": "tsc -p . && tsc-alias -p tsconfig.paths.json", "release:pack": "yarn build && yarn pack", "release:publish": "yarn build && yarn publish", - "test": "jest --no-cache --reporters=default --json --outputFile=build/jest-result.json", + "test": "jest --no-cache --reporters=default", "test:coverage": "jest --coverage", "lint:check": "eslint --cache \"./**/*.{js,jsx,ts,tsx}\"", "lint:fix": "yarn run lint:check --fix", diff --git a/src/clients/inworld.client.ts b/src/clients/inworld.client.ts index 98314201..4ef159f9 100644 --- a/src/clients/inworld.client.ts +++ b/src/clients/inworld.client.ts @@ -1,3 +1,4 @@ +import util = require('node:util'); import { ServiceError } from '@grpc/grpc-js'; import { CapabilitiesRequest, @@ -12,10 +13,12 @@ import { Client, ClientConfiguration, GenerateSessionTokenFn, + GetterSetter, InternalClientConfiguration, User, } from '../common/interfaces'; import { InworldPacket } from '../entities/inworld_packet.entity'; +import { Session } from '../entities/session.entity'; import { ConnectionService } from '../services/connection.service'; import { InworldConnectionService } from '../services/inworld_connection.service'; @@ -27,6 +30,7 @@ export class InworldClient { private config: InternalClientConfiguration; private generateSessionTokenFn: GenerateSessionTokenFn; + private sessionGetterSetter: GetterSetter; private onDisconnect: (() => void) | undefined; private onError: ((err: ServiceError) => void) | undefined; @@ -95,11 +99,17 @@ export class InworldClient { return this; } + setOnSession(props: GetterSetter) { + this.sessionGetterSetter = props; + + return this; + } + async generateSessionToken() { this.validateApiKey(); return new ConnectionService({ - apiKey: this.apiKey!, + apiKey: this.apiKey, }).generateSessionToken(); } @@ -116,6 +126,7 @@ export class InworldClient { onMessage: this.onMessage, onDisconnect: this.onDisconnect, generateSessionToken: this.generateSessionTokenFn, + sessionGetterSetter: this.sessionGetterSetter, }); return new InworldConnectionService(connection); @@ -126,6 +137,8 @@ export class InworldClient { .setAudio(capabilities?.audio ?? true) .setEmotions(capabilities?.emotions ?? false) .setInterruptions(capabilities?.interruptions ?? false) + .setPhonemeInfo(capabilities?.phonemes ?? false) + .setSilenceEvents(capabilities?.silence ?? false) .setText(true) .setTriggers(true); } @@ -146,3 +159,8 @@ export class InworldClient { } } } + +InworldClient.prototype.generateSessionToken = util.deprecate( + InworldClient.prototype.generateSessionToken, + 'setGenerateSessionToken() is deprecated. Use setOnSession() instead to manage session.', +); diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 35d7ef7d..3ef7a398 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -11,6 +11,8 @@ export interface Capabilities { audio?: boolean; emotions?: boolean; interruptions?: boolean; + phonemes?: boolean; + silence?: boolean; } export interface User { @@ -51,6 +53,11 @@ export interface SessionTokenProps { sessionId: string; } +export interface GetterSetter { + get: () => Awaitable | Awaitable; + set: (entity: T) => Awaitable; +} + export enum ConnectionState { ACTIVE = 'ACTIVE', ACTIVATING = 'ACTIVATING', diff --git a/src/entities/character.entity.ts b/src/entities/character.entity.ts index fcdb9430..fca873ad 100644 --- a/src/entities/character.entity.ts +++ b/src/entities/character.entity.ts @@ -1,3 +1,5 @@ +import util = require('node:util'); + export interface CharacterProps { id: string; resourceName: string; @@ -14,10 +16,10 @@ export interface Assets { } export class Character { - private id: string; - private resourceName: string; - private displayName: string; - private assets: Assets; + id: string; + resourceName: string; + displayName: string; + assets: Assets; constructor(props: CharacterProps) { this.id = props.id; @@ -42,3 +44,23 @@ export class Character { return this.assets; } } + +Character.prototype.getId = util.deprecate( + Character.prototype.getId, + 'getId() is deprecated. Use `id` property instead.', +); + +Character.prototype.getDisplayName = util.deprecate( + Character.prototype.getDisplayName, + 'getDisplayName() is deprecated. Use `displayName` property instead.', +); + +Character.prototype.getResourceName = util.deprecate( + Character.prototype.getResourceName, + 'getResourceName() is deprecated. Use `resourceName` property instead.', +); + +Character.prototype.getAssets = util.deprecate( + Character.prototype.getAssets, + 'getAssets() is deprecated. Use `assets` property instead.', +); diff --git a/src/entities/inworld_packet.entity.ts b/src/entities/inworld_packet.entity.ts index dce01d79..aedff271 100644 --- a/src/entities/inworld_packet.entity.ts +++ b/src/entities/inworld_packet.entity.ts @@ -8,6 +8,7 @@ export enum InworldPacketType { TRIGGER = 'TRIGGER', EMOTION = 'EMOTION', CONTROL = 'CONTROL', + SILENCE = 'SILENCE', CANCEL_RESPONSE = 'CANCEL_RESPONSE', } @@ -22,6 +23,7 @@ export interface InworldPacketProps { control?: ControlEvent; trigger?: TriggerEvent; emotions?: EmotionEvent; + silence?: SilenceEvent; packetId: PacketId; routing: Routing; text?: TextEvent; @@ -60,8 +62,18 @@ export interface TriggerEvent { name: string; } +export interface AdditionalPhonemeInfo { + phoneme?: string; + startOffsetS?: number; +} + export interface AudioEvent { chunk: string; + additionalPhonemeInfo?: AdditionalPhonemeInfo[]; +} + +export interface SilenceEvent { + durationMs: number; } export interface CancelResponsesEvent { @@ -86,6 +98,7 @@ export class InworldPacket { control: ControlEvent; trigger: TriggerEvent; emotions: EmotionEvent; + silence: SilenceEvent; cancelResponses: CancelResponsesEvent; constructor(props: InworldPacketProps) { @@ -114,6 +127,10 @@ export class InworldPacket { this.trigger = props.trigger; } + if (this.isSilence()) { + this.silence = props.silence; + } + if (this.isCancelResponse()) { this.cancelResponses = props.cancelResponses; } @@ -146,6 +163,10 @@ export class InworldPacket { ); } + isSilence() { + return this.type === InworldPacketType.SILENCE; + } + isCancelResponse() { return this.type === InworldPacketType.CANCEL_RESPONSE; } diff --git a/src/entities/scene.entity.ts b/src/entities/scene.entity.ts new file mode 100644 index 00000000..a2e802dd --- /dev/null +++ b/src/entities/scene.entity.ts @@ -0,0 +1,56 @@ +import { LoadSceneResponse } from '@proto/world-engine_pb'; + +import { Character } from './character.entity'; + +export interface SceneProps { + characters: Character[]; + key: string; +} + +export class Scene { + characters: Array = []; + key: string; + + constructor(props: SceneProps) { + this.characters = props.characters; + this.key = props.key; + } + + static serialize(scene: Scene) { + return JSON.stringify(scene); + } + + static deserialize(json: string) { + try { + const { characters, key } = JSON.parse(json) as SceneProps; + + return new Scene({ characters, key }); + } catch (e) {} + } + + static fromProto(proto: LoadSceneResponse) { + const characters = proto + .getAgentsList() + .map((agent: LoadSceneResponse.Agent) => { + const assets = agent.getCharacterAssets(); + + return new Character({ + id: agent.getAgentId(), + resourceName: agent.getBrainName(), + displayName: agent.getGivenName(), + assets: { + avatarImg: assets?.getAvatarImg(), + avatarImgOriginal: assets?.getAvatarImgOriginal(), + rpmModelUri: assets?.getRpmModelUri(), + rpmImageUriPortrait: assets?.getRpmImageUriPortrait(), + rpmImageUriPosture: assets?.getRpmImageUriPosture(), + }, + }); + }); + + return new Scene({ + key: proto.getKey(), + characters, + }); + } +} diff --git a/src/entities/session.entity.ts b/src/entities/session.entity.ts new file mode 100644 index 00000000..92e52afe --- /dev/null +++ b/src/entities/session.entity.ts @@ -0,0 +1,37 @@ +import { Scene } from './scene.entity'; +import { SessionToken } from './session_token.entity'; + +export interface SessionProps { + sessionToken: SessionToken; + scene: Scene; +} + +export class Session { + sessionToken: SessionToken; + scene: Scene; + + constructor(props: SessionProps) { + this.sessionToken = props.sessionToken; + this.scene = props.scene; + } + + static serialize(session: Session) { + return JSON.stringify(session); + } + + static deserialize(json: string) { + try { + const { sessionToken, scene } = JSON.parse(json) as SessionProps; + + return new Session({ + sessionToken: new SessionToken({ + token: sessionToken.token, + type: sessionToken.type, + sessionId: sessionToken.sessionId, + expirationTime: new Date(sessionToken.expirationTime), + }), + scene, + }); + } catch (e) {} + } +} diff --git a/src/entities/session_token.entity.ts b/src/entities/session_token.entity.ts index b68b9cef..1202d80d 100644 --- a/src/entities/session_token.entity.ts +++ b/src/entities/session_token.entity.ts @@ -1,10 +1,16 @@ +import util = require('node:util'); + +import { SessionAccessToken } from '@proto/ai/inworld/studio/v1alpha/tokens_pb'; + import { SessionTokenProps } from '../common/interfaces'; +const TIME_DIFF_MS = 50 * 60 * 1000; // 5 minutes + export class SessionToken { - private token: string; - private type: string; - private expirationTime: Date; - private sessionId: string; + token: string; + type: string; + expirationTime: Date; + sessionId: string; constructor(props: SessionTokenProps) { this.token = props.token; @@ -28,4 +34,63 @@ export class SessionToken { getSessionId() { return this.sessionId; } + + static isExpired(token: SessionToken) { + const expirationTime = token.expirationTime; + + return ( + new Date(expirationTime).getTime() - new Date().getTime() <= TIME_DIFF_MS + ); + } + + static serialize(token: SessionToken) { + return JSON.stringify({ + ...token, + expirationTime: token.expirationTime.toISOString(), + }); + } + + static deserialize(json: string) { + try { + const { token, type, expirationTime, sessionId } = JSON.parse( + json, + ) as SessionTokenProps; + + return new SessionToken({ + token, + type, + sessionId, + expirationTime: new Date(expirationTime), + }); + } catch (e) {} + } + + static fromProto(proto: SessionAccessToken) { + return new SessionToken({ + token: proto.getToken(), + type: proto.getType(), + expirationTime: proto.getExpirationTime().toDate(), + sessionId: proto.getSessionId(), + }); + } } + +SessionToken.prototype.getToken = util.deprecate( + SessionToken.prototype.getToken, + 'getToken() is deprecated. Use `token` property instead.', +); + +SessionToken.prototype.getType = util.deprecate( + SessionToken.prototype.getType, + 'getType() is deprecated. Use `type` property instead.', +); + +SessionToken.prototype.getSessionId = util.deprecate( + SessionToken.prototype.getSessionId, + 'getSessionId() is deprecated. Use `sessionId` property instead.', +); + +SessionToken.prototype.getExpirationTime = util.deprecate( + SessionToken.prototype.getExpirationTime, + 'getExpirationTime() is deprecated. Use `expirationTime` property instead.', +); diff --git a/src/factories/event.ts b/src/factories/event.ts index 1b5580b2..fb2c4256 100644 --- a/src/factories/event.ts +++ b/src/factories/event.ts @@ -1,5 +1,6 @@ import { Actor, + AdditionalPhonemeInfo as ProtoAdditionalPhonemeInfo, CancelResponsesEvent, ControlEvent, CustomEvent, @@ -9,6 +10,7 @@ import { Routing, TextEvent, } from '@proto/packets_pb'; +import { Duration } from 'google-protobuf/google/protobuf/duration_pb'; import { v4 } from 'uuid'; import { protoTimestamp } from '../common/helpers'; @@ -17,6 +19,7 @@ import { Character } from '../entities/character.entity'; import { EmotionBehavior } from '../entities/emotion-behavior.entity'; import { EmotionStrength } from '../entities/emotion-strength.entity'; import { + AdditionalPhonemeInfo, InworlControlType, InworldPacket, InworldPacketType, @@ -87,19 +90,21 @@ export class EventFactory { return this.protoPacket().setCancelresponses(event); } - convertToInworldPacket(packet: ProtoPacket): InworldPacket { - const packetId = packet.getPacketId(); - const routing = packet.getRouting(); + static fromProto(proto: ProtoPacket): InworldPacket { + const packetId = proto.getPacketId(); + const routing = proto.getRouting(); const source = routing.getSource(); const target = routing.getTarget(); - const type = this.getType(packet); + const type = this.getType(proto); - const textEvent = packet.getText(); - const emotionEvent = packet.getEmotion(); + const textEvent = proto.getText(); + const emotionEvent = proto.getEmotion(); + const additionalPhonemeInfo = + proto.getDataChunk()?.getAdditionalPhonemeInfoList() ?? []; return new InworldPacket({ type, - date: packet.getTimestamp().toDate().toISOString(), + date: proto.getTimestamp().toDate().toISOString(), packetId: { packetId: packetId.getPacketId(), utteranceId: packetId.getUtteranceId(), @@ -119,7 +124,7 @@ export class EventFactory { }, ...(type === InworldPacketType.TRIGGER && { trigger: { - name: packet.getCustom().getName(), + name: proto.getCustom().getName(), }, }), ...(type === InworldPacketType.TEXT && { @@ -130,12 +135,24 @@ export class EventFactory { }), ...(type === InworldPacketType.AUDIO && { audio: { - chunk: packet.getDataChunk().getChunk_asB64(), + chunk: proto.getDataChunk().getChunk_asB64(), + additionalPhonemeInfo: additionalPhonemeInfo.map( + (info: ProtoAdditionalPhonemeInfo) => + ({ + phoneme: info.getPhoneme(), + startOffsetS: this.durationToSeconds(info.getStartOffset()), + } as AdditionalPhonemeInfo), + ), }, }), ...(type === InworldPacketType.CONTROL && { control: { - type: this.getControlType(packet), + type: this.getControlType(proto), + }, + }), + ...(type === InworldPacketType.SILENCE && { + silence: { + durationMs: proto.getDataChunk().getDurationMs(), }, }), ...(type === InworldPacketType.EMOTION && { @@ -146,8 +163,8 @@ export class EventFactory { }), ...(type === InworldPacketType.CANCEL_RESPONSE && { cancelResponses: { - interactionId: packet.getCancelresponses().getInteractionId(), - utteranceId: packet.getCancelresponses().getUtteranceIdList(), + interactionId: proto.getCancelresponses().getInteractionId(), + utteranceId: proto.getCancelresponses().getUtteranceIdList(), }, }), }); @@ -165,7 +182,7 @@ export class EventFactory { const target = new Actor() .setType(Actor.Type.AGENT) - .setName(this.character?.getId()); + .setName(this.character?.id); return new Routing().setSource(source).setTarget(target); } @@ -174,13 +191,16 @@ export class EventFactory { return new PacketId().setPacketId(v4()); } - private getType(packet: ProtoPacket) { + private static getType(packet: ProtoPacket) { switch (true) { case packet.hasText(): return InworldPacketType.TEXT; case packet.hasDataChunk() && packet.getDataChunk().getType() === DataChunk.DataType.AUDIO: return InworldPacketType.AUDIO; + case packet.hasDataChunk() && + packet.getDataChunk().getType() === DataChunk.DataType.SILENCE: + return InworldPacketType.SILENCE; case packet.hasCustom(): return InworldPacketType.TRIGGER; case packet.hasControl(): @@ -194,7 +214,7 @@ export class EventFactory { } } - private getControlType(packet: ProtoPacket) { + private static getControlType(packet: ProtoPacket) { switch (packet.getControl().getAction()) { case ControlEvent.Action.INTERACTION_END: return InworlControlType.INTERACTION_END; @@ -202,4 +222,8 @@ export class EventFactory { return InworlControlType.UNKNOWN; } } + + private static durationToSeconds(duration: Duration) { + return duration.getSeconds() + duration.getNanos() / 1000000000; + } } diff --git a/src/index.ts b/src/index.ts index 65685291..a93a3ee8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,6 +37,8 @@ import { Routing, TextEvent, } from './entities/inworld_packet.entity'; +import { Scene } from './entities/scene.entity'; +import { Session } from './entities/session.entity'; import { SessionToken } from './entities/session_token.entity'; import { InworldConnectionService } from './services/inworld_connection.service'; @@ -64,7 +66,9 @@ export { InworldPacketType, PacketId, Routing, + Scene, ServiceError, + Session, SessionToken, SessionTokenProps, status, diff --git a/src/services/connection.service.ts b/src/services/connection.service.ts index 1f51f17d..e8078f53 100644 --- a/src/services/connection.service.ts +++ b/src/services/connection.service.ts @@ -1,11 +1,6 @@ import { ClientDuplexStream, ServiceError } from '@grpc/grpc-js'; import { InworldPacket as ProtoPacket } from '@proto/packets_pb'; -import { - ClientRequest, - LoadSceneResponse, - UserRequest, -} from '@proto/world-engine_pb'; -import internal = require('stream'); +import { ClientRequest, UserRequest } from '@proto/world-engine_pb'; import { clearTimeout } from 'timers'; import { @@ -13,10 +8,12 @@ import { Awaitable, ConnectionState, GenerateSessionTokenFn, + GetterSetter, InternalClientConfiguration, } from '../common/interfaces'; -import { Character } from '../entities/character.entity'; import { InworldPacket } from '../entities/inworld_packet.entity'; +import { Scene } from '../entities/scene.entity'; +import { Session } from '../entities/session.entity'; import { SessionToken } from '../entities/session_token.entity'; import { EventFactory } from '../factories/event'; import { TokenClientGrpcService } from './gprc/token_client_grpc.service'; @@ -28,6 +25,7 @@ interface ConnectionProps { user?: UserRequest; client?: ClientRequest; config?: InternalClientConfiguration; + sessionGetterSetter?: GetterSetter; onDisconnect?: () => void; onError?: (err: ServiceError) => void; onMessage?: (message: InworldPacket) => Awaitable; @@ -39,18 +37,14 @@ export interface QueueItem { afterWriting?: (packet: ProtoPacket) => void; } -const TIME_DIFF_MS = 50 * 60 * 1000; // 5 minutes - export class ConnectionService { private state: ConnectionState = ConnectionState.INACTIVE; - private scene: LoadSceneResponse; - private session: SessionToken; + private scene: Scene; + private sessionToken: SessionToken; private stream: ClientDuplexStream; private connectionProps: ConnectionProps; - private characters: Array = []; - private disconnectTimeoutId: NodeJS.Timeout; private eventFactory = new EventFactory(); @@ -58,6 +52,9 @@ export class ConnectionService { private intervals: NodeJS.Timeout[] = []; private packetQueue: QueueItem[] = []; + private tokenService = new TokenClientGrpcService(); + private engineService = new WorldEngineClientGrpcService(); + private onDisconnect: () => void; private onError: (err: ServiceError) => void; private onMessage: ((message: ProtoPacket) => Awaitable) | undefined; @@ -75,9 +72,7 @@ export class ConnectionService { if (this.connectionProps.onMessage) { this.onMessage = async (packet: ProtoPacket) => - this.connectionProps.onMessage( - this.eventFactory.convertToInworldPacket(packet), - ); + this.connectionProps.onMessage(EventFactory.fromProto(packet)); } } @@ -90,17 +85,11 @@ export class ConnectionService { } async generateSessionToken() { - const sessionToken = - await new TokenClientGrpcService().generateSessionToken( - this.connectionProps.apiKey, - ); + const proto = await this.tokenService.generateSessionToken( + this.connectionProps.apiKey, + ); - return new SessionToken({ - token: sessionToken.getToken(), - type: sessionToken.getType(), - expirationTime: sessionToken.getExpirationTime().toDate(), - sessionId: sessionToken.getSessionId(), - }); + return SessionToken.fromProto(proto); } async openManually() { @@ -136,9 +125,11 @@ export class ConnectionService { } async getCharactersList() { - await this.loadScene(); + if (!this.scene) { + await this.loadScene(); + } - return this.characters; + return this.scene.characters; } async open() { @@ -147,10 +138,9 @@ export class ConnectionService { if (this.state === ConnectionState.LOADED) { this.state = ConnectionState.ACTIVATING; - const engineService = new WorldEngineClientGrpcService(); - this.stream = engineService.session({ - session: this.session, + this.stream = this.engineService.session({ + sessionToken: this.sessionToken, onError: this.onError, onDisconnect: this.onDisconnect, ...(this.onMessage && { onMessage: this.onMessage }), @@ -177,8 +167,6 @@ export class ConnectionService { if (this.isAutoReconnected()) { await this.open(); - await this.loadCharactersList(); - return this.write(getPacket); } @@ -188,35 +176,6 @@ export class ConnectionService { } } - private async loadCharactersList() { - if (!this.scene) { - await this.loadScene(); - } - - this.characters = (this.scene?.getAgentsList() || []).map( - (agent: LoadSceneResponse.Agent) => { - const assets = agent.getCharacterAssets(); - - return new Character({ - id: agent.getAgentId(), - resourceName: agent.getBrainName(), - displayName: agent.getGivenName(), - assets: { - avatarImg: assets?.getAvatarImg(), - avatarImgOriginal: assets?.getAvatarImgOriginal(), - rpmModelUri: assets?.getRpmModelUri(), - rpmImageUriPortrait: assets?.getRpmImageUriPortrait(), - rpmImageUriPosture: assets?.getRpmImageUriPosture(), - }, - }); - }, - ); - - if (!this.getEventFactory().getCurrentCharacter() && this.characters[0]) { - this.getEventFactory().setCurrentCharacter(this.characters[0]); - } - } - private write(getPacket: () => ProtoPacket) { let packet: ProtoPacket; @@ -224,7 +183,7 @@ export class ConnectionService { new Promise((resolve) => { const done = (packet: ProtoPacket) => { this.scheduleDisconnect(); - resolve(this.getEventFactory().convertToInworldPacket(packet)); + resolve(EventFactory.fromProto(packet)); }; if (packet) { @@ -270,47 +229,42 @@ export class ConnectionService { private async loadScene() { if (this.state === ConnectionState.LOADING) return; - const { client, name, user } = this.connectionProps; - const generateSessionToken = - this.connectionProps.generateSessionToken || - this.generateSessionToken.bind(this); + let session: Session; + let changed = false; + + // Try to get session from provided storage + if (this.connectionProps.sessionGetterSetter) { + session = await this.connectionProps.sessionGetterSetter.get(); + } try { - const sessionId = this.session?.getSessionId(); - const expirationTime = this.session?.getExpirationTime(); + const sessionToken = await this.getOrLoadSessionToken( + session?.sessionToken, + ); + changed = sessionToken !== this.sessionToken || changed; - // Generate new session token is it's empty or expired - if ( - !expirationTime || - new Date(expirationTime).getTime() - new Date().getTime() <= - TIME_DIFF_MS - ) { - this.state = ConnectionState.LOADING; - this.session = await generateSessionToken(); - - // Reuse session id to keep context of previous conversation - if (sessionId) { - this.session = new SessionToken({ - sessionId, - token: this.session.getToken(), - type: this.session.getType(), - expirationTime: this.session.getExpirationTime(), - }); + this.sessionToken = sessionToken; + + if (!this.scene) { + const scene = await this.getOrLoadScene(session?.scene); + changed = scene !== this.scene || changed; + + this.scene = scene; + + if ( + !this.getEventFactory().getCurrentCharacter() && + this.scene.characters.length + ) { + this.getEventFactory().setCurrentCharacter(this.scene.characters[0]); } } - const engineService = new WorldEngineClientGrpcService(); - - if (!this.scene) { - this.scene = await engineService.loadScene({ - capabilities: this.connectionProps.config.capabilities, - client, - name, - user, - session: this.session, + // Try to save session token to provided storage + if (changed) { + this.connectionProps.sessionGetterSetter?.set({ + sessionToken: this.sessionToken, + scene: this.scene, }); - - await this.loadCharactersList(); } if ( @@ -323,6 +277,53 @@ export class ConnectionService { } } + private async getOrLoadSessionToken(savedSessionToken?: SessionToken) { + let sessionToken = savedSessionToken ?? this.sessionToken; + + const { sessionId } = sessionToken || {}; + + // Generate new session token is it's empty or expired + if (!sessionToken || SessionToken.isExpired(sessionToken)) { + this.state = ConnectionState.LOADING; + + const generateSessionToken = + this.connectionProps.generateSessionToken ?? + this.generateSessionToken.bind(this); + sessionToken = await generateSessionToken(); + + // Reuse session id to keep context of previous conversation + if (sessionId) { + sessionToken = new SessionToken({ + ...sessionToken, + sessionId, + }); + } + } + + return sessionToken; + } + + private async getOrLoadScene(savedScene?: Scene) { + let scene = savedScene; + + // Load scene + if (!scene) { + const { client, name, user } = this.connectionProps; + + const proto = await this.engineService.loadScene({ + client, + name, + user, + capabilities: this.connectionProps.config.capabilities, + sessionToken: this.sessionToken, + }); + + scene = Scene.fromProto(proto); + } + + return scene; + } + private scheduleDisconnect() { if (this.connectionProps.config?.connection?.disconnectTimeout) { this.cancelScheduler(); diff --git a/src/services/gprc/world_engine_client_grpc.service.ts b/src/services/gprc/world_engine_client_grpc.service.ts index d7760f42..1634caf2 100644 --- a/src/services/gprc/world_engine_client_grpc.service.ts +++ b/src/services/gprc/world_engine_client_grpc.service.ts @@ -19,11 +19,11 @@ export interface LoadSceneProps { name: string; client?: ClientRequest; user?: UserRequest; - session: SessionToken; + sessionToken: SessionToken; capabilities: CapabilitiesRequest; } export interface SessionProps { - session: SessionToken; + sessionToken: SessionToken; onDisconnect?: () => void; onError?: (err: ServiceError) => void; onMessage?: (message: InworldPacket) => Awaitable; @@ -40,7 +40,7 @@ export class WorldEngineClientGrpcService { ); public async loadScene(props: LoadSceneProps) { - const { name, session, user, capabilities } = props; + const { name, sessionToken, user, capabilities } = props; const request = new LoadSceneRequest(); request.setName(name); @@ -56,14 +56,14 @@ export class WorldEngineClientGrpcService { return promisify(this.client.loadScene.bind(this.client))( request, - this.getMetadata(session), + this.getMetadata(sessionToken), ); } public session(props: SessionProps) { - const { session, onDisconnect, onError, onMessage } = props; + const { sessionToken, onDisconnect, onError, onMessage } = props; - const connection = this.client.session(this.getMetadata(session)); + const connection = this.client.session(this.getMetadata(sessionToken)); if (onMessage) { connection.on('data', onMessage); @@ -83,11 +83,11 @@ export class WorldEngineClientGrpcService { return connection; } - private getMetadata(session: SessionToken) { + private getMetadata(sessionToken: SessionToken) { const metadata = new Metadata(); - metadata.add('session-id', session.getSessionId()); - metadata.add('authorization', `${session.getType()} ${session.getToken()}`); + metadata.add('session-id', sessionToken.sessionId); + metadata.add('authorization', `${sessionToken.type} ${sessionToken.token}`); return metadata; }